From c34f3254f24536885dc911951b7f73699138a18e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 8 Jul 2020 04:21:39 +0300 Subject: [PATCH 01/43] Improvements in PODArray --- src/Common/PODArray.h | 59 +++++++++++++++++++++++----- src/Common/tests/gtest_pod_array.cpp | 28 +++++++++++++ 2 files changed, 78 insertions(+), 9 deletions(-) diff --git a/src/Common/PODArray.h b/src/Common/PODArray.h index 839990ca9b4..28f3de53950 100644 --- a/src/Common/PODArray.h +++ b/src/Common/PODArray.h @@ -102,7 +102,7 @@ protected: void alloc_for_num_elements(size_t num_elements) { - alloc(roundUpToPowerOfTwoOrZero(minimum_memory_for_elements(num_elements))); + alloc(minimum_memory_for_elements(num_elements)); } template @@ -211,6 +211,13 @@ public: realloc(roundUpToPowerOfTwoOrZero(minimum_memory_for_elements(n)), std::forward(allocator_params)...); } + template + void reserve_exact(size_t n, TAllocatorParams &&... allocator_params) + { + if (n > capacity()) + realloc(minimum_memory_for_elements(n), std::forward(allocator_params)...); + } + template void resize(size_t n, TAllocatorParams &&... allocator_params) { @@ -218,6 +225,13 @@ public: resize_assume_reserved(n); } + template + void resize_exact(size_t n, TAllocatorParams &&... allocator_params) + { + reserve_exact(n, std::forward(allocator_params)...); + resize_assume_reserved(n); + } + void resize_assume_reserved(const size_t n) { c_end = c_start + byte_size(n); @@ -428,6 +442,7 @@ public: void insertSmallAllowReadWriteOverflow15(It1 from_begin, It2 from_end, TAllocatorParams &&... allocator_params) { static_assert(pad_right_ >= 15); + static_assert(sizeof(T) == sizeof(*from_begin)); insertPrepare(from_begin, from_end, std::forward(allocator_params)...); size_t bytes_to_copy = this->byte_size(from_end - from_begin); memcpySmallAllowReadWriteOverflow15(this->c_end, reinterpret_cast(&*from_begin), bytes_to_copy); @@ -438,23 +453,40 @@ public: void insert(iterator it, It1 from_begin, It2 from_end) { size_t bytes_to_copy = this->byte_size(from_end - from_begin); - size_t bytes_to_move = (end() - it) * sizeof(T); + size_t bytes_to_move = this->byte_size(end() - it); insertPrepare(from_begin, from_end); if (unlikely(bytes_to_move)) memcpy(this->c_end + bytes_to_copy - bytes_to_move, this->c_end - bytes_to_move, bytes_to_move); - memcpy(this->c_end - bytes_to_move, reinterpret_cast(&*from_begin), bytes_to_copy); + if constexpr (sizeof(*from_begin) == sizeof(T)) + { + memcpy(this->c_end - bytes_to_move, reinterpret_cast(&*from_begin), bytes_to_copy); + } + else + { + for (auto from_it = from_begin; from_it != from_end; ++from_it, ++it) + *it = *from_it; + } + this->c_end += bytes_to_copy; } template void insert_assume_reserved(It1 from_begin, It2 from_end) { - size_t bytes_to_copy = this->byte_size(from_end - from_begin); - memcpy(this->c_end, reinterpret_cast(&*from_begin), bytes_to_copy); - this->c_end += bytes_to_copy; + if constexpr (sizeof(*from_begin) == sizeof(T)) + { + size_t bytes_to_copy = this->byte_size(from_end - from_begin); + memcpy(this->c_end, reinterpret_cast(&*from_begin), bytes_to_copy); + this->c_end += bytes_to_copy; + } + else + { + for (auto it = from_begin; it != from_end; ++it) + push_back(*it); + } } template @@ -577,7 +609,7 @@ public: template void assign(size_t n, const T & x, TAllocatorParams &&... allocator_params) { - this->resize(n, std::forward(allocator_params)...); + this->resize_exact(n, std::forward(allocator_params)...); std::fill(begin(), end(), x); } @@ -586,10 +618,19 @@ public: { size_t required_capacity = from_end - from_begin; if (required_capacity > this->capacity()) - this->reserve(roundUpToPowerOfTwoOrZero(required_capacity), std::forward(allocator_params)...); + this->reserve_exact(required_capacity, std::forward(allocator_params)...); size_t bytes_to_copy = this->byte_size(required_capacity); - memcpy(this->c_start, reinterpret_cast(&*from_begin), bytes_to_copy); + if constexpr (sizeof(*from_begin) == sizeof(T)) + { + memcpy(this->c_start, reinterpret_cast(&*from_begin), bytes_to_copy); + } + else + { + auto it = begin(); + for (auto from_it = from_begin; from_it != from_end; ++from_it) + *it = *from_it; + } this->c_end = this->c_start + bytes_to_copy; } diff --git a/src/Common/tests/gtest_pod_array.cpp b/src/Common/tests/gtest_pod_array.cpp index b77bc56b59a..8c14811ddd2 100644 --- a/src/Common/tests/gtest_pod_array.cpp +++ b/src/Common/tests/gtest_pod_array.cpp @@ -33,6 +33,34 @@ TEST(Common, PODArrayInsert) EXPECT_EQ(str, std::string(chars.data(), chars.size())); } +TEST(Common, PODInsertIteratorRange) +{ + size_t size = 1 << 20; + char value = 123; + + PODArray big; + PODArray small(size, value); + + EXPECT_EQ(big.size(), 0); + EXPECT_EQ(small.size(), size); + + big.insert(small.begin(), small.end()); + + EXPECT_EQ(big.size(), size); + EXPECT_EQ(big.back(), value); + + big.assign(small.begin(), small.end()); + + EXPECT_EQ(big.size(), size); + EXPECT_EQ(big.back(), value); + + big.insert(big.begin(), small.begin(), small.end()); + + EXPECT_EQ(big.size(), size * 2); + EXPECT_EQ(big.front(), value); + EXPECT_EQ(big.back(), value); +} + TEST(Common, PODPushBackRawMany) { PODArray chars; From 671c544b7d72d4357e68d8f92b042294ca6a71b7 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 8 Jul 2020 04:24:38 +0300 Subject: [PATCH 02/43] Improvements in PODArray --- src/Common/PODArray.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Common/PODArray.h b/src/Common/PODArray.h index 28f3de53950..e8289c6c07d 100644 --- a/src/Common/PODArray.h +++ b/src/Common/PODArray.h @@ -460,7 +460,7 @@ public: if (unlikely(bytes_to_move)) memcpy(this->c_end + bytes_to_copy - bytes_to_move, this->c_end - bytes_to_move, bytes_to_move); - if constexpr (sizeof(*from_begin) == sizeof(T)) + if constexpr (std::is_same_v) { memcpy(this->c_end - bytes_to_move, reinterpret_cast(&*from_begin), bytes_to_copy); } @@ -476,7 +476,7 @@ public: template void insert_assume_reserved(It1 from_begin, It2 from_end) { - if constexpr (sizeof(*from_begin) == sizeof(T)) + if constexpr (std::is_same_v) { size_t bytes_to_copy = this->byte_size(from_end - from_begin); memcpy(this->c_end, reinterpret_cast(&*from_begin), bytes_to_copy); @@ -621,7 +621,7 @@ public: this->reserve_exact(required_capacity, std::forward(allocator_params)...); size_t bytes_to_copy = this->byte_size(required_capacity); - if constexpr (sizeof(*from_begin) == sizeof(T)) + if constexpr (std::is_same_v) { memcpy(this->c_start, reinterpret_cast(&*from_begin), bytes_to_copy); } From 3471e526ffe583312ea55727a7fbd1666081ef2d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 8 Jul 2020 04:26:44 +0300 Subject: [PATCH 03/43] Improvements in PODArray --- src/Common/PODArray.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Common/PODArray.h b/src/Common/PODArray.h index e8289c6c07d..1f97759f314 100644 --- a/src/Common/PODArray.h +++ b/src/Common/PODArray.h @@ -460,7 +460,7 @@ public: if (unlikely(bytes_to_move)) memcpy(this->c_end + bytes_to_copy - bytes_to_move, this->c_end - bytes_to_move, bytes_to_move); - if constexpr (std::is_same_v) + if constexpr (std::is_same_v>) { memcpy(this->c_end - bytes_to_move, reinterpret_cast(&*from_begin), bytes_to_copy); } @@ -476,7 +476,7 @@ public: template void insert_assume_reserved(It1 from_begin, It2 from_end) { - if constexpr (std::is_same_v) + if constexpr (std::is_same_v>) { size_t bytes_to_copy = this->byte_size(from_end - from_begin); memcpy(this->c_end, reinterpret_cast(&*from_begin), bytes_to_copy); @@ -621,7 +621,7 @@ public: this->reserve_exact(required_capacity, std::forward(allocator_params)...); size_t bytes_to_copy = this->byte_size(required_capacity); - if constexpr (std::is_same_v) + if constexpr (std::is_same_v>) { memcpy(this->c_start, reinterpret_cast(&*from_begin), bytes_to_copy); } From 491bd449555195608763e03779dbbdf899b938a2 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 8 Jul 2020 04:41:09 +0300 Subject: [PATCH 04/43] Improve performance of clang build --- src/Common/PODArray.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Common/PODArray.h b/src/Common/PODArray.h index 1f97759f314..3e17205b44d 100644 --- a/src/Common/PODArray.h +++ b/src/Common/PODArray.h @@ -204,8 +204,12 @@ public: void clear() { c_end = c_start; } + /// Always inline is for clang. GCC works fine. + /// It makes sense when "resize" called in a loop, so we want "reserve" to be inlined into "resize". + /// It improves performance of SQL queries with "materialize" of String by 23%. + /// And we don't need always inline for "reserve_exact" because calling it in a loop is already terribly slow. template - void reserve(size_t n, TAllocatorParams &&... allocator_params) + ALWAYS_INLINE void reserve(size_t n, TAllocatorParams &&... allocator_params) { if (n > capacity()) realloc(roundUpToPowerOfTwoOrZero(minimum_memory_for_elements(n)), std::forward(allocator_params)...); From a6dbe421ac494c17cac6258d5db636d72119be6b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 8 Jul 2020 05:31:29 +0300 Subject: [PATCH 05/43] Fix build --- src/Common/PODArray.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Common/PODArray.h b/src/Common/PODArray.h index 3e17205b44d..3b5a1ccd2b2 100644 --- a/src/Common/PODArray.h +++ b/src/Common/PODArray.h @@ -471,7 +471,7 @@ public: else { for (auto from_it = from_begin; from_it != from_end; ++from_it, ++it) - *it = *from_it; + new (&*it) T{*from_it}; } this->c_end += bytes_to_copy; @@ -633,7 +633,7 @@ public: { auto it = begin(); for (auto from_it = from_begin; from_it != from_end; ++from_it) - *it = *from_it; + new (&*it) T{*from_it}; } this->c_end = this->c_start + bytes_to_copy; } From c4e925434a90bc6fbb0f48b0cc9b95219030999b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 8 Jul 2020 06:29:54 +0300 Subject: [PATCH 06/43] Fix build --- src/Common/PODArray.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Common/PODArray.h b/src/Common/PODArray.h index 3b5a1ccd2b2..cc2e80e1e46 100644 --- a/src/Common/PODArray.h +++ b/src/Common/PODArray.h @@ -471,7 +471,7 @@ public: else { for (auto from_it = from_begin; from_it != from_end; ++from_it, ++it) - new (&*it) T{*from_it}; + new (&*it) T(*from_it); } this->c_end += bytes_to_copy; @@ -633,7 +633,7 @@ public: { auto it = begin(); for (auto from_it = from_begin; from_it != from_end; ++from_it) - new (&*it) T{*from_it}; + new (&*it) T(*from_it); } this->c_end = this->c_start + bytes_to_copy; } From d6c87af267521ec6301848537b13226993cd85c2 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 9 Jul 2020 02:04:42 +0300 Subject: [PATCH 07/43] Fix strange code CC @Enmk. Prove: g++ -xc++ -include vector - <<< 'int main() { return std::vector{1, 2, 3}.size(); }' --- src/Storages/tests/gtest_SplitTokenExtractor.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/tests/gtest_SplitTokenExtractor.cpp b/src/Storages/tests/gtest_SplitTokenExtractor.cpp index 9255e5ca817..cb801cff808 100644 --- a/src/Storages/tests/gtest_SplitTokenExtractor.cpp +++ b/src/Storages/tests/gtest_SplitTokenExtractor.cpp @@ -35,7 +35,7 @@ public: { const auto & param = GetParam(); const auto & source = param.source; - data = std::make_unique>(source.data(), source.data() + source.size()); + data = std::make_unique>(source.data(), source.data() + source.size()); // add predefined padding that forms tokens to ensure no reads past end of buffer. const char extra_padding[] = "this is the end \xd1\x8d\xd1\x82\xd0\xbe\xd0\xba\xd0\xbe \xd0\xbd\xd0\xb5\xd1\x86"; @@ -44,7 +44,7 @@ public: data->resize(data->size() - sizeof(extra_padding)); } - std::unique_ptr> data; + std::unique_ptr> data; }; TEST_P(SplitTokenExtractorTest, next) From 70aae6eb447ae5ab7caf4191adae7046a1776197 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 9 Jul 2020 04:21:25 +0300 Subject: [PATCH 08/43] Fix error --- src/Common/PODArray.h | 3 +++ src/Common/tests/gtest_pod_array.cpp | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/src/Common/PODArray.h b/src/Common/PODArray.h index cc2e80e1e46..af0b6280a9b 100644 --- a/src/Common/PODArray.h +++ b/src/Common/PODArray.h @@ -453,9 +453,11 @@ public: this->c_end += bytes_to_copy; } + /// Do not insert into the array a piece of itself. Because with the resize, the iterators on themselves can be invalidated. template void insert(iterator it, It1 from_begin, It2 from_end) { + size_t position = it - begin(); size_t bytes_to_copy = this->byte_size(from_end - from_begin); size_t bytes_to_move = this->byte_size(end() - it); @@ -470,6 +472,7 @@ public: } else { + it = begin() + position; for (auto from_it = from_begin; from_it != from_end; ++from_it, ++it) new (&*it) T(*from_it); } diff --git a/src/Common/tests/gtest_pod_array.cpp b/src/Common/tests/gtest_pod_array.cpp index 305d09b1047..3bd2307452b 100644 --- a/src/Common/tests/gtest_pod_array.cpp +++ b/src/Common/tests/gtest_pod_array.cpp @@ -59,6 +59,13 @@ TEST(Common, PODInsertIteratorRange) EXPECT_EQ(big.size(), size * 2); EXPECT_EQ(big.front(), value); EXPECT_EQ(big.back(), value); + + PODArray arr1{1, 2, 3, 4, 5}; + PODArray arr2{11, 12}; + + arr1.insert(arr1.begin() + 3, arr2.begin(), arr2.end()); + + EXPECT_EQ(arr1, (PODArray{1, 2, 3, 11, 12, 4, 5})); } TEST(Common, PODPushBackRawMany) From edb68b7f6b07f77411c033a16bfbde4f8f9018b4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 9 Jul 2020 04:22:52 +0300 Subject: [PATCH 09/43] Add test --- src/Common/tests/gtest_pod_array.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Common/tests/gtest_pod_array.cpp b/src/Common/tests/gtest_pod_array.cpp index 3bd2307452b..e0c0834644b 100644 --- a/src/Common/tests/gtest_pod_array.cpp +++ b/src/Common/tests/gtest_pod_array.cpp @@ -66,6 +66,10 @@ TEST(Common, PODInsertIteratorRange) arr1.insert(arr1.begin() + 3, arr2.begin(), arr2.end()); EXPECT_EQ(arr1, (PODArray{1, 2, 3, 11, 12, 4, 5})); + + arr2.insert(arr2.begin() + 1, arr1.begin(), arr1.end()); + + EXPECT_EQ(arr2, (PODArray{11, 1, 2, 3, 11, 12, 4, 5, 12})); } TEST(Common, PODPushBackRawMany) From befbae585b9608a529e9a4ef76337c679ecea0d6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 12 Jul 2020 21:31:17 +0300 Subject: [PATCH 10/43] Revert the only change that should affect performance in gcc --- src/Common/PODArray.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/PODArray.h b/src/Common/PODArray.h index af0b6280a9b..a29d7759513 100644 --- a/src/Common/PODArray.h +++ b/src/Common/PODArray.h @@ -102,7 +102,7 @@ protected: void alloc_for_num_elements(size_t num_elements) { - alloc(minimum_memory_for_elements(num_elements)); + alloc(roundUpToPowerOfTwoOrZero(minimum_memory_for_elements(num_elements))); } template From 0c237de00dd5c145435876a24e7aaed96541a4bd Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 15 Jul 2020 00:31:50 +0300 Subject: [PATCH 11/43] Fix build --- src/Common/PODArray.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/PODArray.h b/src/Common/PODArray.h index a29d7759513..655e6d804f6 100644 --- a/src/Common/PODArray.h +++ b/src/Common/PODArray.h @@ -457,7 +457,7 @@ public: template void insert(iterator it, It1 from_begin, It2 from_end) { - size_t position = it - begin(); + size_t position [[maybe_unused]] = it - begin(); size_t bytes_to_copy = this->byte_size(from_end - from_begin); size_t bytes_to_move = this->byte_size(end() - it); From 52146218302e7ab2ee6bd7c1172fd7e9fa32f5bc Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 30 Jul 2020 18:20:47 +0300 Subject: [PATCH 12/43] [draft] only fix int16->int32 insert in PODArray --- src/Common/PODArray.h | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/src/Common/PODArray.h b/src/Common/PODArray.h index 655e6d804f6..22d81145ae2 100644 --- a/src/Common/PODArray.h +++ b/src/Common/PODArray.h @@ -215,13 +215,6 @@ public: realloc(roundUpToPowerOfTwoOrZero(minimum_memory_for_elements(n)), std::forward(allocator_params)...); } - template - void reserve_exact(size_t n, TAllocatorParams &&... allocator_params) - { - if (n > capacity()) - realloc(minimum_memory_for_elements(n), std::forward(allocator_params)...); - } - template void resize(size_t n, TAllocatorParams &&... allocator_params) { @@ -229,13 +222,6 @@ public: resize_assume_reserved(n); } - template - void resize_exact(size_t n, TAllocatorParams &&... allocator_params) - { - reserve_exact(n, std::forward(allocator_params)...); - resize_assume_reserved(n); - } - void resize_assume_reserved(const size_t n) { c_end = c_start + byte_size(n); @@ -616,7 +602,7 @@ public: template void assign(size_t n, const T & x, TAllocatorParams &&... allocator_params) { - this->resize_exact(n, std::forward(allocator_params)...); + this->resize(n, std::forward(allocator_params)...); std::fill(begin(), end(), x); } @@ -625,7 +611,7 @@ public: { size_t required_capacity = from_end - from_begin; if (required_capacity > this->capacity()) - this->reserve_exact(required_capacity, std::forward(allocator_params)...); + this->reserve(roundUpToPowerOfTwoOrZero(required_capacity), std::forward(allocator_params)...); size_t bytes_to_copy = this->byte_size(required_capacity); if constexpr (std::is_same_v>) From c8ed684a725eb93c1d88dc2b96b52edfb7cb7bab Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 2 Aug 2020 05:35:44 +0300 Subject: [PATCH 13/43] Fix assert when decimal has too large negative exponent --- src/IO/readDecimalText.h | 21 +++++++++++++------ ...imal_parse_big_negative_exponent.reference | 6 ++++++ ...25_decimal_parse_big_negative_exponent.sql | 10 +++++++++ 3 files changed, 31 insertions(+), 6 deletions(-) create mode 100644 tests/queries/0_stateless/01425_decimal_parse_big_negative_exponent.reference create mode 100644 tests/queries/0_stateless/01425_decimal_parse_big_negative_exponent.sql diff --git a/src/IO/readDecimalText.h b/src/IO/readDecimalText.h index 44405357a10..06c32ee608a 100644 --- a/src/IO/readDecimalText.h +++ b/src/IO/readDecimalText.h @@ -160,12 +160,21 @@ inline void readDecimalText(ReadBuffer & buf, T & x, uint32_t precision, uint32_ if (static_cast(scale) + exponent < 0) { - /// Too many digits after point. Just cut off excessive digits. - auto divisor = intExp10OfSize(-exponent - static_cast(scale)); - assert(divisor > 0); /// This is for Clang Static Analyzer. It is not smart enough to infer it automatically. - x.value /= divisor; - scale = 0; - return; + if (-exponent >= std::numeric_limits::digits10) + { + /// Too big negative exponent + x.value = 0; + scale = 0; + return; + } + else + { + /// Too many digits after point. Just cut off excessive digits. + auto divisor = intExp10OfSize(-exponent - static_cast(scale)); + x.value /= divisor; + scale = 0; + return; + } } scale += exponent; diff --git a/tests/queries/0_stateless/01425_decimal_parse_big_negative_exponent.reference b/tests/queries/0_stateless/01425_decimal_parse_big_negative_exponent.reference new file mode 100644 index 00000000000..7f41fb5addb --- /dev/null +++ b/tests/queries/0_stateless/01425_decimal_parse_big_negative_exponent.reference @@ -0,0 +1,6 @@ +1E-9 0 +1E-8 0 +1E-7 0 +1e-7 0 +1E-9 0.000000001 +1E-10 0.000000000 diff --git a/tests/queries/0_stateless/01425_decimal_parse_big_negative_exponent.sql b/tests/queries/0_stateless/01425_decimal_parse_big_negative_exponent.sql new file mode 100644 index 00000000000..7f276d1f8d4 --- /dev/null +++ b/tests/queries/0_stateless/01425_decimal_parse_big_negative_exponent.sql @@ -0,0 +1,10 @@ +SELECT '-1E9-1E9-1E9-1E9' AS x, toDecimal32(x, 0); -- { serverError 6 } +SELECT '-1E9' AS x, toDecimal32(x, 0); -- { serverError 69 } +SELECT '1E-9' AS x, toDecimal32(x, 0); +SELECT '1E-8' AS x, toDecimal32(x, 0); +SELECT '1E-7' AS x, toDecimal32(x, 0); +SELECT '1e-7' AS x, toDecimal32(x, 0); +SELECT '1E-9' AS x, toDecimal32(x, 9); +SELECT '1E-9' AS x, toDecimal32(x, 10); -- { serverError 69 } +SELECT '1E-10' AS x, toDecimal32(x, 10); -- { serverError 69 } +SELECT '1E-10' AS x, toDecimal32(x, 9); From 503694a84a6312a4b8237c0cd21aee7f331f3d32 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 2 Aug 2020 23:19:26 +0300 Subject: [PATCH 14/43] Fix error --- src/IO/readDecimalText.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/IO/readDecimalText.h b/src/IO/readDecimalText.h index 06c32ee608a..dcd70c29f68 100644 --- a/src/IO/readDecimalText.h +++ b/src/IO/readDecimalText.h @@ -160,7 +160,9 @@ inline void readDecimalText(ReadBuffer & buf, T & x, uint32_t precision, uint32_ if (static_cast(scale) + exponent < 0) { - if (-exponent >= std::numeric_limits::digits10) + auto divisor_exp = -exponent - static_cast(scale); + + if (divisor_exp >= std::numeric_limits::digits10) { /// Too big negative exponent x.value = 0; @@ -170,7 +172,7 @@ inline void readDecimalText(ReadBuffer & buf, T & x, uint32_t precision, uint32_ else { /// Too many digits after point. Just cut off excessive digits. - auto divisor = intExp10OfSize(-exponent - static_cast(scale)); + auto divisor = intExp10OfSize(divisor_exp); x.value /= divisor; scale = 0; return; From 06d997cd38e04d779e325cab8ee37a6e2cbfb32e Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 3 Aug 2020 04:48:05 +0300 Subject: [PATCH 15/43] Update readDecimalText.h --- src/IO/readDecimalText.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/IO/readDecimalText.h b/src/IO/readDecimalText.h index dcd70c29f68..e3569cb471c 100644 --- a/src/IO/readDecimalText.h +++ b/src/IO/readDecimalText.h @@ -173,6 +173,7 @@ inline void readDecimalText(ReadBuffer & buf, T & x, uint32_t precision, uint32_ { /// Too many digits after point. Just cut off excessive digits. auto divisor = intExp10OfSize(divisor_exp); + assert(divisor > 0); /// This is for Clang Static Analyzer. It is not smart enough to infer it automatically. x.value /= divisor; scale = 0; return; From 59ed586b691c1764aea8a875110b021542e185b7 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Mon, 3 Aug 2020 20:15:08 +0300 Subject: [PATCH 16/43] fixup --- src/Common/PODArray.h | 67 ++++++++++++++++++++++--------------------- 1 file changed, 35 insertions(+), 32 deletions(-) diff --git a/src/Common/PODArray.h b/src/Common/PODArray.h index 22d81145ae2..e7a1447a986 100644 --- a/src/Common/PODArray.h +++ b/src/Common/PODArray.h @@ -21,6 +21,20 @@ #include +template +constexpr bool allow_memcpy = std::is_same_v; + +template +constexpr bool allow_memcpy = std::is_same_v + || std::is_same_v || std::is_same_v; + +template +constexpr bool allow_memcpy = std::is_same_v + || std::is_same_v || std::is_same_v; + +template +constexpr bool allow_memcpy = std::is_same_v + || std::is_same_v || std::is_same_v; namespace DB { @@ -317,7 +331,15 @@ public: insert(from_begin, from_end); } - PODArray(std::initializer_list il) : PODArray(std::begin(il), std::end(il)) {} + PODArray(std::initializer_list il) + { + reserve(std::size(il)); + + for (const auto & x : il) + { + push_back(x); + } + } PODArray(PODArray && other) { @@ -443,6 +465,8 @@ public: template void insert(iterator it, It1 from_begin, It2 from_end) { + static_assert(allow_memcpy, std::decay_t>); + size_t position [[maybe_unused]] = it - begin(); size_t bytes_to_copy = this->byte_size(from_end - from_begin); size_t bytes_to_move = this->byte_size(end() - it); @@ -452,16 +476,7 @@ public: if (unlikely(bytes_to_move)) memcpy(this->c_end + bytes_to_copy - bytes_to_move, this->c_end - bytes_to_move, bytes_to_move); - if constexpr (std::is_same_v>) - { - memcpy(this->c_end - bytes_to_move, reinterpret_cast(&*from_begin), bytes_to_copy); - } - else - { - it = begin() + position; - for (auto from_it = from_begin; from_it != from_end; ++from_it, ++it) - new (&*it) T(*from_it); - } + memcpy(this->c_end - bytes_to_move, reinterpret_cast(&*from_begin), bytes_to_copy); this->c_end += bytes_to_copy; } @@ -469,17 +484,11 @@ public: template void insert_assume_reserved(It1 from_begin, It2 from_end) { - if constexpr (std::is_same_v>) - { - size_t bytes_to_copy = this->byte_size(from_end - from_begin); - memcpy(this->c_end, reinterpret_cast(&*from_begin), bytes_to_copy); - this->c_end += bytes_to_copy; - } - else - { - for (auto it = from_begin; it != from_end; ++it) - push_back(*it); - } + static_assert(allow_memcpy, std::decay_t>); + + size_t bytes_to_copy = this->byte_size(from_end - from_begin); + memcpy(this->c_end, reinterpret_cast(&*from_begin), bytes_to_copy); + this->c_end += bytes_to_copy; } template @@ -609,21 +618,15 @@ public: template void assign(It1 from_begin, It2 from_end, TAllocatorParams &&... allocator_params) { + static_assert(allow_memcpy, std::decay_t>); + size_t required_capacity = from_end - from_begin; if (required_capacity > this->capacity()) this->reserve(roundUpToPowerOfTwoOrZero(required_capacity), std::forward(allocator_params)...); size_t bytes_to_copy = this->byte_size(required_capacity); - if constexpr (std::is_same_v>) - { - memcpy(this->c_start, reinterpret_cast(&*from_begin), bytes_to_copy); - } - else - { - auto it = begin(); - for (auto from_it = from_begin; from_it != from_end; ++from_it) - new (&*it) T(*from_it); - } + memcpy(this->c_start, reinterpret_cast(&*from_begin), bytes_to_copy); + this->c_end = this->c_start + bytes_to_copy; } From 0d4b8bc22677fb2e0e818c5e3f28d45ced5452db Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Mon, 3 Aug 2020 20:22:59 +0300 Subject: [PATCH 17/43] fixup --- src/Common/PODArray.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Common/PODArray.h b/src/Common/PODArray.h index e7a1447a986..36436f7667e 100644 --- a/src/Common/PODArray.h +++ b/src/Common/PODArray.h @@ -333,11 +333,11 @@ public: PODArray(std::initializer_list il) { - reserve(std::size(il)); + this->reserve(std::size(il)); for (const auto & x : il) { - push_back(x); + this->push_back(x); } } From 0d947b64bdccf69854cbfd7b939a89ae65ec0792 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 5 Aug 2020 02:23:31 +0300 Subject: [PATCH 18/43] fixup --- src/Common/tests/gtest_pod_array.cpp | 39 ---------------------------- 1 file changed, 39 deletions(-) diff --git a/src/Common/tests/gtest_pod_array.cpp b/src/Common/tests/gtest_pod_array.cpp index e0c0834644b..988a3e649ba 100644 --- a/src/Common/tests/gtest_pod_array.cpp +++ b/src/Common/tests/gtest_pod_array.cpp @@ -33,45 +33,6 @@ TEST(Common, PODArrayInsert) EXPECT_EQ(str, std::string(chars.data(), chars.size())); } -TEST(Common, PODInsertIteratorRange) -{ - size_t size = 1 << 20; - char value = 123; - - PODArray big; - PODArray small(size, value); - - EXPECT_EQ(big.size(), 0); - EXPECT_EQ(small.size(), size); - - big.insert(small.begin(), small.end()); - - EXPECT_EQ(big.size(), size); - EXPECT_EQ(big.back(), value); - - big.assign(small.begin(), small.end()); - - EXPECT_EQ(big.size(), size); - EXPECT_EQ(big.back(), value); - - big.insert(big.begin(), small.begin(), small.end()); - - EXPECT_EQ(big.size(), size * 2); - EXPECT_EQ(big.front(), value); - EXPECT_EQ(big.back(), value); - - PODArray arr1{1, 2, 3, 4, 5}; - PODArray arr2{11, 12}; - - arr1.insert(arr1.begin() + 3, arr2.begin(), arr2.end()); - - EXPECT_EQ(arr1, (PODArray{1, 2, 3, 11, 12, 4, 5})); - - arr2.insert(arr2.begin() + 1, arr1.begin(), arr1.end()); - - EXPECT_EQ(arr2, (PODArray{11, 1, 2, 3, 11, 12, 4, 5, 12})); -} - TEST(Common, PODPushBackRawMany) { PODArray chars; From ead62bc9d7c0572054b20f54f3a3a47ac0399f6b Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 5 Aug 2020 17:12:41 +0300 Subject: [PATCH 19/43] Remove everything except static_assert for same type --- src/Common/PODArray.h | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/Common/PODArray.h b/src/Common/PODArray.h index 36436f7667e..6de85ea7556 100644 --- a/src/Common/PODArray.h +++ b/src/Common/PODArray.h @@ -218,12 +218,8 @@ public: void clear() { c_end = c_start; } - /// Always inline is for clang. GCC works fine. - /// It makes sense when "resize" called in a loop, so we want "reserve" to be inlined into "resize". - /// It improves performance of SQL queries with "materialize" of String by 23%. - /// And we don't need always inline for "reserve_exact" because calling it in a loop is already terribly slow. template - ALWAYS_INLINE void reserve(size_t n, TAllocatorParams &&... allocator_params) + void reserve(size_t n, TAllocatorParams &&... allocator_params) { if (n > capacity()) realloc(roundUpToPowerOfTwoOrZero(minimum_memory_for_elements(n)), std::forward(allocator_params)...); @@ -467,7 +463,6 @@ public: { static_assert(allow_memcpy, std::decay_t>); - size_t position [[maybe_unused]] = it - begin(); size_t bytes_to_copy = this->byte_size(from_end - from_begin); size_t bytes_to_move = this->byte_size(end() - it); From e7d3154d38b8b8e758c170f09ad17d07f942c0dd Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Thu, 6 Aug 2020 02:37:39 +0300 Subject: [PATCH 20/43] fix queries with order by prefix of primary key and constant columns --- .../Transforms/FinishSortingTransform.cpp | 23 +++++++------------ ...01431_finish_sorting_with_consts.reference | 3 +++ .../01431_finish_sorting_with_consts.sql | 7 ++++++ 3 files changed, 18 insertions(+), 15 deletions(-) create mode 100644 tests/queries/0_stateless/01431_finish_sorting_with_consts.reference create mode 100644 tests/queries/0_stateless/01431_finish_sorting_with_consts.sql diff --git a/src/Processors/Transforms/FinishSortingTransform.cpp b/src/Processors/Transforms/FinishSortingTransform.cpp index b58b008339d..29d0170d907 100644 --- a/src/Processors/Transforms/FinishSortingTransform.cpp +++ b/src/Processors/Transforms/FinishSortingTransform.cpp @@ -25,23 +25,16 @@ FinishSortingTransform::FinishSortingTransform( const SortDescription & description_to_sort_, size_t max_merged_block_size_, UInt64 limit_) : SortingTransform(header, description_to_sort_, max_merged_block_size_, limit_) - , description_sorted(description_sorted_) { - const auto & sample = inputs.front().getHeader(); - - /// Replace column names to column position in description_sorted. - for (auto & column_description : description_sorted) - { - if (!column_description.column_name.empty()) - { - column_description.column_number = sample.getPositionByName(column_description.column_name); - column_description.column_name.clear(); - } - } - - if (!isPrefix(description_sorted, description)) - throw Exception("Can`t finish sorting. SortDescription of already sorted stream is not prefix of " + /// Check for sanity non-modified descriptions + if (!isPrefix(description_sorted_, description_to_sort_)) + throw Exception("Can't finish sorting. SortDescription of already sorted stream is not prefix of " "SortDescription needed to sort", ErrorCodes::LOGICAL_ERROR); + + /// The target description is modified in SortingTransform constructor. + /// To avoid doing the same actions with description_sorted just copy it from prefix of target description. + size_t prefix_size = description_sorted_.size(); + description_sorted.assign(description.begin(), description.begin() + prefix_size); } static bool less(const Columns & lhs, const Columns & rhs, size_t i, size_t j, const SortDescription & descr) diff --git a/tests/queries/0_stateless/01431_finish_sorting_with_consts.reference b/tests/queries/0_stateless/01431_finish_sorting_with_consts.reference new file mode 100644 index 00000000000..1627e09150f --- /dev/null +++ b/tests/queries/0_stateless/01431_finish_sorting_with_consts.reference @@ -0,0 +1,3 @@ +1 2020-05-05 01:00:00 0 +1 2020-05-05 01:00:00 1 +1 2020-05-05 01:00:00 2 diff --git a/tests/queries/0_stateless/01431_finish_sorting_with_consts.sql b/tests/queries/0_stateless/01431_finish_sorting_with_consts.sql new file mode 100644 index 00000000000..8071f4f6ced --- /dev/null +++ b/tests/queries/0_stateless/01431_finish_sorting_with_consts.sql @@ -0,0 +1,7 @@ +DROP TABLE IF EXISTS pk_func; + +CREATE TABLE pk_func (`d` DateTime, `ui` UInt32 ) ENGINE = MergeTree ORDER BY toDate(d); +INSERT INTO pk_func SELECT '2020-05-05 01:00:00', number FROM numbers(1000); +SELECT 1, * FROM pk_func ORDER BY toDate(d) ASC, ui ASC LIMIT 3; + +DROP TABLE IF EXISTS pk_func; From dadcca6c08d2a2c871057c9e0e703f8291491379 Mon Sep 17 00:00:00 2001 From: hcz Date: Thu, 6 Aug 2020 13:58:08 +0800 Subject: [PATCH 21/43] Fix empty output of arrow related formats --- .../Formats/Impl/ArrowBlockOutputFormat.cpp | 14 +++++++++----- .../Formats/Impl/ParquetBlockOutputFormat.cpp | 12 ++++++++---- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp index 150cac49d73..69e42c0ad37 100644 --- a/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp @@ -43,13 +43,17 @@ void ArrowBlockOutputFormat::consume(Chunk chunk) void ArrowBlockOutputFormat::finalize() { - if (writer) + if (!writer) { - auto status = writer->Close(); - if (!status.ok()) - throw Exception(ErrorCodes::UNKNOWN_EXCEPTION, - "Error while closing a table: {}", status.ToString()); + const Block & header = getPort(PortKind::Main).getHeader(); + + consume(Chunk(header.columns(), 0)); } + + auto status = writer->Close(); + if (!status.ok()) + throw Exception(ErrorCodes::UNKNOWN_EXCEPTION, + "Error while closing a table: {}", status.ToString()); } void ArrowBlockOutputFormat::prepareWriter(const std::shared_ptr & schema) diff --git a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp index ae0a58f8303..436f97ccc3c 100644 --- a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp @@ -66,12 +66,16 @@ void ParquetBlockOutputFormat::consume(Chunk chunk) void ParquetBlockOutputFormat::finalize() { - if (file_writer) + if (!file_writer) { - auto status = file_writer->Close(); - if (!status.ok()) - throw Exception{"Error while closing a table: " + status.ToString(), ErrorCodes::UNKNOWN_EXCEPTION}; + const Block & header = getPort(PortKind::Main).getHeader(); + + consume(Chunk(header.columns(), 0)); } + + auto status = file_writer->Close(); + if (!status.ok()) + throw Exception{"Error while closing a table: " + status.ToString(), ErrorCodes::UNKNOWN_EXCEPTION}; } void registerOutputFormatProcessorParquet(FormatFactory & factory) From b774036d6e19d01af094ef26d271bf03071eb2d4 Mon Sep 17 00:00:00 2001 From: hcz Date: Thu, 6 Aug 2020 16:57:50 +0800 Subject: [PATCH 22/43] Add test case --- .../01429_empty_arrow_and_parquet.reference | 6 +++++ .../01429_empty_arrow_and_parquet.sh | 23 +++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 tests/queries/0_stateless/01429_empty_arrow_and_parquet.reference create mode 100755 tests/queries/0_stateless/01429_empty_arrow_and_parquet.sh diff --git a/tests/queries/0_stateless/01429_empty_arrow_and_parquet.reference b/tests/queries/0_stateless/01429_empty_arrow_and_parquet.reference new file mode 100644 index 00000000000..167851e057b --- /dev/null +++ b/tests/queries/0_stateless/01429_empty_arrow_and_parquet.reference @@ -0,0 +1,6 @@ +1 +1 +2 +2 +3 +3 diff --git a/tests/queries/0_stateless/01429_empty_arrow_and_parquet.sh b/tests/queries/0_stateless/01429_empty_arrow_and_parquet.sh new file mode 100755 index 00000000000..893cadfe17c --- /dev/null +++ b/tests/queries/0_stateless/01429_empty_arrow_and_parquet.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -e + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CUR_DIR"/../shell_config.sh + + +${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS test_empty_data" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE test_empty_data (x Int8) ENGINE = Memory" + +(echo "INSERT INTO test_empty_data FORMAT Arrow" && ${CLICKHOUSE_CLIENT} --query="SELECT 1 AS x FORMAT Arrow") | ${CLICKHOUSE_CLIENT} +${CLICKHOUSE_CLIENT} --query="SELECT count() FROM test_empty_data" +(echo "INSERT INTO test_empty_data FORMAT Arrow" && ${CLICKHOUSE_CLIENT} --query="SELECT 1 AS x LIMIT 0 FORMAT Arrow") | ${CLICKHOUSE_CLIENT} +${CLICKHOUSE_CLIENT} --query="SELECT count() FROM test_empty_data" +(echo "INSERT INTO test_empty_data FORMAT ArrowStream" && ${CLICKHOUSE_CLIENT} --query="SELECT 1 AS x FORMAT ArrowStream") | ${CLICKHOUSE_CLIENT} +${CLICKHOUSE_CLIENT} --query="SELECT count() FROM test_empty_data" +(echo "INSERT INTO test_empty_data FORMAT ArrowStream" && ${CLICKHOUSE_CLIENT} --query="SELECT 1 AS x LIMIT 0 FORMAT ArrowStream") | ${CLICKHOUSE_CLIENT} +${CLICKHOUSE_CLIENT} --query="SELECT count() FROM test_empty_data" +(echo "INSERT INTO test_empty_data FORMAT Parquet" && ${CLICKHOUSE_CLIENT} --query="SELECT 1 AS x FORMAT Parquet") | ${CLICKHOUSE_CLIENT} +${CLICKHOUSE_CLIENT} --query="SELECT count() FROM test_empty_data" +(echo "INSERT INTO test_empty_data FORMAT Parquet" && ${CLICKHOUSE_CLIENT} --query="SELECT 1 AS x LIMIT 0 FORMAT Parquet") | ${CLICKHOUSE_CLIENT} +${CLICKHOUSE_CLIENT} --query="SELECT count() FROM test_empty_data" From b28e121967de400b77ee0958678d32222ce3d900 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 6 Aug 2020 15:54:14 +0300 Subject: [PATCH 23/43] Fix tuple with NULL exement comparison. --- src/Functions/FunctionsComparison.h | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Functions/FunctionsComparison.h b/src/Functions/FunctionsComparison.h index a2d39814f7d..18bd2d32a78 100644 --- a/src/Functions/FunctionsComparison.h +++ b/src/Functions/FunctionsComparison.h @@ -13,6 +13,7 @@ #include #include +#include #include #include #include @@ -1135,17 +1136,22 @@ public: FunctionComparison::create(context))); bool has_nullable = false; + bool has_null = false; size_t size = left_tuple->getElements().size(); for (size_t i = 0; i < size; ++i) { ColumnsWithTypeAndName args = {{nullptr, left_tuple->getElements()[i], ""}, {nullptr, right_tuple->getElements()[i], ""}}; - has_nullable = has_nullable || adaptor.build(args)->getReturnType()->isNullable(); + auto element_type = adaptor.build(args)->getReturnType(); + has_nullable = has_nullable || element_type->isNullable(); + has_null = has_null || element_type->onlyNull(); } /// If any element comparison is nullable, return type will also be nullable. /// We useDefaultImplementationForNulls, but it doesn't work for tuples. + if (has_null) + return std::make_shared(std::make_shared()); if (has_nullable) return std::make_shared(std::make_shared()); } From e4a40d0e09776dfd7e815ed1984f2103cef7941d Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 6 Aug 2020 16:16:31 +0300 Subject: [PATCH 24/43] Update tests. --- tests/queries/0_stateless/00250_tuple_comparison.reference | 1 + tests/queries/0_stateless/00250_tuple_comparison.sql | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/tests/queries/0_stateless/00250_tuple_comparison.reference b/tests/queries/0_stateless/00250_tuple_comparison.reference index 05547aaf6a4..ea2477e99b0 100644 --- a/tests/queries/0_stateless/00250_tuple_comparison.reference +++ b/tests/queries/0_stateless/00250_tuple_comparison.reference @@ -13,3 +13,4 @@ 1 0 0 0 1 1 0 1 1 0 1 0 0 1 0 1 0 1 +\N \N \N diff --git a/tests/queries/0_stateless/00250_tuple_comparison.sql b/tests/queries/0_stateless/00250_tuple_comparison.sql index 8f5763e2ebb..46275bbe06e 100644 --- a/tests/queries/0_stateless/00250_tuple_comparison.sql +++ b/tests/queries/0_stateless/00250_tuple_comparison.sql @@ -103,3 +103,7 @@ SELECT tuple(2) > tuple(1), tuple(2) <= tuple(1), tuple(2) >= tuple(1); +SELECT + tuple(NULL) < tuple(1), + tuple(NULL) = tuple(1), + tuple(NULL) <= tuple(1); From 9ca0284c25978f9cbd60d31fc4845a9585659212 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 6 Aug 2020 16:17:54 +0300 Subject: [PATCH 25/43] Update test. --- tests/queries/0_stateless/00250_tuple_comparison.reference | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/00250_tuple_comparison.reference b/tests/queries/0_stateless/00250_tuple_comparison.reference index ea2477e99b0..ee42de3c03a 100644 --- a/tests/queries/0_stateless/00250_tuple_comparison.reference +++ b/tests/queries/0_stateless/00250_tuple_comparison.reference @@ -13,4 +13,4 @@ 1 0 0 0 1 1 0 1 1 0 1 0 0 1 0 1 0 1 -\N \N \N +\N \N \N From e7e71b861543aedef6178dd263d7114a8d2718c0 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 6 Aug 2020 16:34:23 +0300 Subject: [PATCH 26/43] Use the memcpy predicate from 6f8a274ba04fdec65f168ebe63f1acd715bdbd72 --- src/Common/PODArray.h | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/src/Common/PODArray.h b/src/Common/PODArray.h index 6de85ea7556..1fddea4507b 100644 --- a/src/Common/PODArray.h +++ b/src/Common/PODArray.h @@ -21,20 +21,15 @@ #include -template -constexpr bool allow_memcpy = std::is_same_v; - -template -constexpr bool allow_memcpy = std::is_same_v - || std::is_same_v || std::is_same_v; - -template -constexpr bool allow_memcpy = std::is_same_v - || std::is_same_v || std::is_same_v; - -template -constexpr bool allow_memcpy = std::is_same_v - || std::is_same_v || std::is_same_v; +/** Whether we can use memcpy instead of a loop with assignment to T from U. + * It is Ok if types are the same. And if types are integral and of the same size, + * example: char, signed char, unsigned char. + * It's not Ok for int and float. + * Don't forget to apply std::decay when using this constexpr. + */ +template +constexpr bool memcpy_can_be_used_for_assignment = std::is_same_v + || (std::is_integral_v && std::is_integral_v && sizeof(T) == sizeof(U)); namespace DB { @@ -461,7 +456,7 @@ public: template void insert(iterator it, It1 from_begin, It2 from_end) { - static_assert(allow_memcpy, std::decay_t>); + static_assert(memcpy_can_be_used_for_assignment, std::decay_t>); size_t bytes_to_copy = this->byte_size(from_end - from_begin); size_t bytes_to_move = this->byte_size(end() - it); @@ -479,7 +474,7 @@ public: template void insert_assume_reserved(It1 from_begin, It2 from_end) { - static_assert(allow_memcpy, std::decay_t>); + static_assert(memcpy_can_be_used_for_assignment, std::decay_t>); size_t bytes_to_copy = this->byte_size(from_end - from_begin); memcpy(this->c_end, reinterpret_cast(&*from_begin), bytes_to_copy); @@ -613,7 +608,7 @@ public: template void assign(It1 from_begin, It2 from_end, TAllocatorParams &&... allocator_params) { - static_assert(allow_memcpy, std::decay_t>); + static_assert(memcpy_can_be_used_for_assignment, std::decay_t>); size_t required_capacity = from_end - from_begin; if (required_capacity > this->capacity()) From 38c1d3b061aa69d7fa7f7bc099a6abea7d9bf7fc Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 6 Aug 2020 16:47:08 +0300 Subject: [PATCH 27/43] Fix build. --- src/Functions/FunctionsComparison.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Functions/FunctionsComparison.h b/src/Functions/FunctionsComparison.h index 18bd2d32a78..17893416c58 100644 --- a/src/Functions/FunctionsComparison.h +++ b/src/Functions/FunctionsComparison.h @@ -885,6 +885,13 @@ private: if (tuple_size != typeid_cast(*c1.type).getElements().size()) throw Exception("Cannot compare tuples of different sizes.", ErrorCodes::BAD_ARGUMENTS); + auto & res = block.getByPosition(result); + if (res.type->onlyNull()) + { + res.column = res.type->createColumnConstWithDefaultValue(input_rows_count); + return; + } + ColumnsWithTypeAndName x(tuple_size); ColumnsWithTypeAndName y(tuple_size); From 06abfa367ed1448b8007963db45efb1ff4f7862f Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 6 Aug 2020 16:59:49 +0300 Subject: [PATCH 28/43] Update test. --- tests/queries/0_stateless/00250_tuple_comparison.reference | 2 +- tests/queries/0_stateless/00250_tuple_comparison.sql | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/00250_tuple_comparison.reference b/tests/queries/0_stateless/00250_tuple_comparison.reference index ee42de3c03a..078592f1f2a 100644 --- a/tests/queries/0_stateless/00250_tuple_comparison.reference +++ b/tests/queries/0_stateless/00250_tuple_comparison.reference @@ -13,4 +13,4 @@ 1 0 0 0 1 1 0 1 1 0 1 0 0 1 0 1 0 1 -\N \N \N +\N \N \N \N \N diff --git a/tests/queries/0_stateless/00250_tuple_comparison.sql b/tests/queries/0_stateless/00250_tuple_comparison.sql index 46275bbe06e..2ee29ef5863 100644 --- a/tests/queries/0_stateless/00250_tuple_comparison.sql +++ b/tests/queries/0_stateless/00250_tuple_comparison.sql @@ -106,4 +106,6 @@ SELECT SELECT tuple(NULL) < tuple(1), tuple(NULL) = tuple(1), - tuple(NULL) <= tuple(1); + tuple(NULL) <= tuple(1), + tuple(1, NULL) = tuple(2, 1), + tuple(1, NULL) < tuple(2, 1); From 241017bc1fa07a3faf7c6543ecc41905630bf03f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 7 Aug 2020 02:52:18 +0300 Subject: [PATCH 29/43] Fix error in parseDateTimeBestEffort --- src/Common/BitHelpers.h | 2 ++ src/IO/parseDateTimeBestEffort.cpp | 1 + .../01432_parse_date_time_best_effort_timestamp.reference | 3 +++ .../01432_parse_date_time_best_effort_timestamp.sql | 3 +++ 4 files changed, 9 insertions(+) create mode 100644 tests/queries/0_stateless/01432_parse_date_time_best_effort_timestamp.reference create mode 100644 tests/queries/0_stateless/01432_parse_date_time_best_effort_timestamp.sql diff --git a/src/Common/BitHelpers.h b/src/Common/BitHelpers.h index 639a0c026ff..699e379b8d3 100644 --- a/src/Common/BitHelpers.h +++ b/src/Common/BitHelpers.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -11,6 +12,7 @@ */ inline unsigned int bitScanReverse(unsigned int x) { + assert(x != 0); return sizeof(unsigned int) * 8 - 1 - __builtin_clz(x); } diff --git a/src/IO/parseDateTimeBestEffort.cpp b/src/IO/parseDateTimeBestEffort.cpp index c350bd1ddc1..d84ac76b164 100644 --- a/src/IO/parseDateTimeBestEffort.cpp +++ b/src/IO/parseDateTimeBestEffort.cpp @@ -104,6 +104,7 @@ ReturnType parseDateTimeBestEffortImpl( return false; }; + res = 0; UInt16 year = 0; UInt8 month = 0; UInt8 day_of_month = 0; diff --git a/tests/queries/0_stateless/01432_parse_date_time_best_effort_timestamp.reference b/tests/queries/0_stateless/01432_parse_date_time_best_effort_timestamp.reference new file mode 100644 index 00000000000..36cee487d54 --- /dev/null +++ b/tests/queries/0_stateless/01432_parse_date_time_best_effort_timestamp.reference @@ -0,0 +1,3 @@ +2020-08-07 01:29:00 +1973-03-03 12:46:40 +2020-08-07 00:00:00 diff --git a/tests/queries/0_stateless/01432_parse_date_time_best_effort_timestamp.sql b/tests/queries/0_stateless/01432_parse_date_time_best_effort_timestamp.sql new file mode 100644 index 00000000000..071fefe0403 --- /dev/null +++ b/tests/queries/0_stateless/01432_parse_date_time_best_effort_timestamp.sql @@ -0,0 +1,3 @@ +SELECT parseDateTimeBestEffort('1596752940', 'Europe/Moscow'); +SELECT parseDateTimeBestEffort('100000000', 'Europe/Moscow'); +SELECT parseDateTimeBestEffort('20200807', 'Europe/Moscow'); From 1452e1ab3085c437f7834c531f8a536ccdc9b783 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 7 Aug 2020 03:46:49 +0300 Subject: [PATCH 30/43] Added a test for hex floats in INSERT VALUES --- tests/queries/0_stateless/01433_hex_float.reference | 2 ++ tests/queries/0_stateless/01433_hex_float.sql | 4 ++++ 2 files changed, 6 insertions(+) create mode 100644 tests/queries/0_stateless/01433_hex_float.reference create mode 100644 tests/queries/0_stateless/01433_hex_float.sql diff --git a/tests/queries/0_stateless/01433_hex_float.reference b/tests/queries/0_stateless/01433_hex_float.reference new file mode 100644 index 00000000000..a49ea94b283 --- /dev/null +++ b/tests/queries/0_stateless/01433_hex_float.reference @@ -0,0 +1,2 @@ +0.123 +1 diff --git a/tests/queries/0_stateless/01433_hex_float.sql b/tests/queries/0_stateless/01433_hex_float.sql new file mode 100644 index 00000000000..124a1ab0736 --- /dev/null +++ b/tests/queries/0_stateless/01433_hex_float.sql @@ -0,0 +1,4 @@ +CREATE TEMPORARY TABLE t (x Float64); +INSERT INTO t VALUES (0x1.f7ced916872b0p-4); +SELECT * FROM t; +SELECT x = 0x1.f7ced916872b0p-4 FROM t; From cb153d260585d8cfc552e11845d7c122bef30854 Mon Sep 17 00:00:00 2001 From: vladimir golovchenko Date: Thu, 6 Aug 2020 17:53:27 -0700 Subject: [PATCH 31/43] Updated gitignore-files. --- .gitignore | 1 + programs/server/.gitignore | 11 +++++++++++ programs/server/data/.gitignore | 2 -- programs/server/data/default/.gitignore | 2 -- programs/server/metadata/.gitignore | 1 + 5 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 programs/server/.gitignore create mode 100644 programs/server/metadata/.gitignore diff --git a/.gitignore b/.gitignore index afb4e67a1b8..0716fb1ac09 100644 --- a/.gitignore +++ b/.gitignore @@ -79,6 +79,7 @@ configure-stamp *.bin *.mrk *.mrk2 +*.mrk3 .dupload.conf diff --git a/programs/server/.gitignore b/programs/server/.gitignore new file mode 100644 index 00000000000..ba8a531c6d7 --- /dev/null +++ b/programs/server/.gitignore @@ -0,0 +1,11 @@ +/access +/dictionaries_lib +/flags +/format_schemas +/metadata_dropped +/preprocessed_configs +/tmp +/user_files + +status + diff --git a/programs/server/data/.gitignore b/programs/server/data/.gitignore index 3e223bc0920..b9719d9d1d1 100644 --- a/programs/server/data/.gitignore +++ b/programs/server/data/.gitignore @@ -1,5 +1,3 @@ -*.bin -*.mrk *.txt *.dat *.idx diff --git a/programs/server/data/default/.gitignore b/programs/server/data/default/.gitignore index 5d26e622e3e..e69de29bb2d 100644 --- a/programs/server/data/default/.gitignore +++ b/programs/server/data/default/.gitignore @@ -1,2 +0,0 @@ -*.bin -*.mrk diff --git a/programs/server/metadata/.gitignore b/programs/server/metadata/.gitignore new file mode 100644 index 00000000000..d1b811b7de5 --- /dev/null +++ b/programs/server/metadata/.gitignore @@ -0,0 +1 @@ +*.sql From 11727864dd5d8b17f4347d60bd4b111854d43609 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 7 Aug 2020 04:25:45 +0300 Subject: [PATCH 32/43] performance comparison --- docker/test/performance-comparison/report.py | 46 +++++++++++--------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/docker/test/performance-comparison/report.py b/docker/test/performance-comparison/report.py index 8b35498d81b..31817e0ea10 100755 --- a/docker/test/performance-comparison/report.py +++ b/docker/test/performance-comparison/report.py @@ -179,10 +179,10 @@ def advanceRowAnchor(): return currentRowAnchor() -def tr(x): - a = advanceRowAnchor() +def tr(x, anchor=None): #return '{x}'.format(a=a, x=str(x)) - return '{x}'.format(a=a, x=str(x)) + anchor = anchor if anchor else advanceRowAnchor() + return f'{x}' def td(value, cell_attributes = ''): return '{value}'.format( @@ -192,12 +192,14 @@ def td(value, cell_attributes = ''): def th(x): return '' + str(x) + '' -def tableRow(cell_values, cell_attributes = []): - return tr(''.join([td(v, a) - for v, a in itertools.zip_longest( - cell_values, cell_attributes, - fillvalue = '') - if a is not None and v is not None])) +def tableRow(cell_values, cell_attributes = [], anchor=None): + return tr( + ''.join([td(v, a) + for v, a in itertools.zip_longest( + cell_values, cell_attributes, + fillvalue = '') + if a is not None and v is not None]), + anchor) def tableHeader(r): return tr(''.join([th(f) for f in r])) @@ -291,8 +293,8 @@ def add_errors_explained(): if not errors_explained: return - text = tableStart('Error summary') - text += '' + text = '' + text += tableStart('Error summary') text += tableHeader(['Description']) for row in errors_explained: text += tableRow(row) @@ -342,19 +344,20 @@ if args.report == 'main': text += tableHeader(columns) attrs = ['' for c in columns] for row in rows: + anchor = f'{currentTableAnchor()}.{row[2]}.{row[3]}' if float(row[1]) > 0.10: attrs[1] = f'style="background: {color_bad}"' unstable_partial_queries += 1 - errors_explained.append([f'The query no. {row[3]} of test \'{row[2]}\' has excessive variance of run time. Keep it below 10%']) + errors_explained.append([f'The query no. {row[3]} of test \'{row[2]}\' has excessive variance of run time. Keep it below 10%']) else: attrs[1] = '' if float(row[0]) > allowed_single_run_time: attrs[0] = f'style="background: {color_bad}"' - errors_explained.append([f'The query no. {row[3]} of test \'{row[2]}\' is taking too long to run. Keep the run time below {allowed_single_run} seconds"']) + errors_explained.append([f'The query no. {row[3]} of test \'{row[2]}\' is taking too long to run. Keep the run time below {allowed_single_run} seconds"']) slow_average_tests += 1 else: attrs[0] = '' - text += tableRow(row, attrs) + text += tableRow(row, attrs, anchor) text += tableEnd() tables.append(text) @@ -385,6 +388,7 @@ if args.report == 'main': attrs = ['' for c in columns] attrs[5] = None for row in rows: + anchor = f'{currentTableAnchor()}.{row[6]}.{row[7]}' if int(row[5]): if float(row[3]) < 0.: faster_queries += 1 @@ -392,11 +396,11 @@ if args.report == 'main': else: slower_queries += 1 attrs[2] = attrs[3] = f'style="background: {color_bad}"' - errors_explained.append([f'The query no. {row[7]} of test \'{row[6]}\' has slowed down']) + errors_explained.append([f'The query no. {row[7]} of test \'{row[6]}\' has slowed down']) else: attrs[2] = attrs[3] = '' - text += tableRow(row, attrs) + text += tableRow(row, attrs, anchor) text += tableEnd() tables.append(text) @@ -429,13 +433,14 @@ if args.report == 'main': attrs = ['' for c in columns] attrs[4] = None for r in unstable_rows: + anchor = f'{currentTableAnchor()}.{r[5]}.{r[6]}' if int(r[4]): very_unstable_queries += 1 attrs[3] = f'style="background: {color_bad}"' else: attrs[3] = '' - text += tableRow(r, attrs) + text += tableRow(r, attrs, anchor) text += tableEnd() tables.append(text) @@ -477,14 +482,14 @@ if args.report == 'main': # FIXME should be 15s max -- investigate parallel_insert slow_average_tests += 1 attrs[6] = f'style="background: {color_bad}"' - errors_explained.append([f'The test \'{r[0]}\' is too slow to run as a whole. Investigate whether the create and fill queries can be sped up']) + errors_explained.append([f'The test \'{r[0]}\' is too slow to run as a whole. Investigate whether the create and fill queries can be sped up']) else: attrs[6] = '' if float(r[5]) > allowed_single_run_time * total_runs: slow_average_tests += 1 attrs[5] = f'style="background: {color_bad}"' - errors_explained.append([f'Some query of the test \'{r[0]}\' is too slow to run. See the all queries report']) + errors_explained.append([f'Some query of the test \'{r[0]}\' is too slow to run. See the all queries report']) else: attrs[5] = '' @@ -659,6 +664,7 @@ elif args.report == 'all-queries': attrs[0] = None attrs[1] = None for r in rows: + anchor = f'{currentTableAnchor()}.{r[7]}.{r[8]}' if int(r[1]): attrs[6] = f'style="background: {color_bad}"' else: @@ -679,7 +685,7 @@ elif args.report == 'all-queries': attrs[2] = '' attrs[3] = '' - text += tableRow(r, attrs) + text += tableRow(r, attrs, anchor) text += tableEnd() tables.append(text) From 0ac3f0481fb8a8295ddb38e7eea76b34afd6eacd Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 7 Aug 2020 04:27:29 +0300 Subject: [PATCH 33/43] Probably fix error --- src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 7ac8ca6143b..b2f123190e6 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -34,6 +34,7 @@ namespace std static constexpr bool is_integer = true; static constexpr int radix = 2; static constexpr int digits = 128; + static constexpr int digits10 = 38; static constexpr __uint128_t min () { return 0; } // used in boost 1.65.1+ static constexpr __uint128_t max () { return __uint128_t(0) - 1; } // used in boost 1.68.0+ }; From 6ea0943a0f6a0a8bc82896c1b4f4a03d79758c40 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Fri, 7 Aug 2020 04:55:00 +0300 Subject: [PATCH 34/43] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index b84ac2e1d03..e895e10c458 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ ClickHouse is an open-source column-oriented database management system that all * [YouTube channel](https://www.youtube.com/c/ClickHouseDB) has a lot of content about ClickHouse in video format. * [Slack](https://join.slack.com/t/clickhousedb/shared_invite/zt-d2zxkf9e-XyxDa_ucfPxzuH4SJIm~Ng) and [Telegram](https://telegram.me/clickhouse_en) allow to chat with ClickHouse users in real-time. * [Blog](https://clickhouse.yandex/blog/en/) contains various ClickHouse-related articles, as well as announcements and reports about events. +* [Code Browser](https://clickhouse.tech/codebrowser/html_report/ClickHouse/index.html) with syntax highlight and navigation. * [Yandex.Messenger channel](https://yandex.ru/chat/#/join/20e380d9-c7be-4123-ab06-e95fb946975e) shares announcements and useful links in Russian. * [Contacts](https://clickhouse.tech/#contacts) can help to get your questions answered if there are any. * You can also [fill this form](https://clickhouse.tech/#meet) to meet Yandex ClickHouse team in person. From 898e19e18dba3cc5e1fd9ba6004bb3c9b90c5e41 Mon Sep 17 00:00:00 2001 From: hcz Date: Fri, 7 Aug 2020 15:40:05 +0800 Subject: [PATCH 35/43] Fix function names --- src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp | 2 +- src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp index 69e42c0ad37..81f3d92b888 100644 --- a/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp @@ -47,7 +47,7 @@ void ArrowBlockOutputFormat::finalize() { const Block & header = getPort(PortKind::Main).getHeader(); - consume(Chunk(header.columns(), 0)); + consume(Chunk(header.getColumns(), 0)); } auto status = writer->Close(); diff --git a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp index 436f97ccc3c..695b8efa3fb 100644 --- a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp @@ -70,7 +70,7 @@ void ParquetBlockOutputFormat::finalize() { const Block & header = getPort(PortKind::Main).getHeader(); - consume(Chunk(header.columns(), 0)); + consume(Chunk(header.getColumns(), 0)); } auto status = file_writer->Close(); From 7a9b244f3791644c4f9303bbd3d4d115d2d1e684 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 7 Aug 2020 11:29:41 +0300 Subject: [PATCH 36/43] Push ci --- src/Functions/FunctionsComparison.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Functions/FunctionsComparison.h b/src/Functions/FunctionsComparison.h index 17893416c58..44c8ffa23a6 100644 --- a/src/Functions/FunctionsComparison.h +++ b/src/Functions/FunctionsComparison.h @@ -895,8 +895,8 @@ private: ColumnsWithTypeAndName x(tuple_size); ColumnsWithTypeAndName y(tuple_size); - auto x_const = checkAndGetColumnConst(c0.column.get()); - auto y_const = checkAndGetColumnConst(c1.column.get()); + const auto * x_const = checkAndGetColumnConst(c0.column.get()); + const auto * y_const = checkAndGetColumnConst(c1.column.get()); Columns x_columns; Columns y_columns; From b7f08f447a9643cb5bd8eda78f89198ac3f9f27d Mon Sep 17 00:00:00 2001 From: Alexander Millin Date: Fri, 7 Aug 2020 11:48:38 +0300 Subject: [PATCH 37/43] [docs] Fix system.md (#13332) Co-authored-by: Alexander Millin --- docs/en/sql-reference/statements/system.md | 4 ---- docs/ru/sql-reference/statements/system.md | 6 +----- docs/zh/sql-reference/statements/system.md | 4 ---- 3 files changed, 1 insertion(+), 13 deletions(-) diff --git a/docs/en/sql-reference/statements/system.md b/docs/en/sql-reference/statements/system.md index 7a7f6ba44bb..f6ff264e827 100644 --- a/docs/en/sql-reference/statements/system.md +++ b/docs/en/sql-reference/statements/system.md @@ -277,8 +277,4 @@ SYSTEM RESTART REPLICA [db.]replicated_merge_tree_family_table_name Provides possibility to reinitialize Zookeeper sessions state for all `ReplicatedMergeTree` tables, will compare current state with Zookeeper as source of true and add tasks to Zookeeper queue if needed -``` sql -SYSTEM RESTART QUEUES [db.]replicated_merge_tree_family_table_name -``` - [Original article](https://clickhouse.tech/docs/en/query_language/system/) diff --git a/docs/ru/sql-reference/statements/system.md b/docs/ru/sql-reference/statements/system.md index 6c5f550879d..89536d0e2f6 100644 --- a/docs/ru/sql-reference/statements/system.md +++ b/docs/ru/sql-reference/statements/system.md @@ -235,14 +235,10 @@ SYSTEM SYNC REPLICA [db.]replicated_merge_tree_family_table_name Инициализация очереди репликации на основе данных ZooKeeper, происходит так же как при attach table. На короткое время таблица станет недоступной для любых операций. ``` sql -SYSTEM RESTART QUEUES [db.]replicated_merge_tree_family_table_name +SYSTEM RESTART REPLICA [db.]replicated_merge_tree_family_table_name ``` ### RESTART REPLICAS {#query_language-system-restart-replicas} Реинициализация состояния Zookeeper сессий для всех `ReplicatedMergeTree` таблиц, сравнивает текущее состояние с тем что хранится в Zookeeper как источник правды и добавляет задачи Zookeeper очередь если необходимо -``` sql -SYSTEM RESTART QUEUES [db.]replicated_merge_tree_family_table_name -``` - [Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/system/) diff --git a/docs/zh/sql-reference/statements/system.md b/docs/zh/sql-reference/statements/system.md index c5ca7c6c388..c46d1745125 100644 --- a/docs/zh/sql-reference/statements/system.md +++ b/docs/zh/sql-reference/statements/system.md @@ -280,8 +280,4 @@ SYSTEM RESTART REPLICA [db.]replicated_merge_tree_family_table_name 重置所有 `ReplicatedMergeTree`表的ZooKeeper会话状态。该操作会以Zookeeper为参照，对比当前状态，有需要的情况下将任务添加到ZooKeeper队列。 -``` sql -SYSTEM RESTART QUEUES [db.]replicated_merge_tree_family_table_name -``` - [原始文档](https://clickhouse.tech/docs/en/query_language/system/) From 55088d53cdfc90f0dcf3521ceec29394224053d3 Mon Sep 17 00:00:00 2001 From: Roman Bug Date: Fri, 7 Aug 2020 12:02:45 +0300 Subject: [PATCH 38/43] DOCSUP-2035: Edit and translate to Russian some PR. (#13395) * DOCSUP-2035: (ru) Added updates in replacingmergetree.md by PR#11793 * DOCSUP-2035: Added Third-party description by PR#12175. * DOCSUP-2035: Updated translation in third-party/index.md * DOCSUP-2035: (ru) Added translation to table-engines-for-integrations. * DOCSUP-2035: (ru) Added translation to table-engines-for-integrations by PR#12170. * DOCSUP-2035: Added Special table engines and engines for integration sections by PR#1270 * DOCSUP-2035: (ru) Added allow TabSeparatedRaw description in interfaces/formats.md by PR#12009. * DOCSUP-2035: Added footer with link to original pages. * DOCSUP-2035: Update note block. * DOCSUP-2035: Update attention block. * Update docs/ru/interfaces/third-party/index.md Co-authored-by: BayoNet * Update docs/ru/interfaces/third-party/index.md Co-authored-by: BayoNet * Update docs/ru/interfaces/formats.md Co-authored-by: BayoNet * Update docs/ru/engines/table-engines/integrations/index.md Co-authored-by: BayoNet Co-authored-by: romanzhukov Co-authored-by: BayoNet --- docs/ru/engines/table-engines/integrations/index.md | 12 ++++++++++++ .../mergetree-family/replacingmergetree.md | 6 +++--- docs/ru/engines/table-engines/special/index.md | 10 ++++++++++ docs/ru/interfaces/formats.md | 4 ++-- docs/ru/interfaces/third-party/index.md | 12 ++++++++++++ 5 files changed, 39 insertions(+), 5 deletions(-) diff --git a/docs/ru/engines/table-engines/integrations/index.md b/docs/ru/engines/table-engines/integrations/index.md index dd005919ad1..0186f6cf19e 100644 --- a/docs/ru/engines/table-engines/integrations/index.md +++ b/docs/ru/engines/table-engines/integrations/index.md @@ -3,4 +3,16 @@ toc_folder_title: Integrations toc_priority: 30 --- +# Движки таблиц для интеграции {#table-engines-for-integrations} +Для интеграции с внешними системами ClickHouse предоставляет различные средства, включая движки таблиц. Конфигурирование интеграционных движков осуществляется с помощью запросов `CREATE TABLE` или `ALTER TABLE`, как и для других табличных движков. С точки зрения пользователя, настроенная интеграция выглядит как обычная таблица, но запросы к ней передаются через прокси во внешнюю систему. Этот прозрачный запрос является одним из ключевых преимуществ этого подхода по сравнению с альтернативными методами интеграции, такими как внешние словари или табличные функции, которые требуют использования пользовательских методов запроса при каждом использовании. + +Список поддерживаемых интеграций: + +- [ODBC](../../../engines/table-engines/integrations/odbc.md) +- [JDBC](../../../engines/table-engines/integrations/jdbc.md) +- [MySQL](../../../engines/table-engines/integrations/mysql.md) +- [HDFS](../../../engines/table-engines/integrations/hdfs.md) +- [Kafka](../../../engines/table-engines/integrations/kafka.md) + +[Оригинальная статья](https://clickhouse.tech/docs/ru/engines/table-engines/integrations/) diff --git a/docs/ru/engines/table-engines/mergetree-family/replacingmergetree.md b/docs/ru/engines/table-engines/mergetree-family/replacingmergetree.md index 6e3e1224997..4aa1eb556f3 100644 --- a/docs/ru/engines/table-engines/mergetree-family/replacingmergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/replacingmergetree.md @@ -1,6 +1,6 @@ # ReplacingMergeTree {#replacingmergetree} -Движок отличается от [MergeTree](mergetree.md#table_engines-mergetree) тем, что выполняет удаление дублирующихся записей с одинаковым значением первичного ключа (точнее, с одинаковым значением [ключа сортировки](mergetree.md)). +Движок отличается от [MergeTree](mergetree.md#table_engines-mergetree) тем, что выполняет удаление дублирующихся записей с одинаковым значением [ключа сортировки](mergetree.md)). Дедупликация данных производится лишь во время слияний. Слияние происходят в фоне в неизвестный момент времени, на который вы не можете ориентироваться. Некоторая часть данных может остаться необработанной. Хотя вы можете вызвать внеочередное слияние с помощью запроса `OPTIMIZE`, на это не стоит рассчитывать, так как запрос `OPTIMIZE` приводит к чтению и записи большого объёма данных. @@ -27,7 +27,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] - `ver` — столбец с версией, тип `UInt*`, `Date` или `DateTime`. Необязательный параметр. - При слиянии, из всех строк с одинаковым значением первичного ключа `ReplacingMergeTree` оставляет только одну: + При слиянии, из всех строк с одинаковым значением ключа сортировки `ReplacingMergeTree` оставляет только одну: - Последнюю в выборке, если `ver` не задан. - С максимальной версией, если `ver` задан. @@ -40,7 +40,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] Устаревший способ создания таблицы -!!! attention "Attention" +!!! attention "Внимание" Не используйте этот способ в новых проектах и по возможности переведите старые проекты на способ описанный выше. ``` sql diff --git a/docs/ru/engines/table-engines/special/index.md b/docs/ru/engines/table-engines/special/index.md index b1789f34347..dcf46c503b2 100644 --- a/docs/ru/engines/table-engines/special/index.md +++ b/docs/ru/engines/table-engines/special/index.md @@ -3,4 +3,14 @@ toc_folder_title: Special toc_priority: 31 --- +# Специальные движки таблиц {#special-table-engines} +Существует три основные категории движков таблиц: + +- [Семейство MergeTree](../../../engines/table-engines/mergetree-family/index.md) для основного использования. +- [Семейство Log](../../../engines/table-engines/log-family/index.md) для небольших временных данных. +- [Движки таблиц для интеграции](../../../engines/table-engines/integrations/index.md). + +Остальные движки таблиц уникальны по своему назначению и еще не сгруппированы в семейства, поэтому они помещены в эту специальную категорию. + +[Оригинальная статья](https://clickhouse.tech/docs/ru/engines/table-engines/special/) diff --git a/docs/ru/interfaces/formats.md b/docs/ru/interfaces/formats.md index c5cad5e2ac8..6e7bb796bb8 100644 --- a/docs/ru/interfaces/formats.md +++ b/docs/ru/interfaces/formats.md @@ -7,7 +7,7 @@ ClickHouse может принимать (`INSERT`) и отдавать (`SELECT | Формат | INSERT | SELECT | |-----------------------------------------------------------------|--------|--------| | [TabSeparated](#tabseparated) | ✔ | ✔ | -| [TabSeparatedRaw](#tabseparatedraw) | ✗ | ✔ | +| [TabSeparatedRaw](#tabseparatedraw) | ✔ | ✔ | | [TabSeparatedWithNames](#tabseparatedwithnames) | ✔ | ✔ | | [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes) | ✔ | ✔ | | [Template](#format-template) | ✔ | ✔ | @@ -132,7 +132,7 @@ SELECT * FROM nestedt FORMAT TSV ## TabSeparatedRaw {#tabseparatedraw} Отличается от формата `TabSeparated` тем, что строки выводятся без экранирования. -Этот формат подходит только для вывода результата выполнения запроса, но не для парсинга (приёма данных для вставки в таблицу). +Используя этот формат, следите, чтобы в полях не было символов табуляции или разрыва строки. Этот формат также доступен под именем `TSVRaw`. diff --git a/docs/ru/interfaces/third-party/index.md b/docs/ru/interfaces/third-party/index.md index db6e4ad0038..bfdd554aebd 100644 --- a/docs/ru/interfaces/third-party/index.md +++ b/docs/ru/interfaces/third-party/index.md @@ -4,4 +4,16 @@ toc_folder_title: "\u041E\u0442 \u0441\u0442\u043E\u0440\u043E\u043D\u043D\u0438 toc_priority: 24 --- +# Сторонние интерфейсы {#third-party-interfaces} +Раздел содержит список сторонних интерфейсов для ClickHouse. Это может быть визуальный интерфейс, интерфейс командной строки, либо API: + +- [Client libraries](../../interfaces/third-party/client-libraries.md) +- [Integrations](../../interfaces/third-party/integrations.md) +- [GUI](../../interfaces/third-party/gui.md) +- [Proxies](../../interfaces/third-party/proxy.md) + +!!! note "Примечание" + С ClickHouse работают также универсальные инструменты, поддерживающие общий API, такие как [ODBC](../../interfaces/odbc.md) или [JDBC](../../interfaces/jdbc.md). + +[Оригинальная статья](https://clickhouse.tech/docs/ru/interfaces/third-party/) From a00e6ce181103e738181e07a554c9b44500a0627 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 7 Aug 2020 14:10:04 +0300 Subject: [PATCH 39/43] Added missing modification --- website/js/docs.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/website/js/docs.js b/website/js/docs.js index 71708e38afd..2af01bc1929 100644 --- a/website/js/docs.js +++ b/website/js/docs.js @@ -49,9 +49,7 @@ $(document).ready(function () { $('#sidebar .nav-link.active').parents('.collapse').each(function() { var current = $(this); if (current.attr('id') !== 'sidebar') { - current.css('transition-duration', '0s'); current.collapse('show'); - current.css('transition-duration', '0.4s'); } }); $(window).resize(onResize); From f74f5cfa061c8eee90e53bef213102052ffa8d87 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 7 Aug 2020 14:18:41 +0300 Subject: [PATCH 40/43] Attempt to fix "Unbundled" build --- src/IO/readDecimalText.h | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/IO/readDecimalText.h b/src/IO/readDecimalText.h index e3569cb471c..a34b08df1e5 100644 --- a/src/IO/readDecimalText.h +++ b/src/IO/readDecimalText.h @@ -1,9 +1,29 @@ #pragma once +#include #include #include +/// This is only needed for non-official, "unbundled" build. +/// https://stackoverflow.com/questions/41198673/uint128-t-not-working-with-clang-and-libstdc +#if !defined(_LIBCPP_LIMITS) && !defined(__GLIBCXX_BITSIZE_INT_N_0) && defined(__SIZEOF_INT128__) +namespace std +{ + template <> + struct numeric_limits<__int128_t> + { + static constexpr bool is_specialized = true; + static constexpr bool is_signed = true; + static constexpr bool is_integer = true; + static constexpr int radix = 2; + static constexpr int digits = 127; + static constexpr int digits10 = 38; + }; +} +#endif + + namespace DB { From 48466c0960b3abe590552e30bc79ec7813a957a7 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 7 Aug 2020 19:07:52 +0300 Subject: [PATCH 41/43] New version of dpkg-deb --- docker/packager/deb/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/packager/deb/Dockerfile b/docker/packager/deb/Dockerfile index 48e90d16f5d..c1260b5c7ff 100644 --- a/docker/packager/deb/Dockerfile +++ b/docker/packager/deb/Dockerfile @@ -21,7 +21,7 @@ RUN apt-get --allow-unauthenticated update -y \ # Special dpkg-deb (https://github.com/ClickHouse-Extras/dpkg) version which is able # to compress files using pigz (https://zlib.net/pigz/) instead of gzip. # Significantly increase deb packaging speed and compatible with old systems -RUN curl -O https://clickhouse-builds.s3.yandex.net/utils/dpkg-deb +RUN curl -O https://clickhouse-builds.s3.yandex.net/utils/1/dpkg-deb RUN chmod +x dpkg-deb RUN cp dpkg-deb /usr/bin From 19f3800a55a08f19c74000510e65f0550558f37f Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 7 Aug 2020 21:45:15 +0300 Subject: [PATCH 42/43] Auto version update to [20.7.1.4310] [54437] --- cmake/autogenerated_versions.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 40daf2637c5..d7173f99207 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -3,7 +3,7 @@ SET(VERSION_REVISION 54437) SET(VERSION_MAJOR 20) SET(VERSION_MINOR 7) SET(VERSION_PATCH 1) -SET(VERSION_GITHASH d64e51d1a78c1b53c33915ca0f75c97b2333844f) -SET(VERSION_DESCRIBE v20.7.1.1-prestable) -SET(VERSION_STRING 20.7.1.1) +SET(VERSION_GITHASH 5d60ab33a511efd149c7c3de77c0dd4b81e65b13) +SET(VERSION_DESCRIBE v20.7.1.4310-prestable) +SET(VERSION_STRING 20.7.1.4310) # end of autochange From a82c0f53710e29c75b0680994173d30c06c25891 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 7 Aug 2020 21:45:49 +0300 Subject: [PATCH 43/43] Auto version update to [20.8.1.1] [54438] --- cmake/autogenerated_versions.txt | 8 ++++---- debian/changelog | 4 ++-- docker/client/Dockerfile | 2 +- docker/server/Dockerfile | 2 +- docker/test/Dockerfile | 2 +- .../StorageSystemContributors.generated.cpp | 18 ++++++++++++++++++ 6 files changed, 27 insertions(+), 9 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index d7173f99207..ebb9bdcf568 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -1,9 +1,9 @@ # This strings autochanged from release_lib.sh: -SET(VERSION_REVISION 54437) +SET(VERSION_REVISION 54438) SET(VERSION_MAJOR 20) -SET(VERSION_MINOR 7) +SET(VERSION_MINOR 8) SET(VERSION_PATCH 1) SET(VERSION_GITHASH 5d60ab33a511efd149c7c3de77c0dd4b81e65b13) -SET(VERSION_DESCRIBE v20.7.1.4310-prestable) -SET(VERSION_STRING 20.7.1.4310) +SET(VERSION_DESCRIBE v20.8.1.1-prestable) +SET(VERSION_STRING 20.8.1.1) # end of autochange diff --git a/debian/changelog b/debian/changelog index 6b724f13a4e..c82a3c6657b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,5 @@ -clickhouse (20.7.1.1) unstable; urgency=low +clickhouse (20.8.1.1) unstable; urgency=low * Modified source code - -- clickhouse-release Mon, 13 Jul 2020 18:25:58 +0300 + -- clickhouse-release Fri, 07 Aug 2020 21:45:46 +0300 diff --git a/docker/client/Dockerfile b/docker/client/Dockerfile index 1c83c4354eb..fa7e3816959 100644 --- a/docker/client/Dockerfile +++ b/docker/client/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/" -ARG version=20.7.1.* +ARG version=20.8.1.* RUN apt-get update \ && apt-get install --yes --no-install-recommends \ diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile index 3fc9c167ab4..1ba00bf299d 100644 --- a/docker/server/Dockerfile +++ b/docker/server/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:20.04 ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/" -ARG version=20.7.1.* +ARG version=20.8.1.* ARG gosu_ver=1.10 RUN apt-get update \ diff --git a/docker/test/Dockerfile b/docker/test/Dockerfile index 0f48a06d9d3..c9144230da9 100644 --- a/docker/test/Dockerfile +++ b/docker/test/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/" -ARG version=20.7.1.* +ARG version=20.8.1.* RUN apt-get update && \ apt-get install -y apt-transport-https dirmngr && \ diff --git a/src/Storages/System/StorageSystemContributors.generated.cpp b/src/Storages/System/StorageSystemContributors.generated.cpp index c4c731ca166..1aa984e5519 100644 --- a/src/Storages/System/StorageSystemContributors.generated.cpp +++ b/src/Storages/System/StorageSystemContributors.generated.cpp @@ -52,6 +52,7 @@ const char * auto_contributors[] { "Aliaksandr Shylau", "Amos Bird", "Amy Krishnevsky", + "AnaUvarova", "Anastasiya Rodigina", "Anastasiya Tsarkova", "Anatoly Pugachev", @@ -72,8 +73,10 @@ const char * auto_contributors[] { "Andrey Skobtsov", "Andrey Urusov", "Andy Yang", + "Anna", "Anton Ivashkin", "Anton Kobzev", + "Anton Kvasha", "Anton Okhitin", "Anton Okulov", "Anton Patsev", @@ -90,6 +93,7 @@ const char * auto_contributors[] { "Artem Zuikov", "Artemeey", "Artemkin Pavel", + "Arthur Petukhovsky", "Arthur Tokarchuk", "Artur Beglaryan", "AsiaKorushkina", @@ -104,6 +108,7 @@ const char * auto_contributors[] { "BayoNet", "Bharat Nallan", "Big Elephant", + "Bill", "BlahGeek", "Bogdan", "Bogdan Voronin", @@ -130,6 +135,7 @@ const char * auto_contributors[] { "Darío", "Denis Burlaka", "Denis Glazachev", + "Denis Krivak", "Denis Zhuravlev", "Derek Perkins", "Ding Xiang Fei", @@ -152,6 +158,7 @@ const char * auto_contributors[] { "Eldar Zaitov", "Elena Baskakova", "Elghazal Ahmed", + "Elizaveta Mironyuk", "Emmanuel Donin de Rosière", "Eric", "Ernest Poletaev", @@ -259,6 +266,7 @@ const char * auto_contributors[] { "Marek Vavrusa", "Marek Vavruša", "Marek Vavruša", + "Mark Papadakis", "Maroun Maroun", "Marsel Arduanov", "Marti Raudsepp", @@ -307,6 +315,7 @@ const char * auto_contributors[] { "Mohammad Hossein Sekhavat", "MovElb", "Murat Kabilov", + "MyroTk", "NIKITA MIKHAILOV", "Narek Galstyan", "NeZeD [Mac Pro]", @@ -346,6 +355,7 @@ const char * auto_contributors[] { "Pavel Yakunin", "Pavlo Bashynskiy", "Pawel Rog", + "Peng Jian", "Persiyanov Dmitriy Andreevich", "Pervakov Grigorii", "Pervakov Grigory", @@ -359,6 +369,7 @@ const char * auto_contributors[] { "Reilee", "Reto Kromer", "Ri", + "Roman Bug", "Roman Lipovsky", "Roman Nikolaev", "Roman Nozdrin", @@ -466,6 +477,7 @@ const char * auto_contributors[] { "Yurii Vlasenko", "Yuriy", "Yuriy Baranov", + "Yuriy Chernyshov", "Yury Karpovich", "Yury Stankevich", "Zhichang Yu", @@ -483,6 +495,7 @@ const char * auto_contributors[] { "alex.lvxin", "alexander kozhikhov", "alexey-milovidov", + "amoschen", "amudong", "andrei-karpliuk", "andrewsg", @@ -556,6 +569,7 @@ const char * auto_contributors[] { "imgbot[bot]", "ivan-kush", "ivanzhukov", + "jakalletti", "javartisan", "javi", "javi santana", @@ -610,6 +624,7 @@ const char * auto_contributors[] { "objatie_groba", "ogorbacheva", "olegkv", + "olgarev", "orantius", "palasonicq", "peshkurov", @@ -621,6 +636,7 @@ const char * auto_contributors[] { "qianlixiang", "quid", "rainbowsysu", + "ritaank", "robot-clickhouse", "robot-metrika-test", "root", @@ -652,6 +668,7 @@ const char * auto_contributors[] { "vinity", "vitstn", "vivarum", + "vladimir golovchenko", "vxider", "vzakaznikov", "wangchao", @@ -679,6 +696,7 @@ const char * auto_contributors[] { "张风啸", "极客青年", "谢磊", + "贾顺名(Jarvis)", "黄朝晖", "黄璞", "박현우",