From 3929baf9ff2109d366be42589b5665b0ac5090d7 Mon Sep 17 00:00:00 2001 From: yaqi-zhao Date: Thu, 2 Jun 2022 16:17:13 -0400 Subject: [PATCH 001/717] add avx512 support for Lower/Upper function --- src/Functions/LowerUpperImpl.h | 64 +++++++++++++++++++++++++--------- 1 file changed, 47 insertions(+), 17 deletions(-) diff --git a/src/Functions/LowerUpperImpl.h b/src/Functions/LowerUpperImpl.h index a7c38a7f904..ca3b7173032 100644 --- a/src/Functions/LowerUpperImpl.h +++ b/src/Functions/LowerUpperImpl.h @@ -29,31 +29,61 @@ private: { const auto flip_case_mask = 'A' ^ 'a'; +#if defined(__AVX512F__) && defined(__AVX512BW__) /// check if avx512 instructions are compiled + if (isArchSupported(TargetArch::AVX512BW)) + { + /// check if cpu support avx512 dynamically, haveAVX512BW contains check of haveAVX512F + const auto byte_avx512 = sizeof(__m512i); + const auto src_end_avx = src_end - (src_end - src) % byte_avx512; + if (src < src_end_avx) { + const auto v_not_case_lower_bound = _mm512_set1_epi8(not_case_lower_bound - 1); + const auto v_not_case_upper_bound = _mm512_set1_epi8(not_case_upper_bound + 1); + const auto v_flip_case_mask = _mm512_set1_epi8(flip_case_mask); + + for (; src < src_end_avx; src += byte_avx512, dst += byte_avx512) { + const auto chars = _mm512_loadu_si512(reinterpret_cast(src)); + + const auto is_not_case + = _mm512_and_si512(_mm512_movm_epi8(_mm512_cmpgt_epi8_mask(chars, v_not_case_lower_bound)), + _mm512_movm_epi8(_mm512_cmplt_epi8_mask(chars, v_not_case_upper_bound))); + + const auto xor_mask = _mm512_and_si512(v_flip_case_mask, is_not_case); + + const auto cased_chars = _mm512_xor_si512(chars, xor_mask); + + _mm512_storeu_si512(reinterpret_cast<__m512i *>(dst), cased_chars); + } + } + } +#endif + #ifdef __SSE2__ const auto bytes_sse = sizeof(__m128i); const auto * src_end_sse = src_end - (src_end - src) % bytes_sse; - - const auto v_not_case_lower_bound = _mm_set1_epi8(not_case_lower_bound - 1); - const auto v_not_case_upper_bound = _mm_set1_epi8(not_case_upper_bound + 1); - const auto v_flip_case_mask = _mm_set1_epi8(flip_case_mask); - - for (; src < src_end_sse; src += bytes_sse, dst += bytes_sse) + if (src < src_end_sse) { - /// load 16 sequential 8-bit characters - const auto chars = _mm_loadu_si128(reinterpret_cast(src)); + const auto v_not_case_lower_bound = _mm_set1_epi8(not_case_lower_bound - 1); + const auto v_not_case_upper_bound = _mm_set1_epi8(not_case_upper_bound + 1); + const auto v_flip_case_mask = _mm_set1_epi8(flip_case_mask); - /// find which 8-bit sequences belong to range [case_lower_bound, case_upper_bound] - const auto is_not_case - = _mm_and_si128(_mm_cmpgt_epi8(chars, v_not_case_lower_bound), _mm_cmplt_epi8(chars, v_not_case_upper_bound)); + for (; src < src_end_sse; src += bytes_sse, dst += bytes_sse) + { + /// load 16 sequential 8-bit characters + const auto chars = _mm_loadu_si128(reinterpret_cast(src)); - /// keep `flip_case_mask` only where necessary, zero out elsewhere - const auto xor_mask = _mm_and_si128(v_flip_case_mask, is_not_case); + /// find which 8-bit sequences belong to range [case_lower_bound, case_upper_bound] + const auto is_not_case + = _mm_and_si128(_mm_cmpgt_epi8(chars, v_not_case_lower_bound), _mm_cmplt_epi8(chars, v_not_case_upper_bound)); - /// flip case by applying calculated mask - const auto cased_chars = _mm_xor_si128(chars, xor_mask); + /// keep `flip_case_mask` only where necessary, zero out elsewhere + const auto xor_mask = _mm_and_si128(v_flip_case_mask, is_not_case); - /// store result back to destination - _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), cased_chars); + /// flip case by applying calculated mask + const auto cased_chars = _mm_xor_si128(chars, xor_mask); + + /// store result back to destination + _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), cased_chars); + } } #endif From 9f50d409158f94bb57af1fb8dcf99945a9d377d6 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Tue, 7 Jun 2022 10:34:47 +0200 Subject: [PATCH 002/717] style fixes --- src/Functions/LowerUpperImpl.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/Functions/LowerUpperImpl.h b/src/Functions/LowerUpperImpl.h index ca3b7173032..efa088f4bc3 100644 --- a/src/Functions/LowerUpperImpl.h +++ b/src/Functions/LowerUpperImpl.h @@ -30,21 +30,23 @@ private: const auto flip_case_mask = 'A' ^ 'a'; #if defined(__AVX512F__) && defined(__AVX512BW__) /// check if avx512 instructions are compiled - if (isArchSupported(TargetArch::AVX512BW)) + if (isArchSupported(TargetArch::AVX512BW)) { /// check if cpu support avx512 dynamically, haveAVX512BW contains check of haveAVX512F const auto byte_avx512 = sizeof(__m512i); const auto src_end_avx = src_end - (src_end - src) % byte_avx512; - if (src < src_end_avx) { + if (src < src_end_avx) + { const auto v_not_case_lower_bound = _mm512_set1_epi8(not_case_lower_bound - 1); const auto v_not_case_upper_bound = _mm512_set1_epi8(not_case_upper_bound + 1); const auto v_flip_case_mask = _mm512_set1_epi8(flip_case_mask); - for (; src < src_end_avx; src += byte_avx512, dst += byte_avx512) { + for (; src < src_end_avx; src += byte_avx512, dst += byte_avx512) + { const auto chars = _mm512_loadu_si512(reinterpret_cast(src)); const auto is_not_case - = _mm512_and_si512(_mm512_movm_epi8(_mm512_cmpgt_epi8_mask(chars, v_not_case_lower_bound)), + = _mm512_and_si512(_mm512_movm_epi8(_mm512_cmpgt_epi8_mask(chars, v_not_case_lower_bound)), _mm512_movm_epi8(_mm512_cmplt_epi8_mask(chars, v_not_case_upper_bound))); const auto xor_mask = _mm512_and_si512(v_flip_case_mask, is_not_case); From d1b4185392ea5622243726b0a22e04ea9b31044b Mon Sep 17 00:00:00 2001 From: yaqi-zhao <100556669+yaqi-zhao@users.noreply.github.com> Date: Tue, 7 Jun 2022 17:55:09 +0800 Subject: [PATCH 003/717] reduce instructions by _mm512_maskz_set1_epi8 add performance test for lower/upper function --- src/Functions/LowerUpperImpl.h | 7 +++---- tests/performance/lower_upper_function.xml | 11 +++++++++++ 2 files changed, 14 insertions(+), 4 deletions(-) create mode 100644 tests/performance/lower_upper_function.xml diff --git a/src/Functions/LowerUpperImpl.h b/src/Functions/LowerUpperImpl.h index efa088f4bc3..f093e00f7ab 100644 --- a/src/Functions/LowerUpperImpl.h +++ b/src/Functions/LowerUpperImpl.h @@ -39,17 +39,16 @@ private: { const auto v_not_case_lower_bound = _mm512_set1_epi8(not_case_lower_bound - 1); const auto v_not_case_upper_bound = _mm512_set1_epi8(not_case_upper_bound + 1); - const auto v_flip_case_mask = _mm512_set1_epi8(flip_case_mask); for (; src < src_end_avx; src += byte_avx512, dst += byte_avx512) { const auto chars = _mm512_loadu_si512(reinterpret_cast(src)); const auto is_not_case - = _mm512_and_si512(_mm512_movm_epi8(_mm512_cmpgt_epi8_mask(chars, v_not_case_lower_bound)), - _mm512_movm_epi8(_mm512_cmplt_epi8_mask(chars, v_not_case_upper_bound))); + = _mm512_mask_cmplt_epi8_mask(_mm512_cmpgt_epi8_mask(chars, v_not_case_lower_bound), + chars, v_not_case_upper_bound); - const auto xor_mask = _mm512_and_si512(v_flip_case_mask, is_not_case); + const auto xor_mask = _mm512_maskz_set1_epi8(is_not_case, flip_case_mask); const auto cased_chars = _mm512_xor_si512(chars, xor_mask); diff --git a/tests/performance/lower_upper_function.xml b/tests/performance/lower_upper_function.xml new file mode 100644 index 00000000000..1b84a334ace --- /dev/null +++ b/tests/performance/lower_upper_function.xml @@ -0,0 +1,11 @@ + + select lower(randomString(16)) + select lower(randomString(32)) + select lower(randomString(64)) + select lower(randomString(128)) + select lower(randomString(256)) + select lower(randomString(512)) + select lower(randomString(1024)) + select lower(randomString(832)) + select lower(randomString(416)) + From cd2911d635491d71de1c058eef20fdf09f581276 Mon Sep 17 00:00:00 2001 From: zhao zhou Date: Mon, 6 Jun 2022 09:29:47 +0800 Subject: [PATCH 004/717] Add avx512 support for Aggregate Sum, function unary arithmetic, function comparison --- src/AggregateFunctions/AggregateFunctionSum.h | 34 ++++++++++++++++--- src/Common/TargetSpecific.h | 28 ++++++++++++--- src/Functions/FunctionUnaryArithmetic.h | 34 ++++++++++++++++--- src/Functions/FunctionsComparison.h | 34 ++++++++++++++++--- 4 files changed, 114 insertions(+), 16 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionSum.h b/src/AggregateFunctions/AggregateFunctionSum.h index 672e9dad9e5..8c5fc0fd6b6 100644 --- a/src/AggregateFunctions/AggregateFunctionSum.h +++ b/src/AggregateFunctions/AggregateFunctionSum.h @@ -59,7 +59,7 @@ struct AggregateFunctionSumData } /// Vectorized version - MULTITARGET_FUNCTION_AVX2_SSE42( + MULTITARGET_FUNCTION_AVX512BW_AVX512F_AVX2_SSE42( MULTITARGET_FUNCTION_HEADER( template void NO_SANITIZE_UNDEFINED NO_INLINE @@ -107,12 +107,25 @@ struct AggregateFunctionSumData void NO_INLINE addMany(const Value * __restrict ptr, size_t start, size_t end) { #if USE_MULTITARGET_CODE + if (isArchSupported(TargetArch::AVX512BW)) + { + addManyImplAVX512BW(ptr, start, end); + return; + } + + if (isArchSupported(TargetArch::AVX512F)) + { + addManyImplAVX512F(ptr, start, end); + return; + } + if (isArchSupported(TargetArch::AVX2)) { addManyImplAVX2(ptr, start, end); return; } - else if (isArchSupported(TargetArch::SSE42)) + + if (isArchSupported(TargetArch::SSE42)) { addManyImplSSE42(ptr, start, end); return; @@ -122,7 +135,7 @@ struct AggregateFunctionSumData addManyImpl(ptr, start, end); } - MULTITARGET_FUNCTION_AVX2_SSE42( + MULTITARGET_FUNCTION_AVX512BW_AVX512F_AVX2_SSE42( MULTITARGET_FUNCTION_HEADER( template void NO_SANITIZE_UNDEFINED NO_INLINE @@ -198,12 +211,25 @@ struct AggregateFunctionSumData void NO_INLINE addManyConditionalInternal(const Value * __restrict ptr, const UInt8 * __restrict condition_map, size_t start, size_t end) { #if USE_MULTITARGET_CODE + if (isArchSupported(TargetArch::AVX512BW)) + { + addManyConditionalInternalImplAVX512BW(ptr, condition_map, start, end); + return; + } + + if (isArchSupported(TargetArch::AVX512F)) + { + addManyConditionalInternalImplAVX512F(ptr, condition_map, start, end); + return; + } + if (isArchSupported(TargetArch::AVX2)) { addManyConditionalInternalImplAVX2(ptr, condition_map, start, end); return; } - else if (isArchSupported(TargetArch::SSE42)) + + if (isArchSupported(TargetArch::SSE42)) { addManyConditionalInternalImplSSE42(ptr, condition_map, start, end); return; diff --git a/src/Common/TargetSpecific.h b/src/Common/TargetSpecific.h index b045892d2c1..e5b5ccf2ea5 100644 --- a/src/Common/TargetSpecific.h +++ b/src/Common/TargetSpecific.h @@ -256,7 +256,7 @@ DECLARE_AVX512BW_SPECIFIC_CODE( * class TestClass * { * public: - * MULTITARGET_FUNCTION_AVX2_SSE42( + * MULTITARGET_FUNCTION_AVX512BW_AVX512F_AVX2_SSE42( * MULTITARGET_FUNCTION_HEADER(int), testFunctionImpl, MULTITARGET_FUNCTION_BODY((int value) * { * return value; @@ -264,7 +264,15 @@ DECLARE_AVX512BW_SPECIFIC_CODE( * ) * * void testFunction(int value) { - * if (isArchSupported(TargetArch::AVX2)) + * if (isArchSupported(TargetArch::AVX512BW)) + * { + * testFunctionImplAVX512BW(value); + * } + * else if (isArchSupported(TargetArch::AVX512F)) + * { + * testFunctionImplAVX512F(value); + * } + * else if (isArchSupported(TargetArch::AVX2)) * { * testFunctionImplAVX2(value); * } @@ -290,7 +298,19 @@ DECLARE_AVX512BW_SPECIFIC_CODE( #if ENABLE_MULTITARGET_CODE && defined(__GNUC__) && defined(__x86_64__) /// NOLINTNEXTLINE -#define MULTITARGET_FUNCTION_AVX2_SSE42(FUNCTION_HEADER, name, FUNCTION_BODY) \ +#define MULTITARGET_FUNCTION_AVX512BW_AVX512F_AVX2_SSE42(FUNCTION_HEADER, name, FUNCTION_BODY) \ + FUNCTION_HEADER \ + \ + AVX512BW_FUNCTION_SPECIFIC_ATTRIBUTE \ + name##AVX512BW \ + FUNCTION_BODY \ + \ + FUNCTION_HEADER \ + \ + AVX512_FUNCTION_SPECIFIC_ATTRIBUTE \ + name##AVX512F \ + FUNCTION_BODY \ + \ FUNCTION_HEADER \ \ AVX2_FUNCTION_SPECIFIC_ATTRIBUTE \ @@ -311,7 +331,7 @@ DECLARE_AVX512BW_SPECIFIC_CODE( #else /// NOLINTNEXTLINE -#define MULTITARGET_FUNCTION_AVX2_SSE42(FUNCTION_HEADER, name, FUNCTION_BODY) \ +#define MULTITARGET_FUNCTION_AVX512BW_AVX512F_AVX2_SSE42(FUNCTION_HEADER, name, FUNCTION_BODY) \ FUNCTION_HEADER \ \ name \ diff --git a/src/Functions/FunctionUnaryArithmetic.h b/src/Functions/FunctionUnaryArithmetic.h index 445eb45fd9d..af05f122cae 100644 --- a/src/Functions/FunctionUnaryArithmetic.h +++ b/src/Functions/FunctionUnaryArithmetic.h @@ -42,7 +42,7 @@ struct UnaryOperationImpl using ArrayA = typename ColVecA::Container; using ArrayC = typename ColVecC::Container; - MULTITARGET_FUNCTION_AVX2_SSE42( + MULTITARGET_FUNCTION_AVX512BW_AVX512F_AVX2_SSE42( MULTITARGET_FUNCTION_HEADER(static void NO_INLINE), vectorImpl, MULTITARGET_FUNCTION_BODY((const ArrayA & a, ArrayC & c) /// NOLINT { size_t size = a.size(); @@ -53,12 +53,25 @@ struct UnaryOperationImpl static void NO_INLINE vector(const ArrayA & a, ArrayC & c) { #if USE_MULTITARGET_CODE + if (isArchSupported(TargetArch::AVX512BW)) + { + vectorImplAVX512BW(a, c); + return; + } + + if (isArchSupported(TargetArch::AVX512F)) + { + vectorImplAVX512F(a, c); + return; + } + if (isArchSupported(TargetArch::AVX2)) { vectorImplAVX2(a, c); return; } - else if (isArchSupported(TargetArch::SSE42)) + + if (isArchSupported(TargetArch::SSE42)) { vectorImplSSE42(a, c); return; @@ -78,7 +91,7 @@ struct UnaryOperationImpl template struct FixedStringUnaryOperationImpl { - MULTITARGET_FUNCTION_AVX2_SSE42( + MULTITARGET_FUNCTION_AVX512BW_AVX512F_AVX2_SSE42( MULTITARGET_FUNCTION_HEADER(static void NO_INLINE), vectorImpl, MULTITARGET_FUNCTION_BODY((const ColumnFixedString::Chars & a, /// NOLINT ColumnFixedString::Chars & c) { @@ -90,12 +103,25 @@ struct FixedStringUnaryOperationImpl static void NO_INLINE vector(const ColumnFixedString::Chars & a, ColumnFixedString::Chars & c) { #if USE_MULTITARGET_CODE + if (isArchSupported(TargetArch::AVX512BW)) + { + vectorImplAVX512BW(a, c); + return; + } + + if (isArchSupported(TargetArch::AVX512F)) + { + vectorImplAVX512F(a, c); + return; + } + if (isArchSupported(TargetArch::AVX2)) { vectorImplAVX2(a, c); return; } - else if (isArchSupported(TargetArch::SSE42)) + + if (isArchSupported(TargetArch::SSE42)) { vectorImplSSE42(a, c); return; diff --git a/src/Functions/FunctionsComparison.h b/src/Functions/FunctionsComparison.h index 7bbb1c1096c..2d6491efd21 100644 --- a/src/Functions/FunctionsComparison.h +++ b/src/Functions/FunctionsComparison.h @@ -85,7 +85,7 @@ struct NumComparisonImpl using ContainerA = PaddedPODArray; using ContainerB = PaddedPODArray; - MULTITARGET_FUNCTION_AVX2_SSE42( + MULTITARGET_FUNCTION_AVX512BW_AVX512F_AVX2_SSE42( MULTITARGET_FUNCTION_HEADER(static void), vectorVectorImpl, MULTITARGET_FUNCTION_BODY(( /// NOLINT const ContainerA & a, const ContainerB & b, PaddedPODArray & c) { @@ -112,12 +112,25 @@ struct NumComparisonImpl static void NO_INLINE vectorVector(const ContainerA & a, const ContainerB & b, PaddedPODArray & c) { #if USE_MULTITARGET_CODE + if (isArchSupported(TargetArch::AVX512BW)) + { + vectorVectorImplAVX512BW(a, b, c); + return; + } + + if (isArchSupported(TargetArch::AVX512F)) + { + vectorVectorImplAVX512F(a, b, c); + return; + } + if (isArchSupported(TargetArch::AVX2)) { vectorVectorImplAVX2(a, b, c); return; } - else if (isArchSupported(TargetArch::SSE42)) + + if (isArchSupported(TargetArch::SSE42)) { vectorVectorImplSSE42(a, b, c); return; @@ -128,7 +141,7 @@ struct NumComparisonImpl } - MULTITARGET_FUNCTION_AVX2_SSE42( + MULTITARGET_FUNCTION_AVX512BW_AVX512F_AVX2_SSE42( MULTITARGET_FUNCTION_HEADER(static void), vectorConstantImpl, MULTITARGET_FUNCTION_BODY(( /// NOLINT const ContainerA & a, B b, PaddedPODArray & c) { @@ -148,12 +161,25 @@ struct NumComparisonImpl static void NO_INLINE vectorConstant(const ContainerA & a, B b, PaddedPODArray & c) { #if USE_MULTITARGET_CODE + if (isArchSupported(TargetArch::AVX512BW)) + { + vectorConstantImplAVX512BW(a, b, c); + return; + } + + if (isArchSupported(TargetArch::AVX512F)) + { + vectorConstantImplAVX512F(a, b, c); + return; + } + if (isArchSupported(TargetArch::AVX2)) { vectorConstantImplAVX2(a, b, c); return; } - else if (isArchSupported(TargetArch::SSE42)) + + if (isArchSupported(TargetArch::SSE42)) { vectorConstantImplSSE42(a, b, c); return; From ab131af8432f90ddccd9d857464c445e3435af44 Mon Sep 17 00:00:00 2001 From: zhao zhou Date: Mon, 6 Jun 2022 10:02:21 +0800 Subject: [PATCH 005/717] fix format issue --- src/AggregateFunctions/AggregateFunctionSum.h | 16 ++++++++-------- src/Functions/FunctionsComparison.h | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionSum.h b/src/AggregateFunctions/AggregateFunctionSum.h index 8c5fc0fd6b6..35e36effd0b 100644 --- a/src/AggregateFunctions/AggregateFunctionSum.h +++ b/src/AggregateFunctions/AggregateFunctionSum.h @@ -108,16 +108,16 @@ struct AggregateFunctionSumData { #if USE_MULTITARGET_CODE if (isArchSupported(TargetArch::AVX512BW)) - { - addManyImplAVX512BW(ptr, start, end); - return; - } + { + addManyImplAVX512BW(ptr, start, end); + return; + } if (isArchSupported(TargetArch::AVX512F)) - { - addManyImplAVX512F(ptr, start, end); - return; - } + { + addManyImplAVX512F(ptr, start, end); + return; + } if (isArchSupported(TargetArch::AVX2)) { diff --git a/src/Functions/FunctionsComparison.h b/src/Functions/FunctionsComparison.h index 2d6491efd21..8d24aa89c24 100644 --- a/src/Functions/FunctionsComparison.h +++ b/src/Functions/FunctionsComparison.h @@ -118,7 +118,7 @@ struct NumComparisonImpl return; } - if (isArchSupported(TargetArch::AVX512F)) + if (isArchSupported(TargetArch::AVX512F)) { vectorVectorImplAVX512F(a, b, c); return; From 824656a3c73a89a4e45a7c2fd0a52b4d1ef127c9 Mon Sep 17 00:00:00 2001 From: zhao zhou Date: Mon, 6 Jun 2022 10:07:45 +0800 Subject: [PATCH 006/717] fix format issue --- src/Functions/FunctionsComparison.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/FunctionsComparison.h b/src/Functions/FunctionsComparison.h index 8d24aa89c24..2974abe062b 100644 --- a/src/Functions/FunctionsComparison.h +++ b/src/Functions/FunctionsComparison.h @@ -130,7 +130,7 @@ struct NumComparisonImpl return; } - if (isArchSupported(TargetArch::SSE42)) + if (isArchSupported(TargetArch::SSE42)) { vectorVectorImplSSE42(a, b, c); return; From 0b014e0caaf8b808f2240c398ecc7bb1e5941f65 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Fri, 2 Sep 2022 11:06:32 -0300 Subject: [PATCH 007/717] Add MT select final setting --- src/Interpreters/InterpreterSelectQuery.cpp | 6 ++++++ src/Storages/IStorage.h | 2 ++ src/Storages/MergeTree/MergeTreeData.h | 2 ++ src/Storages/MergeTree/MergeTreeSettings.h | 1 + src/Storages/MergeTree/registerStorageMergeTree.cpp | 7 +++++++ 5 files changed, 18 insertions(+) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index c73db82a27b..7bf7dd920ac 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -495,6 +495,12 @@ InterpreterSelectQuery::InterpreterSelectQuery( query_info.additional_filter_ast = parseAdditionalFilterConditionForTable( settings.additional_table_filters, joined_tables.tablesWithColumns().front().table, *context); + + if (!query.final() && storage && storage->forceQueryWithFinal()) + { + query.setFinal(); + } + auto analyze = [&] (bool try_move_to_prewhere) { /// Allow push down and other optimizations for VIEW: replace with subquery and rewrite it. diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index a61bfeaff57..a9709fd7610 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -325,6 +325,8 @@ public: /// It's needed for ReplacingMergeTree wrappers such as MaterializedMySQL and MaterializedPostrgeSQL virtual bool needRewriteQueryWithFinal(const Names & /*column_names*/) const { return false; } + virtual bool forceQueryWithFinal() const { return false; } + private: /** Read a set of columns from the table. * Accepts a list of columns to read, as well as a description of the query, diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index c91c7ba02a8..ad04752bd0b 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -449,6 +449,8 @@ public: bool mayBenefitFromIndexForIn(const ASTPtr & left_in_operand, ContextPtr, const StorageMetadataPtr & metadata_snapshot) const override; + bool forceQueryWithFinal() const override { return getSettings()->force_select_final && supportsFinal(); } + /// Snapshot for MergeTree contains the current set of data parts /// at the moment of the start of query. struct SnapshotData : public StorageSnapshot::Data diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index 07659b1c9dc..34810d38645 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -144,6 +144,7 @@ struct Settings; M(Bool, allow_remote_fs_zero_copy_replication, false, "Don't use this setting in production, because it is not ready.", 0) \ M(String, remote_fs_zero_copy_zookeeper_path, "/clickhouse/zero_copy", "ZooKeeper path for Zero-copy table-independet info.", 0) \ M(Bool, remote_fs_zero_copy_path_compatible_mode, false, "Run zero-copy in compatible mode during conversion process.", 0) \ + M(Bool, force_select_final, false, "Query with the FINAL modifier by default", 0) \ \ /** Obsolete settings. Kept for backward compatibility only. */ \ M(UInt64, min_relative_delay_to_yield_leadership, 120, "Obsolete setting, does nothing.", 0) \ diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index e52a0fed674..41075e29a9d 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -34,6 +34,7 @@ namespace ErrorCodes extern const int NO_REPLICA_NAME_GIVEN; extern const int CANNOT_EXTRACT_TABLE_STRUCTURE; extern const int NOT_IMPLEMENTED; + extern const int ILLEGAL_FINAL; } @@ -677,6 +678,12 @@ static StoragePtr create(const StorageFactory::Arguments & args) if (arg_num != arg_cnt) throw Exception("Wrong number of engine arguments.", ErrorCodes::BAD_ARGUMENTS); + + if (merging_params.mode == MergeTreeData::MergingParams::Mode::Ordinary && storage_settings->force_select_final) + { + throw Exception("Storage MergeTree doesn't support FINAL", ErrorCodes::ILLEGAL_FINAL); + } + if (replicated) { auto storage_policy = args.getContext()->getStoragePolicy(storage_settings->storage_policy); From 2c0d337733a6169d4f6a125947be1c8cde337b65 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Wed, 7 Sep 2022 09:07:29 -0300 Subject: [PATCH 008/717] Add tests & fix select count --- src/Storages/MergeTree/MergeTreeData.cpp | 2 +- .../02420_mt_ordinary_select_final_setting.reference | 0 .../0_stateless/02420_mt_ordinary_select_final_setting.sql | 1 + .../02420_mt_replacing_select_final_setting.reference | 1 + .../0_stateless/02420_mt_replacing_select_final_setting.sql | 6 ++++++ 5 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/02420_mt_ordinary_select_final_setting.reference create mode 100644 tests/queries/0_stateless/02420_mt_ordinary_select_final_setting.sql create mode 100644 tests/queries/0_stateless/02420_mt_replacing_select_final_setting.reference create mode 100644 tests/queries/0_stateless/02420_mt_replacing_select_final_setting.sql diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index b7b68367e98..21f6c98e309 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -5558,7 +5558,7 @@ std::optional MergeTreeData::getQueryProcessingStageWithAgg if (settings.parallel_replicas_count > 1 || settings.max_parallel_replicas > 1) return std::nullopt; - auto query_ptr = query_info.original_query; + auto query_ptr = query_info.query; auto * select_query = query_ptr->as(); if (!select_query) return std::nullopt; diff --git a/tests/queries/0_stateless/02420_mt_ordinary_select_final_setting.reference b/tests/queries/0_stateless/02420_mt_ordinary_select_final_setting.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02420_mt_ordinary_select_final_setting.sql b/tests/queries/0_stateless/02420_mt_ordinary_select_final_setting.sql new file mode 100644 index 00000000000..ec4f17fbcf4 --- /dev/null +++ b/tests/queries/0_stateless/02420_mt_ordinary_select_final_setting.sql @@ -0,0 +1 @@ +create table if not exists ordinary_mt (x String) engine=MergeTree() ORDER BY x SETTINGS force_select_final=1; -- { serverError 181 } \ No newline at end of file diff --git a/tests/queries/0_stateless/02420_mt_replacing_select_final_setting.reference b/tests/queries/0_stateless/02420_mt_replacing_select_final_setting.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/02420_mt_replacing_select_final_setting.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/02420_mt_replacing_select_final_setting.sql b/tests/queries/0_stateless/02420_mt_replacing_select_final_setting.sql new file mode 100644 index 00000000000..cc5cf9b3123 --- /dev/null +++ b/tests/queries/0_stateless/02420_mt_replacing_select_final_setting.sql @@ -0,0 +1,6 @@ +create table if not exists replacing_mt (x String) engine=ReplacingMergeTree() ORDER BY x SETTINGS force_select_final=1; + +insert into replacing_mt values ('abc'); +insert into replacing_mt values ('abc'); + +select count() from replacing_mt \ No newline at end of file From a6d106c31250a4b294016801a334ca8b29866e84 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Wed, 7 Sep 2022 10:25:16 -0300 Subject: [PATCH 009/717] tmp --- src/Storages/MergeTree/MergeTreeData.h | 5 ++++- src/Storages/ReadFinalForExternalReplicaStorage.cpp | 3 +++ src/Storages/StorageMaterializedMySQL.cpp | 3 +-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index ad04752bd0b..fba09854603 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -449,8 +449,11 @@ public: bool mayBenefitFromIndexForIn(const ASTPtr & left_in_operand, ContextPtr, const StorageMetadataPtr & metadata_snapshot) const override; - bool forceQueryWithFinal() const override { return getSettings()->force_select_final && supportsFinal(); } + virtual bool needRewriteQueryWithFinal(const Names & /*column_names*/) const override { return getSettings()->force_select_final && supportsFinal(); } + bool forceQueryWithFinal() const override { + return getSettings()->force_select_final && supportsFinal(); + } /// Snapshot for MergeTree contains the current set of data parts /// at the moment of the start of query. struct SnapshotData : public StorageSnapshot::Data diff --git a/src/Storages/ReadFinalForExternalReplicaStorage.cpp b/src/Storages/ReadFinalForExternalReplicaStorage.cpp index 3ec7a074fd4..14bef0167ba 100644 --- a/src/Storages/ReadFinalForExternalReplicaStorage.cpp +++ b/src/Storages/ReadFinalForExternalReplicaStorage.cpp @@ -21,6 +21,9 @@ namespace DB bool needRewriteQueryWithFinalForStorage(const Names & column_names, const StoragePtr & storage) { + if (storage->needRewriteQueryWithFinal(column_names)) { + return true; + } const StorageMetadataPtr & metadata = storage->getInMemoryMetadataPtr(); Block header = metadata->getSampleBlock(); ColumnWithTypeAndName & version_column = header.getByPosition(header.columns() - 1); diff --git a/src/Storages/StorageMaterializedMySQL.cpp b/src/Storages/StorageMaterializedMySQL.cpp index a7e54960563..de7f8d7e589 100644 --- a/src/Storages/StorageMaterializedMySQL.cpp +++ b/src/Storages/StorageMaterializedMySQL.cpp @@ -27,8 +27,7 @@ StorageMaterializedMySQL::StorageMaterializedMySQL(const StoragePtr & nested_sto setInMemoryMetadata(in_memory_metadata); } -bool StorageMaterializedMySQL::needRewriteQueryWithFinal(const Names & column_names) const -{ +bool StorageMaterializedMySQL::needRewriteQueryWithFinal(const Names & column_names) const { return needRewriteQueryWithFinalForStorage(column_names, nested_storage); } From 64ce1922aba83cf852bb59ac65f0071c210376e9 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Wed, 21 Sep 2022 14:45:43 -0300 Subject: [PATCH 010/717] Minor fixes --- src/Storages/MergeTree/MergeTreeData.cpp | 18 +++++++++++------- .../02420_mt_ordinary_select_final_setting.sql | 2 +- ...02420_mt_replacing_select_final_setting.sql | 2 +- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 21f6c98e309..a89201ecbe3 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -5559,16 +5559,20 @@ std::optional MergeTreeData::getQueryProcessingStageWithAgg return std::nullopt; auto query_ptr = query_info.query; + auto original_query_ptr = query_info.original_query; + auto * select_query = query_ptr->as(); - if (!select_query) + auto * original_select_query = original_query_ptr->as(); + + if (!original_select_query || select_query) return std::nullopt; // Currently projections don't support final yet. - if (select_query->final()) + if (select_query->final() || original_select_query->final()) return std::nullopt; // Currently projections don't support sample yet. - if (select_query->sampleSize()) + if (original_select_query->sampleSize()) return std::nullopt; // Currently projection don't support deduplication when moving parts between shards. @@ -5576,21 +5580,21 @@ std::optional MergeTreeData::getQueryProcessingStageWithAgg return std::nullopt; // Currently projections don't support ARRAY JOIN yet. - if (select_query->arrayJoinExpressionList().first) + if (original_select_query->arrayJoinExpressionList().first) return std::nullopt; // In order to properly analyze joins, aliases should be recognized. However, aliases get lost during projection analysis. // Let's disable projection if there are any JOIN clauses. // TODO: We need a better identifier resolution mechanism for projection analysis. - if (select_query->hasJoin()) + if (original_select_query->hasJoin()) return std::nullopt; // INTERPOLATE expressions may include aliases, so aliases should be preserved - if (select_query->interpolate() && !select_query->interpolate()->children.empty()) + if (original_select_query->interpolate() && !original_select_query->interpolate()->children.empty()) return std::nullopt; // Currently projections don't support GROUPING SET yet. - if (select_query->group_by_with_grouping_sets) + if (original_select_query->group_by_with_grouping_sets) return std::nullopt; auto query_options = SelectQueryOptions( diff --git a/tests/queries/0_stateless/02420_mt_ordinary_select_final_setting.sql b/tests/queries/0_stateless/02420_mt_ordinary_select_final_setting.sql index ec4f17fbcf4..6575de9c59b 100644 --- a/tests/queries/0_stateless/02420_mt_ordinary_select_final_setting.sql +++ b/tests/queries/0_stateless/02420_mt_ordinary_select_final_setting.sql @@ -1 +1 @@ -create table if not exists ordinary_mt (x String) engine=MergeTree() ORDER BY x SETTINGS force_select_final=1; -- { serverError 181 } \ No newline at end of file +create table if not exists ordinary_mt (x String) engine=MergeTree() ORDER BY x SETTINGS force_select_final=1; -- { serverError 181 } diff --git a/tests/queries/0_stateless/02420_mt_replacing_select_final_setting.sql b/tests/queries/0_stateless/02420_mt_replacing_select_final_setting.sql index cc5cf9b3123..3db1d5999b8 100644 --- a/tests/queries/0_stateless/02420_mt_replacing_select_final_setting.sql +++ b/tests/queries/0_stateless/02420_mt_replacing_select_final_setting.sql @@ -3,4 +3,4 @@ create table if not exists replacing_mt (x String) engine=ReplacingMergeTree() O insert into replacing_mt values ('abc'); insert into replacing_mt values ('abc'); -select count() from replacing_mt \ No newline at end of file +select count() from replacing_mt From f97dc8bf6ad4880fc82dc2135cf60413797be758 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Wed, 28 Sep 2022 08:21:17 -0300 Subject: [PATCH 011/717] Fix styling and remove experiment --- src/Storages/MergeTree/MergeTreeData.cpp | 2 +- src/Storages/MergeTree/MergeTreeData.h | 5 ++--- src/Storages/ReadFinalForExternalReplicaStorage.cpp | 3 --- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index a89201ecbe3..70e46dcfa70 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -5564,7 +5564,7 @@ std::optional MergeTreeData::getQueryProcessingStageWithAgg auto * select_query = query_ptr->as(); auto * original_select_query = original_query_ptr->as(); - if (!original_select_query || select_query) + if (!original_select_query || !select_query) return std::nullopt; // Currently projections don't support final yet. diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index fba09854603..0ae382ccb73 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -449,9 +449,8 @@ public: bool mayBenefitFromIndexForIn(const ASTPtr & left_in_operand, ContextPtr, const StorageMetadataPtr & metadata_snapshot) const override; - virtual bool needRewriteQueryWithFinal(const Names & /*column_names*/) const override { return getSettings()->force_select_final && supportsFinal(); } - - bool forceQueryWithFinal() const override { + bool forceQueryWithFinal() const override + { return getSettings()->force_select_final && supportsFinal(); } /// Snapshot for MergeTree contains the current set of data parts diff --git a/src/Storages/ReadFinalForExternalReplicaStorage.cpp b/src/Storages/ReadFinalForExternalReplicaStorage.cpp index 14bef0167ba..3ec7a074fd4 100644 --- a/src/Storages/ReadFinalForExternalReplicaStorage.cpp +++ b/src/Storages/ReadFinalForExternalReplicaStorage.cpp @@ -21,9 +21,6 @@ namespace DB bool needRewriteQueryWithFinalForStorage(const Names & column_names, const StoragePtr & storage) { - if (storage->needRewriteQueryWithFinal(column_names)) { - return true; - } const StorageMetadataPtr & metadata = storage->getInMemoryMetadataPtr(); Block header = metadata->getSampleBlock(); ColumnWithTypeAndName & version_column = header.getByPosition(header.columns() - 1); From ca4e109c0908aa0a097f8300fa31f72ddf454168 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Wed, 28 Sep 2022 08:44:12 -0300 Subject: [PATCH 012/717] fix style --- src/Storages/StorageMaterializedMySQL.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Storages/StorageMaterializedMySQL.cpp b/src/Storages/StorageMaterializedMySQL.cpp index de7f8d7e589..a7e54960563 100644 --- a/src/Storages/StorageMaterializedMySQL.cpp +++ b/src/Storages/StorageMaterializedMySQL.cpp @@ -27,7 +27,8 @@ StorageMaterializedMySQL::StorageMaterializedMySQL(const StoragePtr & nested_sto setInMemoryMetadata(in_memory_metadata); } -bool StorageMaterializedMySQL::needRewriteQueryWithFinal(const Names & column_names) const { +bool StorageMaterializedMySQL::needRewriteQueryWithFinal(const Names & column_names) const +{ return needRewriteQueryWithFinalForStorage(column_names, nested_storage); } From 6a57ea922797c7cdd3334fceb13f072d360aa6fa Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 18 Oct 2022 17:13:54 -0300 Subject: [PATCH 013/717] Add tests to validate select final setting works on joins --- ...ing_join_select_final_setting_both_tables.reference | 1 + ...replacing_join_select_final_setting_both_tables.sql | 10 ++++++++++ ...lacing_join_select_final_setting_rhs_only.reference | 1 + ...mt_replacing_join_select_final_setting_rhs_only.sql | 10 ++++++++++ 4 files changed, 22 insertions(+) create mode 100644 tests/queries/0_stateless/02420_mt_replacing_join_select_final_setting_both_tables.reference create mode 100644 tests/queries/0_stateless/02420_mt_replacing_join_select_final_setting_both_tables.sql create mode 100644 tests/queries/0_stateless/02420_mt_replacing_join_select_final_setting_rhs_only.reference create mode 100644 tests/queries/0_stateless/02420_mt_replacing_join_select_final_setting_rhs_only.sql diff --git a/tests/queries/0_stateless/02420_mt_replacing_join_select_final_setting_both_tables.reference b/tests/queries/0_stateless/02420_mt_replacing_join_select_final_setting_both_tables.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/02420_mt_replacing_join_select_final_setting_both_tables.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/02420_mt_replacing_join_select_final_setting_both_tables.sql b/tests/queries/0_stateless/02420_mt_replacing_join_select_final_setting_both_tables.sql new file mode 100644 index 00000000000..5f7fdaa2f70 --- /dev/null +++ b/tests/queries/0_stateless/02420_mt_replacing_join_select_final_setting_both_tables.sql @@ -0,0 +1,10 @@ +create table if not exists lhs (x String) engine=ReplacingMergeTree() ORDER BY x SETTINGS force_select_final=1; +create table if not exists rhs (x String) engine=ReplacingMergeTree() ORDER BY x SETTINGS force_select_final=1; + +insert into lhs values ('abc'); +insert into lhs values ('abc'); + +insert into rhs values ('abc'); +insert into rhs values ('abc'); + +select count() from lhs inner join rhs on lhs.x = rhs.x; diff --git a/tests/queries/0_stateless/02420_mt_replacing_join_select_final_setting_rhs_only.reference b/tests/queries/0_stateless/02420_mt_replacing_join_select_final_setting_rhs_only.reference new file mode 100644 index 00000000000..0cfbf08886f --- /dev/null +++ b/tests/queries/0_stateless/02420_mt_replacing_join_select_final_setting_rhs_only.reference @@ -0,0 +1 @@ +2 diff --git a/tests/queries/0_stateless/02420_mt_replacing_join_select_final_setting_rhs_only.sql b/tests/queries/0_stateless/02420_mt_replacing_join_select_final_setting_rhs_only.sql new file mode 100644 index 00000000000..c03b8db9e83 --- /dev/null +++ b/tests/queries/0_stateless/02420_mt_replacing_join_select_final_setting_rhs_only.sql @@ -0,0 +1,10 @@ +create table if not exists lhs (x String) engine=ReplacingMergeTree() ORDER BY x; +create table if not exists rhs (x String) engine=ReplacingMergeTree() ORDER BY x SETTINGS force_select_final=1; + +insert into lhs values ('abc'); +insert into lhs values ('abc'); + +insert into rhs values ('abc'); +insert into rhs values ('abc'); + +select count() from lhs inner join rhs on lhs.x = rhs.x; From 0f6b8770734c306f701d847b97cc34d577d873a0 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Fri, 21 Oct 2022 16:27:59 -0300 Subject: [PATCH 014/717] Add ignore_force_select_final setting --- src/Core/Settings.h | 2 ++ src/Interpreters/InterpreterSelectQuery.cpp | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 5dedc6117aa..e015806b562 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -269,6 +269,8 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(Milliseconds, stream_flush_interval_ms, 7500, "Timeout for flushing data from streaming storages.", 0) \ M(Milliseconds, stream_poll_timeout_ms, 500, "Timeout for polling data from/to streaming storages.", 0) \ \ + M(Bool, ignore_force_select_final, false, "Ignores the MT force_select_final setting", 0) \ + \ /** Settings for testing hedged requests */ \ M(Milliseconds, sleep_in_send_tables_status_ms, 0, "Time to sleep in sending tables status response in TCPHandler", 0) \ M(Milliseconds, sleep_in_send_data_ms, 0, "Time to sleep in sending data in TCPHandler", 0) \ diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 1229ac75083..303302d8259 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -496,7 +496,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( settings.additional_table_filters, joined_tables.tablesWithColumns().front().table, *context); - if (!query.final() && storage && storage->forceQueryWithFinal()) + if (!query.final() && storage && storage->forceQueryWithFinal() && !context->getSettingsRef().ignore_force_select_final) { query.setFinal(); } From dcbd5d9c3d34461a5c58e5be47d36109c9329a0f Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 25 Oct 2022 09:49:48 -0300 Subject: [PATCH 015/717] Add ignore force select final setting test --- ...gnore_force_select_final_setting.reference | 16 ++++++++++++++++ ...0_mt_ignore_force_select_final_setting.sql | 19 +++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 tests/queries/0_stateless/02420_mt_ignore_force_select_final_setting.reference create mode 100644 tests/queries/0_stateless/02420_mt_ignore_force_select_final_setting.sql diff --git a/tests/queries/0_stateless/02420_mt_ignore_force_select_final_setting.reference b/tests/queries/0_stateless/02420_mt_ignore_force_select_final_setting.reference new file mode 100644 index 00000000000..ac177354f4a --- /dev/null +++ b/tests/queries/0_stateless/02420_mt_ignore_force_select_final_setting.reference @@ -0,0 +1,16 @@ +-- { echoOn } + +SELECT * FROM tbl WHERE id = 10000; -- single row expected (bar), because the force_select_final is in action +10000 bar +SELECT * FROM tbl final WHERE id = 10000; -- single row expected (bar), because the force_select_final is in action and FINAL is there. +10000 bar +SELECT * FROM tbl WHERE id = 10000 SETTINGS ignore_force_select_final=1; -- now we see 2 'real' rows +10000 bar +10000 foo +SELECT * FROM tbl FINAL WHERE id = 10000 SETTINGS ignore_force_select_final=1; -- now we see single row again. +10000 bar +SYSTEM START MERGES tbl; +OPTIMIZE TABLE tbl FINAL; +SELECT * FROM tbl WHERE id = 10000 SETTINGS ignore_force_select_final=1; -- now we see single row DROP TABLE tbl; +10000 bar +DROP TABLE tbl; diff --git a/tests/queries/0_stateless/02420_mt_ignore_force_select_final_setting.sql b/tests/queries/0_stateless/02420_mt_ignore_force_select_final_setting.sql new file mode 100644 index 00000000000..fdcdea2bc23 --- /dev/null +++ b/tests/queries/0_stateless/02420_mt_ignore_force_select_final_setting.sql @@ -0,0 +1,19 @@ +CREATE TABLE tbl (id UInt64, val String) engine=ReplacingMergeTree() ORDER BY id SETTINGS force_select_final=1; + +SYSTEM STOP MERGES tbl; +INSERT INTO tbl SELECT number as id, 'foo' AS val FROM numbers(100000); +INSERT INTO tbl SELECT number as id, 'bar' AS val FROM numbers(100000); + +-- { echoOn } + +SELECT * FROM tbl WHERE id = 10000; -- single row expected (bar), because the force_select_final is in action +SELECT * FROM tbl final WHERE id = 10000; -- single row expected (bar), because the force_select_final is in action and FINAL is there. + +SELECT * FROM tbl WHERE id = 10000 SETTINGS ignore_force_select_final=1; -- now we see 2 'real' rows +SELECT * FROM tbl FINAL WHERE id = 10000 SETTINGS ignore_force_select_final=1; -- now we see single row again. + +SYSTEM START MERGES tbl; +OPTIMIZE TABLE tbl FINAL; + +SELECT * FROM tbl WHERE id = 10000 SETTINGS ignore_force_select_final=1; -- now we see single row DROP TABLE tbl; +DROP TABLE tbl; \ No newline at end of file From 687677399e28ed4e2e027e540832cd75d60d4ccc Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 25 Oct 2022 10:06:15 -0300 Subject: [PATCH 016/717] group force select final setting tests together --- ...02420_force_select_final_setting.reference | 32 ++++++++++++++ .../02420_force_select_final_setting.sql | 42 +++++++++++++++++++ ...mt_ordinary_select_final_setting.reference | 0 ...02420_mt_ordinary_select_final_setting.sql | 1 - ...select_final_setting_both_tables.reference | 1 - ..._join_select_final_setting_both_tables.sql | 10 ----- ...in_select_final_setting_rhs_only.reference | 1 - ...ing_join_select_final_setting_rhs_only.sql | 10 ----- ...t_replacing_select_final_setting.reference | 1 - ...2420_mt_replacing_select_final_setting.sql | 6 --- 10 files changed, 74 insertions(+), 30 deletions(-) create mode 100644 tests/queries/0_stateless/02420_force_select_final_setting.reference create mode 100644 tests/queries/0_stateless/02420_force_select_final_setting.sql delete mode 100644 tests/queries/0_stateless/02420_mt_ordinary_select_final_setting.reference delete mode 100644 tests/queries/0_stateless/02420_mt_ordinary_select_final_setting.sql delete mode 100644 tests/queries/0_stateless/02420_mt_replacing_join_select_final_setting_both_tables.reference delete mode 100644 tests/queries/0_stateless/02420_mt_replacing_join_select_final_setting_both_tables.sql delete mode 100644 tests/queries/0_stateless/02420_mt_replacing_join_select_final_setting_rhs_only.reference delete mode 100644 tests/queries/0_stateless/02420_mt_replacing_join_select_final_setting_rhs_only.sql delete mode 100644 tests/queries/0_stateless/02420_mt_replacing_select_final_setting.reference delete mode 100644 tests/queries/0_stateless/02420_mt_replacing_select_final_setting.sql diff --git a/tests/queries/0_stateless/02420_force_select_final_setting.reference b/tests/queries/0_stateless/02420_force_select_final_setting.reference new file mode 100644 index 00000000000..466be527e6f --- /dev/null +++ b/tests/queries/0_stateless/02420_force_select_final_setting.reference @@ -0,0 +1,32 @@ +-- { echoOn } +create table if not exists ordinary_mt (x String) engine=MergeTree() ORDER BY x SETTINGS force_select_final=1; -- { serverError 181 } +-- simple test case +create table if not exists replacing_mt (x String) engine=ReplacingMergeTree() ORDER BY x SETTINGS force_select_final=1; +insert into replacing_mt values ('abc'); +insert into replacing_mt values ('abc'); +-- expected output is 1 because force_select_final is turned on +select count() from replacing_mt; +1 +-- JOIN test cases +create table if not exists lhs (x String) engine=ReplacingMergeTree() ORDER BY x SETTINGS force_select_final=1; +create table if not exists rhs (x String) engine=ReplacingMergeTree() ORDER BY x SETTINGS force_select_final=1; +insert into lhs values ('abc'); +insert into lhs values ('abc'); +insert into rhs values ('abc'); +insert into rhs values ('abc'); +-- expected output is 1 because both tables have force_select_final = 1 +select count() from lhs inner join rhs on lhs.x = rhs.x; +1 +drop table lhs; +drop table rhs; +create table if not exists lhs (x String) engine=ReplacingMergeTree() ORDER BY x; +create table if not exists rhs (x String) engine=ReplacingMergeTree() ORDER BY x SETTINGS force_select_final=1; +insert into lhs values ('abc'); +insert into lhs values ('abc'); +insert into rhs values ('abc'); +insert into rhs values ('abc'); +-- expected output is 2 because lhs table doesn't have final applied +select count() from lhs inner join rhs on lhs.x = rhs.x; +2 +-- Engine that does not support final +create table if not exists ordinary_mt (x String) engine=MergeTree() ORDER BY x SETTINGS force_select_final=1; -- { serverError 181 } diff --git a/tests/queries/0_stateless/02420_force_select_final_setting.sql b/tests/queries/0_stateless/02420_force_select_final_setting.sql new file mode 100644 index 00000000000..fae3a74e465 --- /dev/null +++ b/tests/queries/0_stateless/02420_force_select_final_setting.sql @@ -0,0 +1,42 @@ +-- { echoOn } +create table if not exists ordinary_mt (x String) engine=MergeTree() ORDER BY x SETTINGS force_select_final=1; -- { serverError 181 } + +-- simple test case +create table if not exists replacing_mt (x String) engine=ReplacingMergeTree() ORDER BY x SETTINGS force_select_final=1; + +insert into replacing_mt values ('abc'); +insert into replacing_mt values ('abc'); + +-- expected output is 1 because force_select_final is turned on +select count() from replacing_mt; + +-- JOIN test cases +create table if not exists lhs (x String) engine=ReplacingMergeTree() ORDER BY x SETTINGS force_select_final=1; +create table if not exists rhs (x String) engine=ReplacingMergeTree() ORDER BY x SETTINGS force_select_final=1; + +insert into lhs values ('abc'); +insert into lhs values ('abc'); + +insert into rhs values ('abc'); +insert into rhs values ('abc'); + +-- expected output is 1 because both tables have force_select_final = 1 +select count() from lhs inner join rhs on lhs.x = rhs.x; + +drop table lhs; +drop table rhs; + +create table if not exists lhs (x String) engine=ReplacingMergeTree() ORDER BY x; +create table if not exists rhs (x String) engine=ReplacingMergeTree() ORDER BY x SETTINGS force_select_final=1; + +insert into lhs values ('abc'); +insert into lhs values ('abc'); + +insert into rhs values ('abc'); +insert into rhs values ('abc'); + +-- expected output is 2 because lhs table doesn't have final applied +select count() from lhs inner join rhs on lhs.x = rhs.x; + +-- Engine that does not support final +create table if not exists ordinary_mt (x String) engine=MergeTree() ORDER BY x SETTINGS force_select_final=1; -- { serverError 181 } diff --git a/tests/queries/0_stateless/02420_mt_ordinary_select_final_setting.reference b/tests/queries/0_stateless/02420_mt_ordinary_select_final_setting.reference deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/queries/0_stateless/02420_mt_ordinary_select_final_setting.sql b/tests/queries/0_stateless/02420_mt_ordinary_select_final_setting.sql deleted file mode 100644 index 6575de9c59b..00000000000 --- a/tests/queries/0_stateless/02420_mt_ordinary_select_final_setting.sql +++ /dev/null @@ -1 +0,0 @@ -create table if not exists ordinary_mt (x String) engine=MergeTree() ORDER BY x SETTINGS force_select_final=1; -- { serverError 181 } diff --git a/tests/queries/0_stateless/02420_mt_replacing_join_select_final_setting_both_tables.reference b/tests/queries/0_stateless/02420_mt_replacing_join_select_final_setting_both_tables.reference deleted file mode 100644 index d00491fd7e5..00000000000 --- a/tests/queries/0_stateless/02420_mt_replacing_join_select_final_setting_both_tables.reference +++ /dev/null @@ -1 +0,0 @@ -1 diff --git a/tests/queries/0_stateless/02420_mt_replacing_join_select_final_setting_both_tables.sql b/tests/queries/0_stateless/02420_mt_replacing_join_select_final_setting_both_tables.sql deleted file mode 100644 index 5f7fdaa2f70..00000000000 --- a/tests/queries/0_stateless/02420_mt_replacing_join_select_final_setting_both_tables.sql +++ /dev/null @@ -1,10 +0,0 @@ -create table if not exists lhs (x String) engine=ReplacingMergeTree() ORDER BY x SETTINGS force_select_final=1; -create table if not exists rhs (x String) engine=ReplacingMergeTree() ORDER BY x SETTINGS force_select_final=1; - -insert into lhs values ('abc'); -insert into lhs values ('abc'); - -insert into rhs values ('abc'); -insert into rhs values ('abc'); - -select count() from lhs inner join rhs on lhs.x = rhs.x; diff --git a/tests/queries/0_stateless/02420_mt_replacing_join_select_final_setting_rhs_only.reference b/tests/queries/0_stateless/02420_mt_replacing_join_select_final_setting_rhs_only.reference deleted file mode 100644 index 0cfbf08886f..00000000000 --- a/tests/queries/0_stateless/02420_mt_replacing_join_select_final_setting_rhs_only.reference +++ /dev/null @@ -1 +0,0 @@ -2 diff --git a/tests/queries/0_stateless/02420_mt_replacing_join_select_final_setting_rhs_only.sql b/tests/queries/0_stateless/02420_mt_replacing_join_select_final_setting_rhs_only.sql deleted file mode 100644 index c03b8db9e83..00000000000 --- a/tests/queries/0_stateless/02420_mt_replacing_join_select_final_setting_rhs_only.sql +++ /dev/null @@ -1,10 +0,0 @@ -create table if not exists lhs (x String) engine=ReplacingMergeTree() ORDER BY x; -create table if not exists rhs (x String) engine=ReplacingMergeTree() ORDER BY x SETTINGS force_select_final=1; - -insert into lhs values ('abc'); -insert into lhs values ('abc'); - -insert into rhs values ('abc'); -insert into rhs values ('abc'); - -select count() from lhs inner join rhs on lhs.x = rhs.x; diff --git a/tests/queries/0_stateless/02420_mt_replacing_select_final_setting.reference b/tests/queries/0_stateless/02420_mt_replacing_select_final_setting.reference deleted file mode 100644 index d00491fd7e5..00000000000 --- a/tests/queries/0_stateless/02420_mt_replacing_select_final_setting.reference +++ /dev/null @@ -1 +0,0 @@ -1 diff --git a/tests/queries/0_stateless/02420_mt_replacing_select_final_setting.sql b/tests/queries/0_stateless/02420_mt_replacing_select_final_setting.sql deleted file mode 100644 index 3db1d5999b8..00000000000 --- a/tests/queries/0_stateless/02420_mt_replacing_select_final_setting.sql +++ /dev/null @@ -1,6 +0,0 @@ -create table if not exists replacing_mt (x String) engine=ReplacingMergeTree() ORDER BY x SETTINGS force_select_final=1; - -insert into replacing_mt values ('abc'); -insert into replacing_mt values ('abc'); - -select count() from replacing_mt From 1423eb0cbcd0d9e038370025e3f6a4f9b571c4f8 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 25 Oct 2022 11:09:59 -0300 Subject: [PATCH 017/717] Make ignore_force_select_final setting description more descriptive Co-authored-by: filimonov <1549571+filimonov@users.noreply.github.com> --- src/Core/Settings.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index e015806b562..04f444e2ab8 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -269,7 +269,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(Milliseconds, stream_flush_interval_ms, 7500, "Timeout for flushing data from streaming storages.", 0) \ M(Milliseconds, stream_poll_timeout_ms, 500, "Timeout for polling data from/to streaming storages.", 0) \ \ - M(Bool, ignore_force_select_final, false, "Ignores the MT force_select_final setting", 0) \ + M(Bool, ignore_force_select_final, false, "Ignores the mergetree setting force_select_final, allowing to read the raw data (without applying FINAL automatically) is force_select_final is enabled", 0) \ \ /** Settings for testing hedged requests */ \ M(Milliseconds, sleep_in_send_tables_status_ms, 0, "Time to sleep in sending tables status response in TCPHandler", 0) \ From f2f165b612ae1c6e9f5f8aae290dcb4299c21a7d Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Wed, 26 Oct 2022 07:54:33 -0300 Subject: [PATCH 018/717] fix test --- .../02420_mt_ignore_force_select_final_setting.reference | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02420_mt_ignore_force_select_final_setting.reference b/tests/queries/0_stateless/02420_mt_ignore_force_select_final_setting.reference index ac177354f4a..1d270752668 100644 --- a/tests/queries/0_stateless/02420_mt_ignore_force_select_final_setting.reference +++ b/tests/queries/0_stateless/02420_mt_ignore_force_select_final_setting.reference @@ -5,8 +5,8 @@ SELECT * FROM tbl WHERE id = 10000; -- single row expected (bar), because the fo SELECT * FROM tbl final WHERE id = 10000; -- single row expected (bar), because the force_select_final is in action and FINAL is there. 10000 bar SELECT * FROM tbl WHERE id = 10000 SETTINGS ignore_force_select_final=1; -- now we see 2 'real' rows -10000 bar 10000 foo +10000 bar SELECT * FROM tbl FINAL WHERE id = 10000 SETTINGS ignore_force_select_final=1; -- now we see single row again. 10000 bar SYSTEM START MERGES tbl; From 5708780c44dbc9dbb952a149435826caeb3cfa42 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Mon, 31 Oct 2022 11:09:25 -0300 Subject: [PATCH 019/717] add order by to tests to guarantee ordering --- .../02420_mt_ignore_force_select_final_setting.reference | 2 +- .../0_stateless/02420_mt_ignore_force_select_final_setting.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02420_mt_ignore_force_select_final_setting.reference b/tests/queries/0_stateless/02420_mt_ignore_force_select_final_setting.reference index 1d270752668..1ae1def4cb1 100644 --- a/tests/queries/0_stateless/02420_mt_ignore_force_select_final_setting.reference +++ b/tests/queries/0_stateless/02420_mt_ignore_force_select_final_setting.reference @@ -4,7 +4,7 @@ SELECT * FROM tbl WHERE id = 10000; -- single row expected (bar), because the fo 10000 bar SELECT * FROM tbl final WHERE id = 10000; -- single row expected (bar), because the force_select_final is in action and FINAL is there. 10000 bar -SELECT * FROM tbl WHERE id = 10000 SETTINGS ignore_force_select_final=1; -- now we see 2 'real' rows +SELECT * FROM tbl WHERE id = 10000 ORDER BY val DESC SETTINGS ignore_force_select_final=1; -- now we see 2 'real' rows 10000 foo 10000 bar SELECT * FROM tbl FINAL WHERE id = 10000 SETTINGS ignore_force_select_final=1; -- now we see single row again. diff --git a/tests/queries/0_stateless/02420_mt_ignore_force_select_final_setting.sql b/tests/queries/0_stateless/02420_mt_ignore_force_select_final_setting.sql index fdcdea2bc23..87ff5c789bd 100644 --- a/tests/queries/0_stateless/02420_mt_ignore_force_select_final_setting.sql +++ b/tests/queries/0_stateless/02420_mt_ignore_force_select_final_setting.sql @@ -9,7 +9,7 @@ INSERT INTO tbl SELECT number as id, 'bar' AS val FROM numbers(100000); SELECT * FROM tbl WHERE id = 10000; -- single row expected (bar), because the force_select_final is in action SELECT * FROM tbl final WHERE id = 10000; -- single row expected (bar), because the force_select_final is in action and FINAL is there. -SELECT * FROM tbl WHERE id = 10000 SETTINGS ignore_force_select_final=1; -- now we see 2 'real' rows +SELECT * FROM tbl WHERE id = 10000 ORDER BY val DESC SETTINGS ignore_force_select_final=1; -- now we see 2 'real' rows SELECT * FROM tbl FINAL WHERE id = 10000 SETTINGS ignore_force_select_final=1; -- now we see single row again. SYSTEM START MERGES tbl; From 27d63405f28867b4f87300c0e34422d1037c5fec Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 17 Nov 2022 10:21:46 -0300 Subject: [PATCH 020/717] Silently fail when force_select_final setting is used with MT engines that do not support it --- src/Storages/MergeTree/registerStorageMergeTree.cpp | 6 ------ .../0_stateless/02420_force_select_final_setting.sql | 4 ---- 2 files changed, 10 deletions(-) diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index f375a53b42a..659fe5246f1 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -682,12 +682,6 @@ static StoragePtr create(const StorageFactory::Arguments & args) if (arg_num != arg_cnt) throw Exception("Wrong number of engine arguments.", ErrorCodes::BAD_ARGUMENTS); - - if (merging_params.mode == MergeTreeData::MergingParams::Mode::Ordinary && storage_settings->force_select_final) - { - throw Exception("Storage MergeTree doesn't support FINAL", ErrorCodes::ILLEGAL_FINAL); - } - if (replicated) { return std::make_shared( diff --git a/tests/queries/0_stateless/02420_force_select_final_setting.sql b/tests/queries/0_stateless/02420_force_select_final_setting.sql index fae3a74e465..8dcaa6bf934 100644 --- a/tests/queries/0_stateless/02420_force_select_final_setting.sql +++ b/tests/queries/0_stateless/02420_force_select_final_setting.sql @@ -1,5 +1,4 @@ -- { echoOn } -create table if not exists ordinary_mt (x String) engine=MergeTree() ORDER BY x SETTINGS force_select_final=1; -- { serverError 181 } -- simple test case create table if not exists replacing_mt (x String) engine=ReplacingMergeTree() ORDER BY x SETTINGS force_select_final=1; @@ -37,6 +36,3 @@ insert into rhs values ('abc'); -- expected output is 2 because lhs table doesn't have final applied select count() from lhs inner join rhs on lhs.x = rhs.x; - --- Engine that does not support final -create table if not exists ordinary_mt (x String) engine=MergeTree() ORDER BY x SETTINGS force_select_final=1; -- { serverError 181 } From 5b84a4fb090990d54195a96f47826dcbb69ca312 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 17 Nov 2022 10:32:50 -0300 Subject: [PATCH 021/717] remove illegal final definition --- src/Storages/MergeTree/registerStorageMergeTree.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index 659fe5246f1..6982521f76a 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -34,7 +34,6 @@ namespace ErrorCodes extern const int UNKNOWN_STORAGE; extern const int NO_REPLICA_NAME_GIVEN; extern const int CANNOT_EXTRACT_TABLE_STRUCTURE; - extern const int ILLEGAL_FINAL; } From e33b1a39da83330f43c1b0aab4866ddbb993ba4c Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 17 Nov 2022 13:32:22 -0300 Subject: [PATCH 022/717] fix tst --- .../0_stateless/02420_force_select_final_setting.reference | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/queries/0_stateless/02420_force_select_final_setting.reference b/tests/queries/0_stateless/02420_force_select_final_setting.reference index 466be527e6f..f9044ba05aa 100644 --- a/tests/queries/0_stateless/02420_force_select_final_setting.reference +++ b/tests/queries/0_stateless/02420_force_select_final_setting.reference @@ -1,5 +1,4 @@ -- { echoOn } -create table if not exists ordinary_mt (x String) engine=MergeTree() ORDER BY x SETTINGS force_select_final=1; -- { serverError 181 } -- simple test case create table if not exists replacing_mt (x String) engine=ReplacingMergeTree() ORDER BY x SETTINGS force_select_final=1; insert into replacing_mt values ('abc'); @@ -28,5 +27,3 @@ insert into rhs values ('abc'); -- expected output is 2 because lhs table doesn't have final applied select count() from lhs inner join rhs on lhs.x = rhs.x; 2 --- Engine that does not support final -create table if not exists ordinary_mt (x String) engine=MergeTree() ORDER BY x SETTINGS force_select_final=1; -- { serverError 181 } From e9b19ffa847e8f19c0ad4b63ed127c9847221f05 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 17 Nov 2022 14:05:21 -0300 Subject: [PATCH 023/717] Update reference file --- tests/queries/0_stateless/02420_force_select_final_setting.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/queries/0_stateless/02420_force_select_final_setting.sql b/tests/queries/0_stateless/02420_force_select_final_setting.sql index 8dcaa6bf934..de91db071e1 100644 --- a/tests/queries/0_stateless/02420_force_select_final_setting.sql +++ b/tests/queries/0_stateless/02420_force_select_final_setting.sql @@ -1,5 +1,4 @@ -- { echoOn } - -- simple test case create table if not exists replacing_mt (x String) engine=ReplacingMergeTree() ORDER BY x SETTINGS force_select_final=1; From 6ee21e63d7dc1e84cfa7e40b25545937f356d73f Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Fri, 25 Nov 2022 13:31:08 -0300 Subject: [PATCH 024/717] Move force select final to user lvel setting and remove ignore force select final --- src/Core/Settings.h | 2 +- src/Interpreters/InterpreterSelectQuery.cpp | 2 +- src/Storages/IStorage.h | 2 -- src/Storages/MergeTree/MergeTreeData.h | 4 --- src/Storages/MergeTree/MergeTreeSettings.h | 2 -- ...02420_force_select_final_setting.reference | 30 ++++++++--------- .../02420_force_select_final_setting.sql | 32 ++++++++----------- ...gnore_force_select_final_setting.reference | 16 ---------- ...0_mt_ignore_force_select_final_setting.sql | 19 ----------- 9 files changed, 31 insertions(+), 78 deletions(-) delete mode 100644 tests/queries/0_stateless/02420_mt_ignore_force_select_final_setting.reference delete mode 100644 tests/queries/0_stateless/02420_mt_ignore_force_select_final_setting.sql diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 591557a6031..8e5b9d5c690 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -274,7 +274,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(Milliseconds, stream_flush_interval_ms, 7500, "Timeout for flushing data from streaming storages.", 0) \ M(Milliseconds, stream_poll_timeout_ms, 500, "Timeout for polling data from/to streaming storages.", 0) \ \ - M(Bool, ignore_force_select_final, false, "Ignores the mergetree setting force_select_final, allowing to read the raw data (without applying FINAL automatically) is force_select_final is enabled", 0) \ + M(Bool, force_select_final, false, "Query with the FINAL modifier by default", 0) \ \ /** Settings for testing hedged requests */ \ M(Milliseconds, sleep_in_send_tables_status_ms, 0, "Time to sleep in sending tables status response in TCPHandler", 0) \ diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 766c595b2f1..d4183da2f11 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -500,7 +500,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( settings.additional_table_filters, joined_tables.tablesWithColumns().front().table, *context); - if (!query.final() && storage && storage->forceQueryWithFinal() && !context->getSettingsRef().ignore_force_select_final) + if (!query.final() && context->getSettingsRef().force_select_final) { query.setFinal(); } diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index 186addbbd71..fd48d22b12b 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -329,8 +329,6 @@ public: /// It's needed for ReplacingMergeTree wrappers such as MaterializedMySQL and MaterializedPostrgeSQL virtual bool needRewriteQueryWithFinal(const Names & /*column_names*/) const { return false; } - virtual bool forceQueryWithFinal() const { return false; } - private: /** Read a set of columns from the table. * Accepts a list of columns to read, as well as a description of the query, diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index b572423859e..8bd0fc1f280 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -437,10 +437,6 @@ public: bool mayBenefitFromIndexForIn(const ASTPtr & left_in_operand, ContextPtr, const StorageMetadataPtr & metadata_snapshot) const override; - bool forceQueryWithFinal() const override - { - return getSettings()->force_select_final && supportsFinal(); - } /// Snapshot for MergeTree contains the current set of data parts /// at the moment of the start of query. struct SnapshotData : public StorageSnapshot::Data diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index 3099adaa48b..998e870484c 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -147,8 +147,6 @@ struct Settings; M(Bool, allow_remote_fs_zero_copy_replication, false, "Don't use this setting in production, because it is not ready.", 0) \ M(String, remote_fs_zero_copy_zookeeper_path, "/clickhouse/zero_copy", "ZooKeeper path for Zero-copy table-independet info.", 0) \ M(Bool, remote_fs_zero_copy_path_compatible_mode, false, "Run zero-copy in compatible mode during conversion process.", 0) \ - M(Bool, force_select_final, false, "Query with the FINAL modifier by default", 0) \ - \ /** Compress marks and primary key. */ \ M(Bool, compress_marks, false, "Marks support compression, reduce mark file size and speed up network transmission.", 0) \ M(Bool, compress_primary_key, false, "Primary key support compression, reduce primary key file size and speed up network transmission.", 0) \ diff --git a/tests/queries/0_stateless/02420_force_select_final_setting.reference b/tests/queries/0_stateless/02420_force_select_final_setting.reference index f9044ba05aa..ab9cf9225ef 100644 --- a/tests/queries/0_stateless/02420_force_select_final_setting.reference +++ b/tests/queries/0_stateless/02420_force_select_final_setting.reference @@ -1,29 +1,29 @@ -- { echoOn } + +SYSTEM STOP MERGES tbl; -- simple test case -create table if not exists replacing_mt (x String) engine=ReplacingMergeTree() ORDER BY x SETTINGS force_select_final=1; +create table if not exists replacing_mt (x String) engine=ReplacingMergeTree() ORDER BY x; insert into replacing_mt values ('abc'); insert into replacing_mt values ('abc'); +-- expected output is 2 because force_select_final is turned off +select count() from replacing_mt; +2 +set force_select_final = 1; -- expected output is 1 because force_select_final is turned on select count() from replacing_mt; 1 -- JOIN test cases -create table if not exists lhs (x String) engine=ReplacingMergeTree() ORDER BY x SETTINGS force_select_final=1; -create table if not exists rhs (x String) engine=ReplacingMergeTree() ORDER BY x SETTINGS force_select_final=1; +create table if not exists lhs (x String) engine=ReplacingMergeTree() ORDER BY x; +create table if not exists rhs (x String) engine=ReplacingMergeTree() ORDER BY x; insert into lhs values ('abc'); insert into lhs values ('abc'); insert into rhs values ('abc'); insert into rhs values ('abc'); --- expected output is 1 because both tables have force_select_final = 1 +set force_select_final = 0; +-- expected output is 4 because select_final == 0 +select count() from lhs inner join rhs on lhs.x = rhs.x; +4 +set force_select_final = 1; +-- expected output is 1 because force_select_final == 1 select count() from lhs inner join rhs on lhs.x = rhs.x; 1 -drop table lhs; -drop table rhs; -create table if not exists lhs (x String) engine=ReplacingMergeTree() ORDER BY x; -create table if not exists rhs (x String) engine=ReplacingMergeTree() ORDER BY x SETTINGS force_select_final=1; -insert into lhs values ('abc'); -insert into lhs values ('abc'); -insert into rhs values ('abc'); -insert into rhs values ('abc'); --- expected output is 2 because lhs table doesn't have final applied -select count() from lhs inner join rhs on lhs.x = rhs.x; -2 diff --git a/tests/queries/0_stateless/02420_force_select_final_setting.sql b/tests/queries/0_stateless/02420_force_select_final_setting.sql index de91db071e1..ae925a5dd14 100644 --- a/tests/queries/0_stateless/02420_force_select_final_setting.sql +++ b/tests/queries/0_stateless/02420_force_select_final_setting.sql @@ -1,31 +1,22 @@ -- { echoOn } +SYSTEM STOP MERGES tbl; + -- simple test case -create table if not exists replacing_mt (x String) engine=ReplacingMergeTree() ORDER BY x SETTINGS force_select_final=1; +create table if not exists replacing_mt (x String) engine=ReplacingMergeTree() ORDER BY x; insert into replacing_mt values ('abc'); insert into replacing_mt values ('abc'); +-- expected output is 2 because force_select_final is turned off +select count() from replacing_mt; + +set force_select_final = 1; -- expected output is 1 because force_select_final is turned on select count() from replacing_mt; -- JOIN test cases -create table if not exists lhs (x String) engine=ReplacingMergeTree() ORDER BY x SETTINGS force_select_final=1; -create table if not exists rhs (x String) engine=ReplacingMergeTree() ORDER BY x SETTINGS force_select_final=1; - -insert into lhs values ('abc'); -insert into lhs values ('abc'); - -insert into rhs values ('abc'); -insert into rhs values ('abc'); - --- expected output is 1 because both tables have force_select_final = 1 -select count() from lhs inner join rhs on lhs.x = rhs.x; - -drop table lhs; -drop table rhs; - create table if not exists lhs (x String) engine=ReplacingMergeTree() ORDER BY x; -create table if not exists rhs (x String) engine=ReplacingMergeTree() ORDER BY x SETTINGS force_select_final=1; +create table if not exists rhs (x String) engine=ReplacingMergeTree() ORDER BY x; insert into lhs values ('abc'); insert into lhs values ('abc'); @@ -33,5 +24,10 @@ insert into lhs values ('abc'); insert into rhs values ('abc'); insert into rhs values ('abc'); --- expected output is 2 because lhs table doesn't have final applied +set force_select_final = 0; +-- expected output is 4 because select_final == 0 +select count() from lhs inner join rhs on lhs.x = rhs.x; + +set force_select_final = 1; +-- expected output is 1 because force_select_final == 1 select count() from lhs inner join rhs on lhs.x = rhs.x; diff --git a/tests/queries/0_stateless/02420_mt_ignore_force_select_final_setting.reference b/tests/queries/0_stateless/02420_mt_ignore_force_select_final_setting.reference deleted file mode 100644 index 1ae1def4cb1..00000000000 --- a/tests/queries/0_stateless/02420_mt_ignore_force_select_final_setting.reference +++ /dev/null @@ -1,16 +0,0 @@ --- { echoOn } - -SELECT * FROM tbl WHERE id = 10000; -- single row expected (bar), because the force_select_final is in action -10000 bar -SELECT * FROM tbl final WHERE id = 10000; -- single row expected (bar), because the force_select_final is in action and FINAL is there. -10000 bar -SELECT * FROM tbl WHERE id = 10000 ORDER BY val DESC SETTINGS ignore_force_select_final=1; -- now we see 2 'real' rows -10000 foo -10000 bar -SELECT * FROM tbl FINAL WHERE id = 10000 SETTINGS ignore_force_select_final=1; -- now we see single row again. -10000 bar -SYSTEM START MERGES tbl; -OPTIMIZE TABLE tbl FINAL; -SELECT * FROM tbl WHERE id = 10000 SETTINGS ignore_force_select_final=1; -- now we see single row DROP TABLE tbl; -10000 bar -DROP TABLE tbl; diff --git a/tests/queries/0_stateless/02420_mt_ignore_force_select_final_setting.sql b/tests/queries/0_stateless/02420_mt_ignore_force_select_final_setting.sql deleted file mode 100644 index 87ff5c789bd..00000000000 --- a/tests/queries/0_stateless/02420_mt_ignore_force_select_final_setting.sql +++ /dev/null @@ -1,19 +0,0 @@ -CREATE TABLE tbl (id UInt64, val String) engine=ReplacingMergeTree() ORDER BY id SETTINGS force_select_final=1; - -SYSTEM STOP MERGES tbl; -INSERT INTO tbl SELECT number as id, 'foo' AS val FROM numbers(100000); -INSERT INTO tbl SELECT number as id, 'bar' AS val FROM numbers(100000); - --- { echoOn } - -SELECT * FROM tbl WHERE id = 10000; -- single row expected (bar), because the force_select_final is in action -SELECT * FROM tbl final WHERE id = 10000; -- single row expected (bar), because the force_select_final is in action and FINAL is there. - -SELECT * FROM tbl WHERE id = 10000 ORDER BY val DESC SETTINGS ignore_force_select_final=1; -- now we see 2 'real' rows -SELECT * FROM tbl FINAL WHERE id = 10000 SETTINGS ignore_force_select_final=1; -- now we see single row again. - -SYSTEM START MERGES tbl; -OPTIMIZE TABLE tbl FINAL; - -SELECT * FROM tbl WHERE id = 10000 SETTINGS ignore_force_select_final=1; -- now we see single row DROP TABLE tbl; -DROP TABLE tbl; \ No newline at end of file From 21449847361438f018e3e22bae0bc923e432592d Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Fri, 25 Nov 2022 15:01:38 -0300 Subject: [PATCH 025/717] fix tests --- .../0_stateless/02420_force_select_final_setting.reference | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/queries/0_stateless/02420_force_select_final_setting.reference b/tests/queries/0_stateless/02420_force_select_final_setting.reference index ab9cf9225ef..36b8905289b 100644 --- a/tests/queries/0_stateless/02420_force_select_final_setting.reference +++ b/tests/queries/0_stateless/02420_force_select_final_setting.reference @@ -1,5 +1,4 @@ -- { echoOn } - SYSTEM STOP MERGES tbl; -- simple test case create table if not exists replacing_mt (x String) engine=ReplacingMergeTree() ORDER BY x; From eb3f4c9fe25792924b8bfeb8356728d0d2c6f09f Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 29 Nov 2022 13:14:06 -0300 Subject: [PATCH 026/717] prevent force select final from affecting non table based queries --- src/Interpreters/InterpreterSelectQuery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index d4183da2f11..1d5eeb706f6 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -500,7 +500,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( settings.additional_table_filters, joined_tables.tablesWithColumns().front().table, *context); - if (!query.final() && context->getSettingsRef().force_select_final) + if (context->getSettingsRef().force_select_final && !query.final() && query.tables()) { query.setFinal(); } From 252575ee23d8151b3a7c5435c82616cd6ff7fbe9 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 29 Nov 2022 13:14:52 -0300 Subject: [PATCH 027/717] doc --- src/Interpreters/InterpreterSelectQuery.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 1d5eeb706f6..da08ee8d389 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -500,6 +500,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( settings.additional_table_filters, joined_tables.tablesWithColumns().front().table, *context); + // query.tables() is required because not all queries have tables in it, it could be a function. if (context->getSettingsRef().force_select_final && !query.final() && query.tables()) { query.setFinal(); From 32395165219fbe37ceffde0eed4482368273d755 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Mon, 5 Dec 2022 10:18:44 -0300 Subject: [PATCH 028/717] apply final only on storages that support it --- src/Interpreters/InterpreterSelectQuery.cpp | 13 ++++++++++--- src/Interpreters/InterpreterSelectQuery.h | 1 + .../02420_force_select_final_setting.reference | 7 +++++++ .../02420_force_select_final_setting.sql | 7 +++++++ 4 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index da08ee8d389..6e7259a98e3 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -499,9 +499,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( query_info.additional_filter_ast = parseAdditionalFilterConditionForTable( settings.additional_table_filters, joined_tables.tablesWithColumns().front().table, *context); - - // query.tables() is required because not all queries have tables in it, it could be a function. - if (context->getSettingsRef().force_select_final && !query.final() && query.tables()) + if (forceSelectFinalOnSelectQuery(query)) { query.setFinal(); } @@ -2916,6 +2914,15 @@ void InterpreterSelectQuery::ignoreWithTotals() getSelectQuery().group_by_with_totals = false; } +bool InterpreterSelectQuery::forceSelectFinalOnSelectQuery(ASTSelectQuery & query) +{ + // query.tables() is required because not all queries have tables in it, it could be a function. + auto isFinalSupported = storage && storage->supportsFinal() && query.tables(); + auto isForceSelectFinalSettingOn = context->getSettingsRef().force_select_final; + auto isQueryAlreadyFinal = query.final(); + + return isForceSelectFinalSettingOn && !isQueryAlreadyFinal && isFinalSupported; +} void InterpreterSelectQuery::initSettings() { diff --git a/src/Interpreters/InterpreterSelectQuery.h b/src/Interpreters/InterpreterSelectQuery.h index 761eea8e1b8..f679051532c 100644 --- a/src/Interpreters/InterpreterSelectQuery.h +++ b/src/Interpreters/InterpreterSelectQuery.h @@ -194,6 +194,7 @@ private: void executeDistinct(QueryPlan & query_plan, bool before_order, Names columns, bool pre_distinct); void executeExtremes(QueryPlan & query_plan); void executeSubqueriesInSetsAndJoins(QueryPlan & query_plan); + bool forceSelectFinalOnSelectQuery(ASTSelectQuery & select_query); enum class Modificator { diff --git a/tests/queries/0_stateless/02420_force_select_final_setting.reference b/tests/queries/0_stateless/02420_force_select_final_setting.reference index 36b8905289b..ad1a0bc1567 100644 --- a/tests/queries/0_stateless/02420_force_select_final_setting.reference +++ b/tests/queries/0_stateless/02420_force_select_final_setting.reference @@ -26,3 +26,10 @@ set force_select_final = 1; -- expected output is 1 because force_select_final == 1 select count() from lhs inner join rhs on lhs.x = rhs.x; 1 +-- regular non final table +set force_select_final = 1; +create table if not exists regular_mt_table (x String) engine=MergeTree() ORDER BY x; +insert into regular_mt_table values ('abc'); +-- expected output is 1, it should silently ignore final modifier +select count() from regular_mt_table; +1 diff --git a/tests/queries/0_stateless/02420_force_select_final_setting.sql b/tests/queries/0_stateless/02420_force_select_final_setting.sql index ae925a5dd14..7dd7eb05b12 100644 --- a/tests/queries/0_stateless/02420_force_select_final_setting.sql +++ b/tests/queries/0_stateless/02420_force_select_final_setting.sql @@ -31,3 +31,10 @@ select count() from lhs inner join rhs on lhs.x = rhs.x; set force_select_final = 1; -- expected output is 1 because force_select_final == 1 select count() from lhs inner join rhs on lhs.x = rhs.x; + +-- regular non final table +set force_select_final = 1; +create table if not exists regular_mt_table (x String) engine=MergeTree() ORDER BY x; +insert into regular_mt_table values ('abc'); +-- expected output is 1, it should silently ignore final modifier +select count() from regular_mt_table; From 1cf8dc6daaf1876064ca2754a621b96ee02ef8b7 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Mon, 5 Dec 2022 13:21:50 -0300 Subject: [PATCH 029/717] add view test --- .../02420_force_select_final_setting.reference | 9 ++++++++- .../0_stateless/02420_force_select_final_setting.sql | 8 ++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02420_force_select_final_setting.reference b/tests/queries/0_stateless/02420_force_select_final_setting.reference index ad1a0bc1567..8be3da3545c 100644 --- a/tests/queries/0_stateless/02420_force_select_final_setting.reference +++ b/tests/queries/0_stateless/02420_force_select_final_setting.reference @@ -30,6 +30,13 @@ select count() from lhs inner join rhs on lhs.x = rhs.x; set force_select_final = 1; create table if not exists regular_mt_table (x String) engine=MergeTree() ORDER BY x; insert into regular_mt_table values ('abc'); +insert into regular_mt_table values ('abc'); -- expected output is 1, it should silently ignore final modifier select count() from regular_mt_table; -1 +2 +-- view test +create materialized VIEW mv_regular_mt_table TO regular_mt_table AS SELECT * FROM regular_mt_table; +create view nv_regular_mt_table AS SELECT * FROM mv_regular_mt_table; +set force_select_final=1; +select count() from nv_regular_mt_table; +2 diff --git a/tests/queries/0_stateless/02420_force_select_final_setting.sql b/tests/queries/0_stateless/02420_force_select_final_setting.sql index 7dd7eb05b12..84d4c40445a 100644 --- a/tests/queries/0_stateless/02420_force_select_final_setting.sql +++ b/tests/queries/0_stateless/02420_force_select_final_setting.sql @@ -36,5 +36,13 @@ select count() from lhs inner join rhs on lhs.x = rhs.x; set force_select_final = 1; create table if not exists regular_mt_table (x String) engine=MergeTree() ORDER BY x; insert into regular_mt_table values ('abc'); +insert into regular_mt_table values ('abc'); -- expected output is 1, it should silently ignore final modifier select count() from regular_mt_table; + +-- view test +create materialized VIEW mv_regular_mt_table TO regular_mt_table AS SELECT * FROM regular_mt_table; +create view nv_regular_mt_table AS SELECT * FROM mv_regular_mt_table; + +set force_select_final=1; +select count() from nv_regular_mt_table; From 7f8a4f8d8779bb1192e53b7a7605885e7bac28e5 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 6 Dec 2022 17:28:30 -0300 Subject: [PATCH 030/717] do not apply final on distributed engine, only on underlying --- src/Interpreters/InterpreterSelectQuery.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 6e7259a98e3..9e5b462cef4 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -2917,11 +2917,11 @@ void InterpreterSelectQuery::ignoreWithTotals() bool InterpreterSelectQuery::forceSelectFinalOnSelectQuery(ASTSelectQuery & query) { // query.tables() is required because not all queries have tables in it, it could be a function. - auto isFinalSupported = storage && storage->supportsFinal() && query.tables(); - auto isForceSelectFinalSettingOn = context->getSettingsRef().force_select_final; - auto isQueryAlreadyFinal = query.final(); + auto is_force_select_final_setting_on = context->getSettingsRef().force_select_final; + auto is_final_supported = storage && storage->supportsFinal() && !storage->isRemote() && query.tables(); + auto is_query_already_final = query.final(); - return isForceSelectFinalSettingOn && !isQueryAlreadyFinal && isFinalSupported; + return is_force_select_final_setting_on && !is_query_already_final && is_final_supported; } void InterpreterSelectQuery::initSettings() From db6e1acc14317c8765e337142eac6955345da53f Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Mon, 12 Dec 2022 22:04:53 +0000 Subject: [PATCH 031/717] Remove redundant DISTINCT - it doesn't consider prefixes yet --- src/Core/Settings.h | 1 + .../QueryPlan/Optimizations/Optimizations.h | 6 +- .../QueryPlanOptimizationSettings.cpp | 1 + .../QueryPlanOptimizationSettings.h | 3 + .../Optimizations/removeRedundantDistinct.cpp | 62 +++++++++++++++++++ .../02500_remove_redundant_distinct.reference | 24 +++++++ .../02500_remove_redundant_distinct.sql | 29 +++++++++ 7 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp create mode 100644 tests/queries/0_stateless/02500_remove_redundant_distinct.reference create mode 100644 tests/queries/0_stateless/02500_remove_redundant_distinct.sql diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 5c57d2082f5..a820cccb18a 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -584,6 +584,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(Bool, query_plan_optimize_primary_key, true, "Analyze primary key using query plan (instead of AST)", 0) \ M(Bool, query_plan_read_in_order, true, "Use query plan for read-in-order optimisation", 0) \ M(Bool, query_plan_aggregation_in_order, true, "Use query plan for aggregation-in-order optimisation", 0) \ + M(Bool, query_plan_remove_redundant_distinct, true, "Remove redundant DISTINCT clauses", 0) \ M(UInt64, regexp_max_matches_per_row, 1000, "Max matches of any single regexp per row, used to safeguard 'extractAllGroupsHorizontal' against consuming too much memory with greedy RE.", 0) \ \ M(UInt64, limit, 0, "Limit on read rows from the most 'end' result for select query, default 0 means no limit length", 0) \ diff --git a/src/Processors/QueryPlan/Optimizations/Optimizations.h b/src/Processors/QueryPlan/Optimizations/Optimizations.h index 7f435463d64..0cad734e557 100644 --- a/src/Processors/QueryPlan/Optimizations/Optimizations.h +++ b/src/Processors/QueryPlan/Optimizations/Optimizations.h @@ -58,6 +58,9 @@ size_t tryReuseStorageOrderingForWindowFunctions(QueryPlan::Node * parent_node, /// Reading in order from MergeTree table if DISTINCT columns match or form a prefix of MergeTree sorting key size_t tryDistinctReadInOrder(QueryPlan::Node * node); +/// Remove redundant DISTINCT clauses +size_t tryRemoveRedundantDistinct(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes); + /// Put some steps under union, so that plan optimisation could be applied to union parts separately. /// For example, the plan can be rewritten like: /// - Something - - Expression - Something - @@ -67,7 +70,7 @@ size_t tryLiftUpUnion(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes); inline const auto & getOptimizations() { - static const std::array optimizations = {{ + static const std::array optimizations = {{ {tryLiftUpArrayJoin, "liftUpArrayJoin", &QueryPlanOptimizationSettings::optimize_plan}, {tryPushDownLimit, "pushDownLimit", &QueryPlanOptimizationSettings::optimize_plan}, {trySplitFilter, "splitFilter", &QueryPlanOptimizationSettings::optimize_plan}, @@ -76,6 +79,7 @@ inline const auto & getOptimizations() {tryExecuteFunctionsAfterSorting, "liftUpFunctions", &QueryPlanOptimizationSettings::optimize_plan}, {tryReuseStorageOrderingForWindowFunctions, "reuseStorageOrderingForWindowFunctions", &QueryPlanOptimizationSettings::optimize_plan}, {tryLiftUpUnion, "liftUpUnion", &QueryPlanOptimizationSettings::optimize_plan}, + {tryRemoveRedundantDistinct, "removeRedundantDistinct", &QueryPlanOptimizationSettings::remove_redundant_distinct}, }}; return optimizations; diff --git a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp index 00abd803d2a..92dca42c347 100644 --- a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp +++ b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp @@ -14,6 +14,7 @@ QueryPlanOptimizationSettings QueryPlanOptimizationSettings::fromSettings(const settings.distinct_in_order = from.optimize_distinct_in_order; settings.read_in_order = from.optimize_read_in_order && from.query_plan_read_in_order; settings.aggregation_in_order = from.optimize_aggregation_in_order && from.query_plan_aggregation_in_order; + settings.remove_redundant_distinct = from.query_plan_remove_redundant_distinct; return settings; } diff --git a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h index d4989b86b68..5130c87228b 100644 --- a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h +++ b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h @@ -30,6 +30,9 @@ struct QueryPlanOptimizationSettings /// If aggregation-in-order optimisation is enabled bool aggregation_in_order = false; + /// true if removing redundant DISTINCT clauses is enabled + bool remove_redundant_distinct = true; + static QueryPlanOptimizationSettings fromSettings(const Settings & from); static QueryPlanOptimizationSettings fromContext(ContextPtr from); }; diff --git a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp new file mode 100644 index 00000000000..79d08c64543 --- /dev/null +++ b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp @@ -0,0 +1,62 @@ +#include +#include +#include +#include +#include +#include +#include "Processors/QueryPlan/ExpressionStep.h" + +namespace DB::QueryPlanOptimizations +{ + +static std::set getDistinctColumns(const DistinctStep * distinct) +{ + /// find non-const columns in DISTINCT + const ColumnsWithTypeAndName & distinct_columns = distinct->getOutputStream().header.getColumnsWithTypeAndName(); + std::set non_const_columns; + for (const auto & column : distinct_columns) + { + if (!isColumnConst(*column.column)) + non_const_columns.emplace(column.name); + } + return non_const_columns; +} + +size_t tryRemoveRedundantDistinct(QueryPlan::Node * parent_node, QueryPlan::Nodes & /* nodes*/) +{ + if (parent_node->children.empty()) + return 0; + + /// check if it is preliminary distinct node + QueryPlan::Node * distinct_node = nullptr; + DistinctStep * distinct_step = typeid_cast(parent_node->children.front()->step.get()); + if (!distinct_step || !distinct_step->isPreliminary()) + return 0; + + distinct_node = parent_node->children.front(); + + DistinctStep * inner_distinct_step = nullptr; + QueryPlan::Node * node = distinct_node; + while (!node->children.empty()) + { + node = node->children.front(); + inner_distinct_step = typeid_cast(node->step.get()); + if (inner_distinct_step) + break; + } + if (!inner_distinct_step) + return 0; + + if (getDistinctColumns(inner_distinct_step) != getDistinctColumns(inner_distinct_step)) + return 0; + + chassert(!distinct_node->children.empty()); + chassert(typeid_cast(distinct_node->children.front()->step.get())); + + /// delete current distinct + parent_node->children[0] = distinct_node->children.front(); + + return 1; +} + +} diff --git a/tests/queries/0_stateless/02500_remove_redundant_distinct.reference b/tests/queries/0_stateless/02500_remove_redundant_distinct.reference new file mode 100644 index 00000000000..71cd9d2e51d --- /dev/null +++ b/tests/queries/0_stateless/02500_remove_redundant_distinct.reference @@ -0,0 +1,24 @@ +Expression (Project names) +Header: number UInt64 + Distinct (Preliminary DISTINCT) + Header: number_0 UInt64 + Expression ((Projection + (Change column names to column identifiers + Project names))) + Header: number_0 UInt64 + Distinct (Preliminary DISTINCT) + Header: number_1 UInt64 + Expression ((Projection + (Change column names to column identifiers + Project names))) + Header: number_1 UInt64 + Distinct (Preliminary DISTINCT) + Header: number_2 UInt64 + Expression ((Projection + Change column names to column identifiers)) + Header: number_2 UInt64 + ReadFromStorage (SystemNumbers) + Header: number UInt64 +Expression ((Project names + (Projection + (Change column names to column identifiers + (Project names + (Projection + (Change column names to column identifiers + Project names))))))) +Header: number UInt64 + Distinct (Preliminary DISTINCT) + Header: number_2 UInt64 + Expression ((Projection + Change column names to column identifiers)) + Header: number_2 UInt64 + ReadFromStorage (SystemNumbers) + Header: number UInt64 diff --git a/tests/queries/0_stateless/02500_remove_redundant_distinct.sql b/tests/queries/0_stateless/02500_remove_redundant_distinct.sql new file mode 100644 index 00000000000..1e9f8bca29e --- /dev/null +++ b/tests/queries/0_stateless/02500_remove_redundant_distinct.sql @@ -0,0 +1,29 @@ +set allow_experimental_analyzer=1; +set optimize_duplicate_order_by_and_distinct = 0; + +set query_plan_remove_redundant_distinct=0; + +EXPLAIN header=1 +SELECT DISTINCT * +FROM +( + SELECT DISTINCT * + FROM + ( + SELECT DISTINCT * + FROM numbers(3) + ) +); + +set query_plan_remove_redundant_distinct=1; +EXPLAIN header=1 +SELECT DISTINCT * +FROM +( + SELECT DISTINCT * + FROM + ( + SELECT DISTINCT * + FROM numbers(3) + ) +); From 03f7bd119240be8bd50c500b116bee4106cd992d Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Wed, 14 Dec 2022 13:54:42 +0000 Subject: [PATCH 032/717] Fix for distributed queries --- .../QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp | 2 +- .../QueryPlan/Optimizations/removeRedundantDistinct.cpp | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp index 92dca42c347..37a3fcf584a 100644 --- a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp +++ b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp @@ -14,7 +14,7 @@ QueryPlanOptimizationSettings QueryPlanOptimizationSettings::fromSettings(const settings.distinct_in_order = from.optimize_distinct_in_order; settings.read_in_order = from.optimize_read_in_order && from.query_plan_read_in_order; settings.aggregation_in_order = from.optimize_aggregation_in_order && from.query_plan_aggregation_in_order; - settings.remove_redundant_distinct = from.query_plan_remove_redundant_distinct; + settings.remove_redundant_distinct = from.query_plan_remove_redundant_distinct && !from.distributed_group_by_no_merge; return settings; } diff --git a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp index 79d08c64543..5976d8238e0 100644 --- a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp +++ b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp @@ -51,7 +51,6 @@ size_t tryRemoveRedundantDistinct(QueryPlan::Node * parent_node, QueryPlan::Node return 0; chassert(!distinct_node->children.empty()); - chassert(typeid_cast(distinct_node->children.front()->step.get())); /// delete current distinct parent_node->children[0] = distinct_node->children.front(); From 11fb78191c873ce5493f4819c84fa2431c94a3d1 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 6 Jan 2023 10:18:00 +0000 Subject: [PATCH 033/717] (draft) Remove SMP deactivation on big machines This might be a bit controversial but 1. AMD offers 48/67/64 (true) core systems already (Milan, (*)), up to 96 cores/CPU soon (Genoa), 2. Intel sells a few Xeon CPUs with > cores (**), and we shouldn't artificially limit core utilization on such sytems. (*) https://en.wikipedia.org/wiki/Epyc (**) https://www.intel.com/content/www/us/en/products/details/processors/xeon/scalable/platinum/products.html --- src/Common/getNumberOfPhysicalCPUCores.cpp | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/Common/getNumberOfPhysicalCPUCores.cpp b/src/Common/getNumberOfPhysicalCPUCores.cpp index 7a1f10b6435..3dbe0461069 100644 --- a/src/Common/getNumberOfPhysicalCPUCores.cpp +++ b/src/Common/getNumberOfPhysicalCPUCores.cpp @@ -40,18 +40,6 @@ static unsigned getNumberOfPhysicalCPUCoresImpl() { unsigned cpu_count = std::thread::hardware_concurrency(); - /// Most of x86_64 CPUs have 2-way Hyper-Threading - /// Aarch64 and RISC-V don't have SMT so far. - /// POWER has SMT and it can be multiple way (like 8-way), but we don't know how ClickHouse really behaves, so use all of them. - -#if defined(__x86_64__) - /// Let's limit ourself to the number of physical cores. - /// But if the number of logical cores is small - maybe it is a small machine - /// or very limited cloud instance and it is reasonable to use all the cores. - if (cpu_count >= 32) - cpu_count /= 2; -#endif - #if defined(OS_LINUX) cpu_count = getCGroupLimitedCPUCores(cpu_count); #endif From a733303184f3e02e2a3951271cbfc656034153a7 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 6 Jan 2023 19:55:55 +0000 Subject: [PATCH 034/717] Disable HyperThreading on big machines --- src/Common/getNumberOfPhysicalCPUCores.cpp | 67 +++++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/src/Common/getNumberOfPhysicalCPUCores.cpp b/src/Common/getNumberOfPhysicalCPUCores.cpp index 3dbe0461069..86fff6732e0 100644 --- a/src/Common/getNumberOfPhysicalCPUCores.cpp +++ b/src/Common/getNumberOfPhysicalCPUCores.cpp @@ -6,7 +6,10 @@ # include #endif +#include + #include +#include #if defined(OS_LINUX) static int32_t readFrom(const char * filename, int default_value) @@ -34,11 +37,73 @@ static uint32_t getCGroupLimitedCPUCores(unsigned default_cpu_count) return std::min(default_cpu_count, quota_count); } + +/// Returns number of physical cores. If Intel hyper-threading (2-way SMP) is enabled, then getPhysicalConcurrency() returns half of of +/// std::thread::hardware_concurrency(), otherwise both values are the same. +static uint32_t physical_concurrency() +try +{ + /// Derive physical cores from /proc/cpuinfo. Unfortunately, the CPUID instruction isn't reliable accross different vendors / CPU + /// models. Implementation based on boost::thread::physical_concurrency(). See + /// https://doc.callmematthi.eu/static/webArticles/Understanding%20Linux%20_proc_cpuinfo.pdf for semantics of /proc/cpuinfo in the + /// presence of multiple cores per socket, multiple sockets and multiple hyperthreading cores per physical core. + std::ifstream proc_cpuinfo("/proc/cpuinfo"); + + using CoreEntry = std::pair; + + std::set core_entries; + CoreEntry cur_core_entry; + std::string line; + + while (std::getline(proc_cpuinfo, line)) + { + size_t pos = line.find(std::string(":")); + if (pos == std::string::npos) + continue; + + std::string key = line.substr(0, pos); + std::string val = line.substr(pos + 1); + + boost::trim(key); + boost::trim(val); + + if (key == "physical id") + { + cur_core_entry.first = std::stoi(val); + continue; + } + + if (key == "core id") + { + cur_core_entry.second = std::stoi(val); + core_entries.insert(cur_core_entry); + continue; + } + } + return core_entries.empty() ? /*unexpected format*/ std::thread::hardware_concurrency() : static_cast(core_entries.size()); +} +catch (...) +{ + /// parsing error + return std::thread::hardware_concurrency(); +} #endif static unsigned getNumberOfPhysicalCPUCoresImpl() { - unsigned cpu_count = std::thread::hardware_concurrency(); + unsigned cpu_count = std::thread::hardware_concurrency(); /// logical cores + + /// Most x86_64 CPUs have 2-way Hyper-Threading + /// Aarch64 and RISC-V don't have SMT so far. + /// POWER has SMT and it can be multiple way (like 8-way), but we don't know how ClickHouse really behaves, so use all of them. + +#if defined(__x86_64__) + /// On big machines, Hyper-Threading is detrimental to performance (+ ~5% overhead in ClickBench). + /// Let's limit ourself to of physical cores. + /// For few cores - maybe it is a small machine, or runs in a VM or is a limited cloud instance --> it is reasonable to use all the cores. + if (cpu_count >= 32) + cpu_count = physical_concurrency(); +#endif #if defined(OS_LINUX) cpu_count = getCGroupLimitedCPUCores(cpu_count); From 301b684933f54a6ec1c75958c7eba443e0b3c9ca Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 12 Jan 2023 10:00:18 -0300 Subject: [PATCH 035/717] remove trialing whitespace? --- src/Storages/MergeTree/MergeTreeData.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index bc4e8c27b82..d8242953d60 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -6087,7 +6087,7 @@ std::optional MergeTreeData::getQueryProcessingStageWithAgg /// Cannot use projections in case of additional filter. if (query_info.additional_filter_ast) return std::nullopt; - + auto query_ptr = query_info.query; auto original_query_ptr = query_info.original_query; From 92646e3c277fdb41b837abb8ac6206c56cd6098d Mon Sep 17 00:00:00 2001 From: flynn Date: Sat, 14 Jan 2023 17:55:59 +0000 Subject: [PATCH 036/717] initial --- src/Storages/StorageIceberg.cpp | 313 ++++++++++++++++++++++++++++++++ src/Storages/StorageIceberg.h | 117 ++++++++++++ 2 files changed, 430 insertions(+) create mode 100644 src/Storages/StorageIceberg.cpp create mode 100644 src/Storages/StorageIceberg.h diff --git a/src/Storages/StorageIceberg.cpp b/src/Storages/StorageIceberg.cpp new file mode 100644 index 00000000000..415cedc1920 --- /dev/null +++ b/src/Storages/StorageIceberg.cpp @@ -0,0 +1,313 @@ +#include "config.h" +#if USE_AWS_S3 + +# include +# include + +# include +# include +# include +# include + +# include +# include +# include + +# include + +# include +# include +# include + +# include + +# include +# include +# include + +# include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int S3_ERROR; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int INCORRECT_DATA; +} + +JsonMetadataGetter::JsonMetadataGetter(StorageS3::S3Configuration & configuration_, const String & table_path_, ContextPtr context) + : base_configuration(configuration_), table_path(table_path_) +{ + init(context); +} + +void JsonMetadataGetter::init(ContextPtr context) +{ + auto keys = getJsonLogFiles(); + + // read data from every json log file + for (const String & key : keys) + { + auto buf = createS3ReadBuffer(key, context); + + char c; + while (!buf->eof()) + { + /// May be some invalid characters before json. + while (buf->peek(c) && c != '{') + buf->ignore(); + + if (buf->eof()) + break; + + String json_str; + readJSONObjectPossiblyInvalid(json_str, *buf); + + if (json_str.empty()) + continue; + + const JSON json(json_str); + handleJSON(json); + } + } +} + +std::vector JsonMetadataGetter::getJsonLogFiles() +{ + std::vector keys; + + const auto & client = base_configuration.client; + + Aws::S3::Model::ListObjectsV2Request request; + Aws::S3::Model::ListObjectsV2Outcome outcome; + + bool is_finished{false}; + const auto bucket{base_configuration.uri.bucket}; + + request.SetBucket(bucket); + + /// DeltaLake format stores all metadata json files in _delta_log directory + static constexpr auto deltalake_metadata_directory = "_delta_log"; + request.SetPrefix(std::filesystem::path(table_path) / deltalake_metadata_directory); + + while (!is_finished) + { + outcome = client->ListObjectsV2(request); + if (!outcome.IsSuccess()) + throw Exception( + ErrorCodes::S3_ERROR, + "Could not list objects in bucket {} with key {}, S3 exception: {}, message: {}", + quoteString(bucket), + quoteString(table_path), + backQuote(outcome.GetError().GetExceptionName()), + quoteString(outcome.GetError().GetMessage())); + + const auto & result_batch = outcome.GetResult().GetContents(); + for (const auto & obj : result_batch) + { + const auto & filename = obj.GetKey(); + + // DeltaLake metadata files have json extension + if (std::filesystem::path(filename).extension() == ".json") + keys.push_back(filename); + } + + /// Needed in case any more results are available + /// if so, we will continue reading, and not read keys that were already read + request.SetContinuationToken(outcome.GetResult().GetNextContinuationToken()); + + /// Set to false if all of the results were returned. Set to true if more keys + /// are available to return. If the number of results exceeds that specified by + /// MaxKeys, all of the results might not be returned + is_finished = !outcome.GetResult().GetIsTruncated(); + } + + return keys; +} + +std::shared_ptr JsonMetadataGetter::createS3ReadBuffer(const String & key, ContextPtr context) +{ + /// TODO: add parallel downloads + S3Settings::RequestSettings request_settings; + request_settings.max_single_read_retries = 10; + return std::make_shared( + base_configuration.client, + base_configuration.uri.bucket, + key, + base_configuration.uri.version_id, + request_settings, + context->getReadSettings()); +} + +void JsonMetadataGetter::handleJSON(const JSON & json) +{ + if (json.has("add")) + { + auto path = json["add"]["path"].getString(); + auto timestamp = json["add"]["modificationTime"].getInt(); + + metadata.setLastModifiedTime(path, timestamp); + } + else if (json.has("remove")) + { + auto path = json["remove"]["path"].getString(); + auto timestamp = json["remove"]["deletionTimestamp"].getInt(); + + metadata.remove(path, timestamp); + } +} + +namespace +{ + +StorageS3::S3Configuration getBaseConfiguration(const StorageS3Configuration & configuration) +{ + return {configuration.url, configuration.auth_settings, configuration.request_settings, configuration.headers}; +} + +// generateQueryFromKeys constructs query from all parquet filenames +// for underlying StorageS3 engine +String generateQueryFromKeys(const std::vector & keys) +{ + std::string new_query = fmt::format("{{{}}}", fmt::join(keys, ",")); + return new_query; +} + + +StorageS3Configuration getAdjustedS3Configuration( + const ContextPtr & context, + StorageS3::S3Configuration & base_configuration, + const StorageS3Configuration & configuration, + const std::string & table_path, + Poco::Logger * log) +{ + JsonMetadataGetter getter{base_configuration, table_path, context}; + + auto keys = getter.getFiles(); + auto new_uri = base_configuration.uri.uri.toString() + generateQueryFromKeys(keys); + + LOG_DEBUG(log, "New uri: {}", new_uri); + LOG_DEBUG(log, "Table path: {}", table_path); + + // set new url in configuration + StorageS3Configuration new_configuration; + new_configuration.url = new_uri; + new_configuration.auth_settings.access_key_id = configuration.auth_settings.access_key_id; + new_configuration.auth_settings.secret_access_key = configuration.auth_settings.secret_access_key; + new_configuration.format = configuration.format; + + return new_configuration; +} + +} + +StorageIceberg::StorageIceberg( + const StorageS3Configuration & configuration_, + const StorageID & table_id_, + ColumnsDescription columns_, + const ConstraintsDescription & constraints_, + const String & comment, + ContextPtr context_, + std::optional format_settings_) + : IStorage(table_id_) + , base_configuration{getBaseConfiguration(configuration_)} + , log(&Poco::Logger::get("StorageIceberg(" + table_id_.table_name + ")")) + , table_path(base_configuration.uri.key) +{ + StorageInMemoryMetadata storage_metadata; + StorageS3::updateS3Configuration(context_, base_configuration); + + auto new_configuration = getAdjustedS3Configuration(context_, base_configuration, configuration_, table_path, log); + + if (columns_.empty()) + { + columns_ = StorageS3::getTableStructureFromData( + new_configuration, /*distributed processing*/ false, format_settings_, context_, nullptr); + storage_metadata.setColumns(columns_); + } + else + storage_metadata.setColumns(columns_); + + + storage_metadata.setConstraints(constraints_); + storage_metadata.setComment(comment); + setInMemoryMetadata(storage_metadata); + + s3engine = std::make_shared( + new_configuration, + table_id_, + columns_, + constraints_, + comment, + context_, + format_settings_, + /* distributed_processing_ */ false, + nullptr); +} + +Pipe StorageIceberg::read( + const Names & column_names, + const StorageSnapshotPtr & storage_snapshot, + SelectQueryInfo & query_info, + ContextPtr context, + QueryProcessingStage::Enum processed_stage, + size_t max_block_size, + size_t num_streams) +{ + StorageS3::updateS3Configuration(context, base_configuration); + + return s3engine->read(column_names, storage_snapshot, query_info, context, processed_stage, max_block_size, num_streams); +} + +ColumnsDescription StorageIceberg::getTableStructureFromData( + const StorageS3Configuration & configuration, const std::optional & format_settings, ContextPtr ctx) +{ + auto base_configuration = getBaseConfiguration(configuration); + StorageS3::updateS3Configuration(ctx, base_configuration); + auto new_configuration = getAdjustedS3Configuration( + ctx, base_configuration, configuration, base_configuration.uri.key, &Poco::Logger::get("StorageIceberg")); + return StorageS3::getTableStructureFromData( + new_configuration, /*distributed processing*/ false, format_settings, ctx, /*object_infos*/ nullptr); +} + +void registerStorageIceberg(StorageFactory & factory) +{ + factory.registerStorage( + "Iceberg", + [](const StorageFactory::Arguments & args) + { + auto & engine_args = args.engine_args; + if (engine_args.empty() || engine_args.size() < 3) + throw Exception( + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Storage Iceberg requires 3 to 4 arguments: table_url, access_key, secret_access_key, [format]"); + + StorageS3Configuration configuration; + + configuration.url = checkAndGetLiteralArgument(engine_args[0], "url"); + configuration.auth_settings.access_key_id = checkAndGetLiteralArgument(engine_args[1], "access_key_id"); + configuration.auth_settings.secret_access_key = checkAndGetLiteralArgument(engine_args[2], "secret_access_key"); + + if (engine_args.size() == 4) + configuration.format = checkAndGetLiteralArgument(engine_args[3], "format"); + else + { + /// Iceberg uses Parquet by default. + configuration.format = "Parquet"; + } + + return std::make_shared( + configuration, args.table_id, args.columns, args.constraints, args.comment, args.getContext(), std::nullopt); + }, + { + .supports_settings = true, + .supports_schema_inference = true, + .source_access_type = AccessType::S3, + }); +} + +} + +#endif diff --git a/src/Storages/StorageIceberg.h b/src/Storages/StorageIceberg.h new file mode 100644 index 00000000000..d1e7bc6bcbd --- /dev/null +++ b/src/Storages/StorageIceberg.h @@ -0,0 +1,117 @@ +#pragma once + +#include "config.h" + +#if USE_AWS_S3 + +# include +# include + +# include +# include + +namespace Poco +{ +class Logger; +} + +namespace Aws::S3 +{ +class S3Client; +} + +namespace DB +{ + +// Class to parse iceberg metadata and find files needed for query in table +// Iceberg table directory outlook: +// table/ +// data/ +// metadata/ +// The metadata has three layers: metadata -> manifest list -> manifest files +class IcebergMetaParser +{ +public: + IcebergMetaParser(const StorageS3Configuration & configuration_, const String & table_path_, ContextPtr context_); + + void parseMeta(); + + String getNewestMetaFile(); + String getManiFestList(String metadata); + std::vector getManifestFiles(String manifest_list); + void getFilesForRead(const std::vector manifest_files); + + auto getFiles() const {return keys}; + +private: + std::vector keys; + + StorageS3Configuration base_configuration; + String table_path; + ContextPtr context; +}; + +// class to get deltalake log json files and read json from them +class JsonMetadataGetter +{ +public: + JsonMetadataGetter(StorageS3::S3Configuration & configuration_, const String & table_path_, ContextPtr context); + + std::vector getFiles() { return std::move(metadata).listCurrentFiles(); } + +private: + void init(ContextPtr context); + + std::vector getJsonLogFiles(); + + std::shared_ptr createS3ReadBuffer(const String & key, ContextPtr context); + + void handleJSON(const JSON & json); + + StorageS3::S3Configuration base_configuration; + String table_path; + DeltaLakeMetadata metadata; +}; + +class StorageIceberg : public IStorage +{ +public: + // 1. Parses internal file structure of table + // 2. Finds out parts with latest version + // 3. Creates url for underlying StorageS3 enigne to handle reads + StorageIceberg( + const StorageS3Configuration & configuration_, + const StorageID & table_id_, + ColumnsDescription columns_, + const ConstraintsDescription & constraints_, + const String & comment, + ContextPtr context_, + std::optional format_settings_); + + String getName() const override { return "Iceberg"; } + + // Reads latest version of Iceberg table + Pipe read( + const Names & column_names, + const StorageSnapshotPtr & storage_snapshot, + SelectQueryInfo & query_info, + ContextPtr context, + QueryProcessingStage::Enum processed_stage, + size_t max_block_size, + size_t num_streams) override; + + static ColumnsDescription getTableStructureFromData( + const StorageS3Configuration & configuration, + const std::optional & format_settings, + ContextPtr ctx); +private: + + StorageS3::S3Configuration base_configuration; + std::shared_ptr s3engine; + Poco::Logger * log; + String table_path; +}; + +} + +#endif From a8e136315159ab28a5484344f75eeba4a2d09747 Mon Sep 17 00:00:00 2001 From: flynn Date: Wed, 18 Jan 2023 03:43:20 +0000 Subject: [PATCH 037/717] implement storage iceberg --- .../Formats/Impl/AvroRowInputFormat.cpp | 50 ++-- .../Formats/Impl/AvroRowInputFormat.h | 19 +- src/Storages/StorageIceberg.cpp | 232 +++++++++++++----- src/Storages/StorageIceberg.h | 35 +-- 4 files changed, 216 insertions(+), 120 deletions(-) diff --git a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp index 8c6cd8bd91b..5dfcf1ea589 100644 --- a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp @@ -68,35 +68,35 @@ namespace ErrorCodes extern const int CANNOT_READ_ALL_DATA; } -class InputStreamReadBufferAdapter : public avro::InputStream +bool AvroInputStreamReadBufferAdapter::next(const uint8_t ** data, size_t * len) { -public: - explicit InputStreamReadBufferAdapter(ReadBuffer & in_) : in(in_) {} - - bool next(const uint8_t ** data, size_t * len) override + if (in.eof()) { - if (in.eof()) - { - *len = 0; - return false; - } - - *data = reinterpret_cast(in.position()); - *len = in.available(); - - in.position() += in.available(); - return true; + *len = 0; + return false; } - void backup(size_t len) override { in.position() -= len; } + *data = reinterpret_cast(in.position()); + *len = in.available(); - void skip(size_t len) override { in.tryIgnore(len); } + in.position() += in.available(); + return true; +} - size_t byteCount() const override { return in.count(); } +void AvroInputStreamReadBufferAdapter::backup(size_t len) +{ + in.position() -= len; +} -private: - ReadBuffer & in; -}; +void AvroInputStreamReadBufferAdapter::skip(size_t len) +{ + in.tryIgnore(len); +} + +size_t AvroInputStreamReadBufferAdapter::byteCount() const +{ + return in.count(); +} /// Insert value with conversion to the column of target type. template @@ -758,7 +758,7 @@ AvroRowInputFormat::AvroRowInputFormat(const Block & header_, ReadBuffer & in_, void AvroRowInputFormat::readPrefix() { - file_reader_ptr = std::make_unique(std::make_unique(*in)); + file_reader_ptr = std::make_unique(std::make_unique(*in)); deserializer_ptr = std::make_unique( output.getHeader(), file_reader_ptr->dataSchema(), format_settings.avro.allow_missing_fields, format_settings.avro.null_as_default); file_reader_ptr->init(); @@ -915,7 +915,7 @@ AvroConfluentRowInputFormat::AvroConfluentRowInputFormat( void AvroConfluentRowInputFormat::readPrefix() { - input_stream = std::make_unique(*in); + input_stream = std::make_unique(*in); decoder = avro::binaryDecoder(); decoder->init(*input_stream); } @@ -972,7 +972,7 @@ NamesAndTypesList AvroSchemaReader::readSchema() } else { - auto file_reader_ptr = std::make_unique(std::make_unique(in)); + auto file_reader_ptr = std::make_unique(std::make_unique(in)); root_node = file_reader_ptr->dataSchema().root(); } diff --git a/src/Processors/Formats/Impl/AvroRowInputFormat.h b/src/Processors/Formats/Impl/AvroRowInputFormat.h index 4525d7d33b0..cef792da546 100644 --- a/src/Processors/Formats/Impl/AvroRowInputFormat.h +++ b/src/Processors/Formats/Impl/AvroRowInputFormat.h @@ -28,6 +28,23 @@ namespace ErrorCodes extern const int INCORRECT_DATA; } +class AvroInputStreamReadBufferAdapter : public avro::InputStream +{ +public: + explicit AvroInputStreamReadBufferAdapter(ReadBuffer & in_) : in(in_) {} + + bool next(const uint8_t ** data, size_t * len) override; + + void backup(size_t len) override; + + void skip(size_t len) override; + + size_t byteCount() const override; + +private: + ReadBuffer & in; +}; + class AvroDeserializer { public: @@ -185,8 +202,8 @@ public: NamesAndTypesList readSchema() override; + static DataTypePtr avroNodeToDataType(avro::NodePtr node); private: - DataTypePtr avroNodeToDataType(avro::NodePtr node); bool confluent; const FormatSettings format_settings; diff --git a/src/Storages/StorageIceberg.cpp b/src/Storages/StorageIceberg.cpp index 415cedc1920..94c013a404b 100644 --- a/src/Storages/StorageIceberg.cpp +++ b/src/Storages/StorageIceberg.cpp @@ -25,7 +25,11 @@ # include # include -# include +# include + +# include +# include +# include namespace DB { @@ -35,48 +39,35 @@ namespace ErrorCodes extern const int S3_ERROR; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int INCORRECT_DATA; + extern const int FILE_DOESNT_EXIST; } -JsonMetadataGetter::JsonMetadataGetter(StorageS3::S3Configuration & configuration_, const String & table_path_, ContextPtr context) - : base_configuration(configuration_), table_path(table_path_) +IcebergMetaParser::IcebergMetaParser(const StorageS3Configuration & configuration_, const String & table_path_, ContextPtr context_) + : base_configuration(configuration_), table_path(table_path_), context(context_) { - init(context); } -void JsonMetadataGetter::init(ContextPtr context) +std::vector IcebergMetaParser::getFiles() const { - auto keys = getJsonLogFiles(); + auto metadata = getNewestMetaFile(); + auto manifest_list = getManiFestList(metadata); - // read data from every json log file - for (const String & key : keys) + /// When table first created and does not have any data + if (manifest_list.empty()) { - auto buf = createS3ReadBuffer(key, context); - - char c; - while (!buf->eof()) - { - /// May be some invalid characters before json. - while (buf->peek(c) && c != '{') - buf->ignore(); - - if (buf->eof()) - break; - - String json_str; - readJSONObjectPossiblyInvalid(json_str, *buf); - - if (json_str.empty()) - continue; - - const JSON json(json_str); - handleJSON(json); - } + return {}; } + + auto manifest_files = getManifestFiles(manifest_list); + return getFilesForRead(manifest_files); } -std::vector JsonMetadataGetter::getJsonLogFiles() +String IcebergMetaParser::getNewestMetaFile() const { - std::vector keys; + /// Iceberg stores all the metadata.json in metadata directory, and the + /// newest version has the max version name, so we should list all of them + /// then find the newest metadata. + std::vector metadata_files; const auto & client = base_configuration.client; @@ -88,9 +79,8 @@ std::vector JsonMetadataGetter::getJsonLogFiles() request.SetBucket(bucket); - /// DeltaLake format stores all metadata json files in _delta_log directory - static constexpr auto deltalake_metadata_directory = "_delta_log"; - request.SetPrefix(std::filesystem::path(table_path) / deltalake_metadata_directory); + static constexpr auto metadata_directory = "metadata"; + request.SetPrefix(std::filesystem::path(table_path) / metadata_directory); while (!is_finished) { @@ -109,27 +99,156 @@ std::vector JsonMetadataGetter::getJsonLogFiles() { const auto & filename = obj.GetKey(); - // DeltaLake metadata files have json extension if (std::filesystem::path(filename).extension() == ".json") - keys.push_back(filename); + metadata_files.push_back(filename); } - /// Needed in case any more results are available - /// if so, we will continue reading, and not read keys that were already read request.SetContinuationToken(outcome.GetResult().GetNextContinuationToken()); - /// Set to false if all of the results were returned. Set to true if more keys - /// are available to return. If the number of results exceeds that specified by - /// MaxKeys, all of the results might not be returned is_finished = !outcome.GetResult().GetIsTruncated(); } - return keys; + if (metadata_files.empty()) + throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "The metadata file for Iceberg table with path {} doesn't exist", table_path); + + auto it = std::max_element(metadata_files.begin(), metadata_files.end()); + return *it; } -std::shared_ptr JsonMetadataGetter::createS3ReadBuffer(const String & key, ContextPtr context) +String IcebergMetaParser::getManiFestList(String metadata_name) const +{ + auto buffer = createS3ReadBuffer(metadata_name, context); + String json_str; + readJSONObjectPossiblyInvalid(json_str, file); + + /// Looks like base/base/JSON.h can not parse this json file + Poco::JSON::Parser parser; + Poco::Dynamic::Var json = parser.parse(json_str); + Poco::JSON::Object::Ptr object = json.extract(); + + auto current_snapshot_id = object->getValue("current-snapshot-id"); + + auto snapshots = object->get("snapshots").extract(); + + for (size_t i = 0; i < snapshots->size(); ++i) + { + auto snapshot = snapshots->getObject(static_cast(i)); + if (snapshot->getValue("snapshot-id") == current_snapshot_id) + return object->getValue("manifest-list"); + } + + return {}; +} + +static ColumnPtr +parseAvro(const std::uniq_ptr & file_reader, const DataTypePtr & data_type, const String & field_name) +{ + auto deserializer = std::make_unique( + Block{{data_type->createColumn(), data_type, field_name}}, file_reader->dataSchema(), true, true); + file_reader->init(); + MutableColumns columns; + columns.emplace_back(data_type->createColumn()); + + RowReadExtension ext; + while (file_reader->hasMore()) + { + file_reader->decr(); + deserializer->deserializeRow(columns, file_reader->decoder, ext); + } + return columns.at(0); +} + +std::vector IcebergMetaParser::getManifestFiles(const String & manifest_list) const +{ + auto buffer = createS3ReadBuffer(manifest_list, context); + + auto file_reader = std::make_unique(std::make_unique(in)); + + static constexpr manifest_path = "manifest_path"; + + /// The manifest_path is the first field in manifest list file, + /// And its have String data type + /// {'manifest_path': 'xxx', ...} + auto data_type = AvroSchemaReader::avroNodeToDataType(file_reader->dataSchema().root()->leafAt(0)); + auto col = parseAvro(file_reader, data_type, manifest_path); + + std::vector res; + if (col->getDataType() == TypeIndex::String) + { + const auto * col_str = typeid_cast(col.get()); + size_t col_size = col_str->size(); + for (size_t i = 0; i < col_size; ++i) + { + auto file_path = col_str[i].safeGet(); + /// We just need obtain the file name + std::filesystem::path path(file_path); + res.emplace_back(path.filename()); + } + + return res; + } + Throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "The parsed column from Avro file for manifest_path should have data type String, but get {}", + col->getFamilyName()); +} + +std::vector IcebergMetaParser::getFilesForRead(const std::vector & manifest_files) const +{ + std::vector keys; + for (const auto & manifest_file : manifest_files) + { + auto buffer = createS3ReadBuffer(manifest_file, context); + + auto file_reader = std::make_unique(std::make_unique(in)); + + static constexpr manifest_path = "data_file"; + + /// The data_file filed at the 3rd position of the manifest file: + /// {'status': xx, 'snapshot_id': xx, 'data_file': {'file_path': 'xxx', ...}, ...} + /// and it's also a nested record, so its result type is a nested Tuple + auto data_type = AvroSchemaReader::avroNodeToDataType(file_reader->dataSchema().root()->leafAt(2)); + auto col = parseAvro(file_reader, data_type, manifest_path); + + std::vector res; + if (col->getDataType() == TypeIndex::Tuple) + { + auto * col_tuple = typeid_cast(col.get()); + auto * col_str = col_tuple->getColumnPtr(0); + if (col_str->getDataType() == TypeIndex::String) + { + const auto * str_col = typeid_cast(col_str.get()); + size_t col_size = str_col->size(); + for (size_t i = 0; i < col_size; ++i) + { + auto file_path = std_col[i].safeGet(); + /// We just obtain the parition/file name + std::filesystem::path path(file_path); + res.emplace_back(path.parent_path().filename() + '/' + path.filename()); + } + } + else + { + Throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "The parsed column from Avro file for file_path should have data type String, got {}", + col_str->getFamilyName()); + } + } + else + { + Throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "The parsed column from Avro file for data_file field should have data type Tuple, got {}", + col->getFamilyName()); + } + } + + return res; +} + +std::shared_ptr IcebergMetaParser::createS3ReadBuffer(const String & key, ContextPtr context) { - /// TODO: add parallel downloads S3Settings::RequestSettings request_settings; request_settings.max_single_read_retries = 10; return std::make_shared( @@ -141,24 +260,6 @@ std::shared_ptr JsonMetadataGetter::createS3ReadBuffer(const String context->getReadSettings()); } -void JsonMetadataGetter::handleJSON(const JSON & json) -{ - if (json.has("add")) - { - auto path = json["add"]["path"].getString(); - auto timestamp = json["add"]["modificationTime"].getInt(); - - metadata.setLastModifiedTime(path, timestamp); - } - else if (json.has("remove")) - { - auto path = json["remove"]["path"].getString(); - auto timestamp = json["remove"]["deletionTimestamp"].getInt(); - - metadata.remove(path, timestamp); - } -} - namespace { @@ -183,10 +284,11 @@ StorageS3Configuration getAdjustedS3Configuration( const std::string & table_path, Poco::Logger * log) { - JsonMetadataGetter getter{base_configuration, table_path, context}; + IcebergMetaParser parser{base_configuration, table_path, context}; - auto keys = getter.getFiles(); - auto new_uri = base_configuration.uri.uri.toString() + generateQueryFromKeys(keys); + auto keys = parser.getFiles(); + static constexpr iceberg_data_directory = "data"; + auto new_uri = std::filesystem::path(base_configuration.uri.uri.toString()) / iceberg_data_directory / generateQueryFromKeys(keys); LOG_DEBUG(log, "New uri: {}", new_uri); LOG_DEBUG(log, "Table path: {}", table_path); diff --git a/src/Storages/StorageIceberg.h b/src/Storages/StorageIceberg.h index d1e7bc6bcbd..a139a40fc29 100644 --- a/src/Storages/StorageIceberg.h +++ b/src/Storages/StorageIceberg.h @@ -34,43 +34,20 @@ class IcebergMetaParser public: IcebergMetaParser(const StorageS3Configuration & configuration_, const String & table_path_, ContextPtr context_); - void parseMeta(); - - String getNewestMetaFile(); - String getManiFestList(String metadata); - std::vector getManifestFiles(String manifest_list); - void getFilesForRead(const std::vector manifest_files); - - auto getFiles() const {return keys}; + std::vector getFiles() const; private: - std::vector keys; - StorageS3Configuration base_configuration; String table_path; ContextPtr context; -}; -// class to get deltalake log json files and read json from them -class JsonMetadataGetter -{ -public: - JsonMetadataGetter(StorageS3::S3Configuration & configuration_, const String & table_path_, ContextPtr context); - - std::vector getFiles() { return std::move(metadata).listCurrentFiles(); } - -private: - void init(ContextPtr context); - - std::vector getJsonLogFiles(); + /// Just get file name + String getNewestMetaFile() const; + String getManiFestList(const String & metadata_name) const; + std::vector getManifestFiles(const String & manifest_list) const; + std::vector getFilesForRead(const std::vector & manifest_files); std::shared_ptr createS3ReadBuffer(const String & key, ContextPtr context); - - void handleJSON(const JSON & json); - - StorageS3::S3Configuration base_configuration; - String table_path; - DeltaLakeMetadata metadata; }; class StorageIceberg : public IStorage From 2fb32dc56c5576a0d75448ae71dab7aa44be0d2b Mon Sep 17 00:00:00 2001 From: flynn Date: Wed, 18 Jan 2023 08:33:55 +0000 Subject: [PATCH 038/717] fix and add test --- src/Storages/StorageIceberg.cpp | 72 ++--- src/Storages/StorageIceberg.h | 9 +- src/Storages/StorageS3.h | 1 + src/Storages/registerStorages.cpp | 4 +- src/TableFunctions/TableFunctionIceberg.cpp | 169 ++++++++++++ src/TableFunctions/TableFunctionIceberg.h | 44 +++ src/TableFunctions/registerTableFunctions.cpp | 1 + src/TableFunctions/registerTableFunctions.h | 1 + .../test_storage_iceberg/__init__.py | 0 ...4-43d2-a01a-484f107210cb-00001.parquet.crc | Bin 0 -> 24 bytes ...3-4b49-aa8d-3ebbb5c69379-00001.parquet.crc | Bin 0 -> 24 bytes ...f-436e-96ae-5c639106aec0-00001.parquet.crc | Bin 0 -> 24 bytes ...3-45ea-bc9f-f087ab7c2a69-00001.parquet.crc | Bin 0 -> 24 bytes ...-dc44-43d2-a01a-484f107210cb-00001.parquet | Bin 0 -> 1565 bytes ...-da93-4b49-aa8d-3ebbb5c69379-00001.parquet | Bin 0 -> 1603 bytes ...-116f-436e-96ae-5c639106aec0-00001.parquet | Bin 0 -> 1819 bytes ...-5243-45ea-bc9f-f087ab7c2a69-00001.parquet | Bin 0 -> 1819 bytes ...4-43d2-a01a-484f107210cb-00002.parquet.crc | Bin 0 -> 24 bytes ...5-48e0-b560-d28fc767f054-00001.parquet.crc | Bin 0 -> 20 bytes ...3-4b49-aa8d-3ebbb5c69379-00002.parquet.crc | Bin 0 -> 24 bytes ...f-436e-96ae-5c639106aec0-00002.parquet.crc | Bin 0 -> 24 bytes ...3-45ea-bc9f-f087ab7c2a69-00002.parquet.crc | Bin 0 -> 24 bytes ...-dc44-43d2-a01a-484f107210cb-00002.parquet | Bin 0 -> 1620 bytes ...-2325-48e0-b560-d28fc767f054-00001.parquet | Bin 0 -> 1501 bytes ...-da93-4b49-aa8d-3ebbb5c69379-00002.parquet | Bin 0 -> 1677 bytes ...-116f-436e-96ae-5c639106aec0-00002.parquet | Bin 0 -> 1841 bytes ...-5243-45ea-bc9f-f087ab7c2a69-00002.parquet | Bin 0 -> 1846 bytes ...5-48e0-b560-d28fc767f054-00002.parquet.crc | Bin 0 -> 20 bytes ...3-4b49-aa8d-3ebbb5c69379-00003.parquet.crc | Bin 0 -> 20 bytes ...f-436e-96ae-5c639106aec0-00003.parquet.crc | Bin 0 -> 24 bytes ...3-45ea-bc9f-f087ab7c2a69-00003.parquet.crc | Bin 0 -> 24 bytes ...-2325-48e0-b560-d28fc767f054-00002.parquet | Bin 0 -> 1499 bytes ...-da93-4b49-aa8d-3ebbb5c69379-00003.parquet | Bin 0 -> 1535 bytes ...-116f-436e-96ae-5c639106aec0-00003.parquet | Bin 0 -> 1801 bytes ...-5243-45ea-bc9f-f087ab7c2a69-00003.parquet | Bin 0 -> 1801 bytes ...5-48e0-b560-d28fc767f054-00003.parquet.crc | Bin 0 -> 20 bytes ...3-4b49-aa8d-3ebbb5c69379-00004.parquet.crc | Bin 0 -> 24 bytes ...f-436e-96ae-5c639106aec0-00004.parquet.crc | Bin 0 -> 24 bytes ...3-45ea-bc9f-f087ab7c2a69-00004.parquet.crc | Bin 0 -> 24 bytes ...-2325-48e0-b560-d28fc767f054-00003.parquet | Bin 0 -> 1500 bytes ...-da93-4b49-aa8d-3ebbb5c69379-00004.parquet | Bin 0 -> 1539 bytes ...-116f-436e-96ae-5c639106aec0-00004.parquet | Bin 0 -> 1802 bytes ...-5243-45ea-bc9f-f087ab7c2a69-00004.parquet | Bin 0 -> 1802 bytes ...5-48e0-b560-d28fc767f054-00004.parquet.crc | Bin 0 -> 20 bytes ...b-4dbc-9825-61a5b85c6dba-00001.parquet.crc | Bin 0 -> 20 bytes ...3-4b49-aa8d-3ebbb5c69379-00005.parquet.crc | Bin 0 -> 24 bytes ...f-436e-96ae-5c639106aec0-00005.parquet.crc | Bin 0 -> 24 bytes ...3-45ea-bc9f-f087ab7c2a69-00005.parquet.crc | Bin 0 -> 24 bytes ...-2325-48e0-b560-d28fc767f054-00004.parquet | Bin 0 -> 1500 bytes ...-366b-4dbc-9825-61a5b85c6dba-00001.parquet | Bin 0 -> 1501 bytes ...-da93-4b49-aa8d-3ebbb5c69379-00005.parquet | Bin 0 -> 1603 bytes ...-116f-436e-96ae-5c639106aec0-00005.parquet | Bin 0 -> 1818 bytes ...-5243-45ea-bc9f-f087ab7c2a69-00005.parquet | Bin 0 -> 1818 bytes ...b-4dbc-9825-61a5b85c6dba-00002.parquet.crc | Bin 0 -> 20 bytes ...3-4b49-aa8d-3ebbb5c69379-00006.parquet.crc | Bin 0 -> 20 bytes ...f-436e-96ae-5c639106aec0-00006.parquet.crc | Bin 0 -> 24 bytes ...3-45ea-bc9f-f087ab7c2a69-00006.parquet.crc | Bin 0 -> 24 bytes ...-366b-4dbc-9825-61a5b85c6dba-00002.parquet | Bin 0 -> 1499 bytes ...-da93-4b49-aa8d-3ebbb5c69379-00006.parquet | Bin 0 -> 1535 bytes ...-116f-436e-96ae-5c639106aec0-00006.parquet | Bin 0 -> 1801 bytes ...-5243-45ea-bc9f-f087ab7c2a69-00006.parquet | Bin 0 -> 1801 bytes ...b-4dbc-9825-61a5b85c6dba-00003.parquet.crc | Bin 0 -> 20 bytes ...3-4b49-aa8d-3ebbb5c69379-00007.parquet.crc | Bin 0 -> 24 bytes ...f-436e-96ae-5c639106aec0-00007.parquet.crc | Bin 0 -> 24 bytes ...3-45ea-bc9f-f087ab7c2a69-00007.parquet.crc | Bin 0 -> 24 bytes ...-366b-4dbc-9825-61a5b85c6dba-00003.parquet | Bin 0 -> 1501 bytes ...-da93-4b49-aa8d-3ebbb5c69379-00007.parquet | Bin 0 -> 1539 bytes ...-116f-436e-96ae-5c639106aec0-00007.parquet | Bin 0 -> 1803 bytes ...-5243-45ea-bc9f-f087ab7c2a69-00007.parquet | Bin 0 -> 1803 bytes ...b-4dbc-9825-61a5b85c6dba-00004.parquet.crc | Bin 0 -> 20 bytes ...3-4b49-aa8d-3ebbb5c69379-00008.parquet.crc | Bin 0 -> 24 bytes ...f-436e-96ae-5c639106aec0-00008.parquet.crc | Bin 0 -> 24 bytes ...3-45ea-bc9f-f087ab7c2a69-00008.parquet.crc | Bin 0 -> 24 bytes ...-366b-4dbc-9825-61a5b85c6dba-00004.parquet | Bin 0 -> 1500 bytes ...-da93-4b49-aa8d-3ebbb5c69379-00008.parquet | Bin 0 -> 1539 bytes ...-116f-436e-96ae-5c639106aec0-00008.parquet | Bin 0 -> 1802 bytes ...-5243-45ea-bc9f-f087ab7c2a69-00008.parquet | Bin 0 -> 1802 bytes ...2-480e-9a05-6f8ec703e52b.metadata.json.crc | Bin 0 -> 28 bytes ...9-404a-907a-9e33b85a3eee.metadata.json.crc | Bin 0 -> 36 bytes ...4-480b-86e8-8483b4343fdf.metadata.json.crc | Bin 0 -> 40 bytes ...e-432b-9b40-a50d081f5691.metadata.json.crc | Bin 0 -> 48 bytes ...1-4596-a87b-9265403d2052.metadata.json.crc | Bin 0 -> 56 bytes ...3-4eba-ac85-7a8b10a00ca1.metadata.json.crc | Bin 0 -> 64 bytes ...b-4fca-be66-0fca929084ea.metadata.json.crc | Bin 0 -> 72 bytes ...4b-b53c-41d4-a71e-06bf618bad7b-m0.avro.crc | Bin 0 -> 60 bytes ...a9-1537-455f-98e5-0a067af5752a-m0.avro.crc | Bin 0 -> 60 bytes ...bd-fe20-4a55-917c-36cb8f6a488c-m0.avro.crc | Bin 0 -> 60 bytes ...1d-ea86-42dd-82d1-1981332a0f6d-m0.avro.crc | Bin 0 -> 60 bytes ...dd-c8eb-4a35-a17f-75b68ee25005-m0.avro.crc | Bin 0 -> 60 bytes ...e9-fbd3-4411-a5c6-0cc14a2f1392-m0.avro.crc | Bin 0 -> 60 bytes ...891e9-fbd3-4411-a5c6-0cc14a2f1392.avro.crc | Bin 0 -> 40 bytes ...db3dd-c8eb-4a35-a17f-75b68ee25005.avro.crc | Bin 0 -> 40 bytes ...dd31d-ea86-42dd-82d1-1981332a0f6d.avro.crc | Bin 0 -> 40 bytes ...cf54b-b53c-41d4-a71e-06bf618bad7b.avro.crc | Bin 0 -> 40 bytes ...c62a9-1537-455f-98e5-0a067af5752a.avro.crc | Bin 0 -> 40 bytes ...325bd-fe20-4a55-917c-36cb8f6a488c.avro.crc | Bin 0 -> 40 bytes ...-26f2-480e-9a05-6f8ec703e52b.metadata.json | 99 +++++++ ...-2209-404a-907a-9e33b85a3eee.metadata.json | 129 +++++++++ ...-46c4-480b-86e8-8483b4343fdf.metadata.json | 155 +++++++++++ ...-9bae-432b-9b40-a50d081f5691.metadata.json | 181 ++++++++++++ ...-8841-4596-a87b-9265403d2052.metadata.json | 207 ++++++++++++++ ...-70b3-4eba-ac85-7a8b10a00ca1.metadata.json | 233 ++++++++++++++++ ...-535b-4fca-be66-0fca929084ea.metadata.json | 259 ++++++++++++++++++ ...acf54b-b53c-41d4-a71e-06bf618bad7b-m0.avro | Bin 0 -> 6510 bytes ...3c62a9-1537-455f-98e5-0a067af5752a-m0.avro | Bin 0 -> 6324 bytes ...e325bd-fe20-4a55-917c-36cb8f6a488c-m0.avro | Bin 0 -> 6370 bytes ...1dd31d-ea86-42dd-82d1-1981332a0f6d-m0.avro | Bin 0 -> 6363 bytes ...5db3dd-c8eb-4a35-a17f-75b68ee25005-m0.avro | Bin 0 -> 6501 bytes ...e891e9-fbd3-4411-a5c6-0cc14a2f1392-m0.avro | Bin 0 -> 6488 bytes ...-f9e891e9-fbd3-4411-a5c6-0cc14a2f1392.avro | Bin 0 -> 3948 bytes ...-e25db3dd-c8eb-4a35-a17f-75b68ee25005.avro | Bin 0 -> 4042 bytes ...-a51dd31d-ea86-42dd-82d1-1981332a0f6d.avro | Bin 0 -> 3899 bytes ...-08acf54b-b53c-41d4-a71e-06bf618bad7b.avro | Bin 0 -> 3990 bytes ...-5e3c62a9-1537-455f-98e5-0a067af5752a.avro | Bin 0 -> 3764 bytes ...-7ae325bd-fe20-4a55-917c-36cb8f6a488c.avro | Bin 0 -> 3845 bytes .../integration/test_storage_iceberg/test.py | 158 +++++++++++ 116 files changed, 1685 insertions(+), 37 deletions(-) create mode 100644 src/TableFunctions/TableFunctionIceberg.cpp create mode 100644 src/TableFunctions/TableFunctionIceberg.h create mode 100644 tests/integration/test_storage_iceberg/__init__.py create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=1/.00000-0-c070e655-dc44-43d2-a01a-484f107210cb-00001.parquet.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=1/.00000-3-eea1ad05-da93-4b49-aa8d-3ebbb5c69379-00001.parquet.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=1/.00000-4-c4940e8e-116f-436e-96ae-5c639106aec0-00001.parquet.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=1/.00000-5-f80715f7-5243-45ea-bc9f-f087ab7c2a69-00001.parquet.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=1/00000-0-c070e655-dc44-43d2-a01a-484f107210cb-00001.parquet create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=1/00000-3-eea1ad05-da93-4b49-aa8d-3ebbb5c69379-00001.parquet create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=1/00000-4-c4940e8e-116f-436e-96ae-5c639106aec0-00001.parquet create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=1/00000-5-f80715f7-5243-45ea-bc9f-f087ab7c2a69-00001.parquet create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=2/.00000-0-c070e655-dc44-43d2-a01a-484f107210cb-00002.parquet.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=2/.00000-1-3a214094-2325-48e0-b560-d28fc767f054-00001.parquet.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=2/.00000-3-eea1ad05-da93-4b49-aa8d-3ebbb5c69379-00002.parquet.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=2/.00000-4-c4940e8e-116f-436e-96ae-5c639106aec0-00002.parquet.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=2/.00000-5-f80715f7-5243-45ea-bc9f-f087ab7c2a69-00002.parquet.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=2/00000-0-c070e655-dc44-43d2-a01a-484f107210cb-00002.parquet create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=2/00000-1-3a214094-2325-48e0-b560-d28fc767f054-00001.parquet create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=2/00000-3-eea1ad05-da93-4b49-aa8d-3ebbb5c69379-00002.parquet create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=2/00000-4-c4940e8e-116f-436e-96ae-5c639106aec0-00002.parquet create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=2/00000-5-f80715f7-5243-45ea-bc9f-f087ab7c2a69-00002.parquet create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=3/.00000-1-3a214094-2325-48e0-b560-d28fc767f054-00002.parquet.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=3/.00000-3-eea1ad05-da93-4b49-aa8d-3ebbb5c69379-00003.parquet.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=3/.00000-4-c4940e8e-116f-436e-96ae-5c639106aec0-00003.parquet.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=3/.00000-5-f80715f7-5243-45ea-bc9f-f087ab7c2a69-00003.parquet.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=3/00000-1-3a214094-2325-48e0-b560-d28fc767f054-00002.parquet create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=3/00000-3-eea1ad05-da93-4b49-aa8d-3ebbb5c69379-00003.parquet create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=3/00000-4-c4940e8e-116f-436e-96ae-5c639106aec0-00003.parquet create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=3/00000-5-f80715f7-5243-45ea-bc9f-f087ab7c2a69-00003.parquet create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=4/.00000-1-3a214094-2325-48e0-b560-d28fc767f054-00003.parquet.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=4/.00000-3-eea1ad05-da93-4b49-aa8d-3ebbb5c69379-00004.parquet.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=4/.00000-4-c4940e8e-116f-436e-96ae-5c639106aec0-00004.parquet.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=4/.00000-5-f80715f7-5243-45ea-bc9f-f087ab7c2a69-00004.parquet.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=4/00000-1-3a214094-2325-48e0-b560-d28fc767f054-00003.parquet create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=4/00000-3-eea1ad05-da93-4b49-aa8d-3ebbb5c69379-00004.parquet create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=4/00000-4-c4940e8e-116f-436e-96ae-5c639106aec0-00004.parquet create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=4/00000-5-f80715f7-5243-45ea-bc9f-f087ab7c2a69-00004.parquet create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=5/.00000-1-3a214094-2325-48e0-b560-d28fc767f054-00004.parquet.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=5/.00000-2-cbd5000d-366b-4dbc-9825-61a5b85c6dba-00001.parquet.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=5/.00000-3-eea1ad05-da93-4b49-aa8d-3ebbb5c69379-00005.parquet.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=5/.00000-4-c4940e8e-116f-436e-96ae-5c639106aec0-00005.parquet.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=5/.00000-5-f80715f7-5243-45ea-bc9f-f087ab7c2a69-00005.parquet.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=5/00000-1-3a214094-2325-48e0-b560-d28fc767f054-00004.parquet create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=5/00000-2-cbd5000d-366b-4dbc-9825-61a5b85c6dba-00001.parquet create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=5/00000-3-eea1ad05-da93-4b49-aa8d-3ebbb5c69379-00005.parquet create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=5/00000-4-c4940e8e-116f-436e-96ae-5c639106aec0-00005.parquet create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=5/00000-5-f80715f7-5243-45ea-bc9f-f087ab7c2a69-00005.parquet create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=6/.00000-2-cbd5000d-366b-4dbc-9825-61a5b85c6dba-00002.parquet.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=6/.00000-3-eea1ad05-da93-4b49-aa8d-3ebbb5c69379-00006.parquet.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=6/.00000-4-c4940e8e-116f-436e-96ae-5c639106aec0-00006.parquet.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=6/.00000-5-f80715f7-5243-45ea-bc9f-f087ab7c2a69-00006.parquet.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=6/00000-2-cbd5000d-366b-4dbc-9825-61a5b85c6dba-00002.parquet create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=6/00000-3-eea1ad05-da93-4b49-aa8d-3ebbb5c69379-00006.parquet create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=6/00000-4-c4940e8e-116f-436e-96ae-5c639106aec0-00006.parquet create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=6/00000-5-f80715f7-5243-45ea-bc9f-f087ab7c2a69-00006.parquet create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=7/.00000-2-cbd5000d-366b-4dbc-9825-61a5b85c6dba-00003.parquet.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=7/.00000-3-eea1ad05-da93-4b49-aa8d-3ebbb5c69379-00007.parquet.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=7/.00000-4-c4940e8e-116f-436e-96ae-5c639106aec0-00007.parquet.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=7/.00000-5-f80715f7-5243-45ea-bc9f-f087ab7c2a69-00007.parquet.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=7/00000-2-cbd5000d-366b-4dbc-9825-61a5b85c6dba-00003.parquet create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=7/00000-3-eea1ad05-da93-4b49-aa8d-3ebbb5c69379-00007.parquet create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=7/00000-4-c4940e8e-116f-436e-96ae-5c639106aec0-00007.parquet create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=7/00000-5-f80715f7-5243-45ea-bc9f-f087ab7c2a69-00007.parquet create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=8/.00000-2-cbd5000d-366b-4dbc-9825-61a5b85c6dba-00004.parquet.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=8/.00000-3-eea1ad05-da93-4b49-aa8d-3ebbb5c69379-00008.parquet.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=8/.00000-4-c4940e8e-116f-436e-96ae-5c639106aec0-00008.parquet.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=8/.00000-5-f80715f7-5243-45ea-bc9f-f087ab7c2a69-00008.parquet.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=8/00000-2-cbd5000d-366b-4dbc-9825-61a5b85c6dba-00004.parquet create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=8/00000-3-eea1ad05-da93-4b49-aa8d-3ebbb5c69379-00008.parquet create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=8/00000-4-c4940e8e-116f-436e-96ae-5c639106aec0-00008.parquet create mode 100644 tests/integration/test_storage_iceberg/taxis/data/vendor_id=8/00000-5-f80715f7-5243-45ea-bc9f-f087ab7c2a69-00008.parquet create mode 100644 tests/integration/test_storage_iceberg/taxis/metadata/.00000-8908cc23-26f2-480e-9a05-6f8ec703e52b.metadata.json.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/metadata/.00001-8fffa3b5-2209-404a-907a-9e33b85a3eee.metadata.json.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/metadata/.00002-c55cb6e3-46c4-480b-86e8-8483b4343fdf.metadata.json.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/metadata/.00003-c1ceebf1-9bae-432b-9b40-a50d081f5691.metadata.json.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/metadata/.00004-0af373c4-8841-4596-a87b-9265403d2052.metadata.json.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/metadata/.00005-d0b691c7-70b3-4eba-ac85-7a8b10a00ca1.metadata.json.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/metadata/.00006-366f138d-535b-4fca-be66-0fca929084ea.metadata.json.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/metadata/.08acf54b-b53c-41d4-a71e-06bf618bad7b-m0.avro.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/metadata/.5e3c62a9-1537-455f-98e5-0a067af5752a-m0.avro.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/metadata/.7ae325bd-fe20-4a55-917c-36cb8f6a488c-m0.avro.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/metadata/.a51dd31d-ea86-42dd-82d1-1981332a0f6d-m0.avro.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/metadata/.e25db3dd-c8eb-4a35-a17f-75b68ee25005-m0.avro.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/metadata/.f9e891e9-fbd3-4411-a5c6-0cc14a2f1392-m0.avro.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/metadata/.snap-1793608066486471262-1-f9e891e9-fbd3-4411-a5c6-0cc14a2f1392.avro.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/metadata/.snap-5277285803176961576-1-e25db3dd-c8eb-4a35-a17f-75b68ee25005.avro.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/metadata/.snap-5735460159761889536-1-a51dd31d-ea86-42dd-82d1-1981332a0f6d.avro.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/metadata/.snap-608017560956539142-1-08acf54b-b53c-41d4-a71e-06bf618bad7b.avro.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/metadata/.snap-6850377589038341628-1-5e3c62a9-1537-455f-98e5-0a067af5752a.avro.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/metadata/.snap-7171740521400098346-1-7ae325bd-fe20-4a55-917c-36cb8f6a488c.avro.crc create mode 100644 tests/integration/test_storage_iceberg/taxis/metadata/00000-8908cc23-26f2-480e-9a05-6f8ec703e52b.metadata.json create mode 100644 tests/integration/test_storage_iceberg/taxis/metadata/00001-8fffa3b5-2209-404a-907a-9e33b85a3eee.metadata.json create mode 100644 tests/integration/test_storage_iceberg/taxis/metadata/00002-c55cb6e3-46c4-480b-86e8-8483b4343fdf.metadata.json create mode 100644 tests/integration/test_storage_iceberg/taxis/metadata/00003-c1ceebf1-9bae-432b-9b40-a50d081f5691.metadata.json create mode 100644 tests/integration/test_storage_iceberg/taxis/metadata/00004-0af373c4-8841-4596-a87b-9265403d2052.metadata.json create mode 100644 tests/integration/test_storage_iceberg/taxis/metadata/00005-d0b691c7-70b3-4eba-ac85-7a8b10a00ca1.metadata.json create mode 100644 tests/integration/test_storage_iceberg/taxis/metadata/00006-366f138d-535b-4fca-be66-0fca929084ea.metadata.json create mode 100644 tests/integration/test_storage_iceberg/taxis/metadata/08acf54b-b53c-41d4-a71e-06bf618bad7b-m0.avro create mode 100644 tests/integration/test_storage_iceberg/taxis/metadata/5e3c62a9-1537-455f-98e5-0a067af5752a-m0.avro create mode 100644 tests/integration/test_storage_iceberg/taxis/metadata/7ae325bd-fe20-4a55-917c-36cb8f6a488c-m0.avro create mode 100644 tests/integration/test_storage_iceberg/taxis/metadata/a51dd31d-ea86-42dd-82d1-1981332a0f6d-m0.avro create mode 100644 tests/integration/test_storage_iceberg/taxis/metadata/e25db3dd-c8eb-4a35-a17f-75b68ee25005-m0.avro create mode 100644 tests/integration/test_storage_iceberg/taxis/metadata/f9e891e9-fbd3-4411-a5c6-0cc14a2f1392-m0.avro create mode 100644 tests/integration/test_storage_iceberg/taxis/metadata/snap-1793608066486471262-1-f9e891e9-fbd3-4411-a5c6-0cc14a2f1392.avro create mode 100644 tests/integration/test_storage_iceberg/taxis/metadata/snap-5277285803176961576-1-e25db3dd-c8eb-4a35-a17f-75b68ee25005.avro create mode 100644 tests/integration/test_storage_iceberg/taxis/metadata/snap-5735460159761889536-1-a51dd31d-ea86-42dd-82d1-1981332a0f6d.avro create mode 100644 tests/integration/test_storage_iceberg/taxis/metadata/snap-608017560956539142-1-08acf54b-b53c-41d4-a71e-06bf618bad7b.avro create mode 100644 tests/integration/test_storage_iceberg/taxis/metadata/snap-6850377589038341628-1-5e3c62a9-1537-455f-98e5-0a067af5752a.avro create mode 100644 tests/integration/test_storage_iceberg/taxis/metadata/snap-7171740521400098346-1-7ae325bd-fe20-4a55-917c-36cb8f6a488c.avro create mode 100644 tests/integration/test_storage_iceberg/test.py diff --git a/src/Storages/StorageIceberg.cpp b/src/Storages/StorageIceberg.cpp index 94c013a404b..aa02e920abf 100644 --- a/src/Storages/StorageIceberg.cpp +++ b/src/Storages/StorageIceberg.cpp @@ -4,6 +4,10 @@ # include # include +# include +# include +# include + # include # include # include @@ -40,9 +44,10 @@ namespace ErrorCodes extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int INCORRECT_DATA; extern const int FILE_DOESNT_EXIST; + extern const int ILLEGAL_COLUMN; } -IcebergMetaParser::IcebergMetaParser(const StorageS3Configuration & configuration_, const String & table_path_, ContextPtr context_) +IcebergMetaParser::IcebergMetaParser(const StorageS3::S3Configuration & configuration_, const String & table_path_, ContextPtr context_) : base_configuration(configuration_), table_path(table_path_), context(context_) { } @@ -79,7 +84,6 @@ String IcebergMetaParser::getNewestMetaFile() const request.SetBucket(bucket); - static constexpr auto metadata_directory = "metadata"; request.SetPrefix(std::filesystem::path(table_path) / metadata_directory); while (!is_finished) @@ -115,11 +119,11 @@ String IcebergMetaParser::getNewestMetaFile() const return *it; } -String IcebergMetaParser::getManiFestList(String metadata_name) const +String IcebergMetaParser::getManiFestList(const String & metadata_name) const { - auto buffer = createS3ReadBuffer(metadata_name, context); + auto buffer = createS3ReadBuffer(metadata_name); String json_str; - readJSONObjectPossiblyInvalid(json_str, file); + readJSONObjectPossiblyInvalid(json_str, *buffer); /// Looks like base/base/JSON.h can not parse this json file Poco::JSON::Parser parser; @@ -134,14 +138,17 @@ String IcebergMetaParser::getManiFestList(String metadata_name) const { auto snapshot = snapshots->getObject(static_cast(i)); if (snapshot->getValue("snapshot-id") == current_snapshot_id) - return object->getValue("manifest-list"); + { + auto path = snapshot->getValue("manifest-list"); + return std::filesystem::path(table_path) / metadata_directory / std::filesystem::path(path).filename(); + } } return {}; } -static ColumnPtr -parseAvro(const std::uniq_ptr & file_reader, const DataTypePtr & data_type, const String & field_name) +static MutableColumns +parseAvro(const std::unique_ptr & file_reader, const DataTypePtr & data_type, const String & field_name) { auto deserializer = std::make_unique( Block{{data_type->createColumn(), data_type, field_name}}, file_reader->dataSchema(), true, true); @@ -153,24 +160,25 @@ parseAvro(const std::uniq_ptr & file_reader, const Dat while (file_reader->hasMore()) { file_reader->decr(); - deserializer->deserializeRow(columns, file_reader->decoder, ext); + deserializer->deserializeRow(columns, file_reader->decoder(), ext); } - return columns.at(0); + return columns; } std::vector IcebergMetaParser::getManifestFiles(const String & manifest_list) const { - auto buffer = createS3ReadBuffer(manifest_list, context); + auto buffer = createS3ReadBuffer(manifest_list); - auto file_reader = std::make_unique(std::make_unique(in)); + auto file_reader = std::make_unique(std::make_unique(*buffer)); - static constexpr manifest_path = "manifest_path"; + static constexpr auto manifest_path = "manifest_path"; /// The manifest_path is the first field in manifest list file, /// And its have String data type /// {'manifest_path': 'xxx', ...} auto data_type = AvroSchemaReader::avroNodeToDataType(file_reader->dataSchema().root()->leafAt(0)); - auto col = parseAvro(file_reader, data_type, manifest_path); + auto columns = parseAvro(file_reader, data_type, manifest_path); + auto & col = columns.at(0); std::vector res; if (col->getDataType() == TypeIndex::String) @@ -179,15 +187,15 @@ std::vector IcebergMetaParser::getManifestFiles(const String & manifest_ size_t col_size = col_str->size(); for (size_t i = 0; i < col_size; ++i) { - auto file_path = col_str[i].safeGet(); + auto file_path = col_str->getDataAt(i).toView(); /// We just need obtain the file name std::filesystem::path path(file_path); - res.emplace_back(path.filename()); + res.emplace_back(std::filesystem::path(table_path) / metadata_directory / path.filename()); } return res; } - Throw Exception( + throw Exception( ErrorCodes::ILLEGAL_COLUMN, "The parsed column from Avro file for manifest_path should have data type String, but get {}", col->getFamilyName()); @@ -198,38 +206,38 @@ std::vector IcebergMetaParser::getFilesForRead(const std::vector std::vector keys; for (const auto & manifest_file : manifest_files) { - auto buffer = createS3ReadBuffer(manifest_file, context); + auto buffer = createS3ReadBuffer(manifest_file); - auto file_reader = std::make_unique(std::make_unique(in)); + auto file_reader = std::make_unique(std::make_unique(*buffer)); - static constexpr manifest_path = "data_file"; + static constexpr auto manifest_path = "data_file"; /// The data_file filed at the 3rd position of the manifest file: /// {'status': xx, 'snapshot_id': xx, 'data_file': {'file_path': 'xxx', ...}, ...} /// and it's also a nested record, so its result type is a nested Tuple auto data_type = AvroSchemaReader::avroNodeToDataType(file_reader->dataSchema().root()->leafAt(2)); - auto col = parseAvro(file_reader, data_type, manifest_path); + auto columns = parseAvro(file_reader, data_type, manifest_path); + auto & col = columns.at(0); - std::vector res; if (col->getDataType() == TypeIndex::Tuple) { auto * col_tuple = typeid_cast(col.get()); - auto * col_str = col_tuple->getColumnPtr(0); + auto & col_str = col_tuple->getColumnPtr(0); if (col_str->getDataType() == TypeIndex::String) { - const auto * str_col = typeid_cast(col_str.get()); + const auto * str_col = typeid_cast(col_str.get()); size_t col_size = str_col->size(); for (size_t i = 0; i < col_size; ++i) { - auto file_path = std_col[i].safeGet(); + auto file_path = str_col->getDataAt(i).toView(); /// We just obtain the parition/file name std::filesystem::path path(file_path); - res.emplace_back(path.parent_path().filename() + '/' + path.filename()); + keys.emplace_back(path.parent_path().filename() / path.filename()); } } else { - Throw Exception( + throw Exception( ErrorCodes::ILLEGAL_COLUMN, "The parsed column from Avro file for file_path should have data type String, got {}", col_str->getFamilyName()); @@ -237,17 +245,17 @@ std::vector IcebergMetaParser::getFilesForRead(const std::vector } else { - Throw Exception( + throw Exception( ErrorCodes::ILLEGAL_COLUMN, "The parsed column from Avro file for data_file field should have data type Tuple, got {}", col->getFamilyName()); } } - return res; + return keys; } -std::shared_ptr IcebergMetaParser::createS3ReadBuffer(const String & key, ContextPtr context) +std::shared_ptr IcebergMetaParser::createS3ReadBuffer(const String & key) const { S3Settings::RequestSettings request_settings; request_settings.max_single_read_retries = 10; @@ -287,10 +295,10 @@ StorageS3Configuration getAdjustedS3Configuration( IcebergMetaParser parser{base_configuration, table_path, context}; auto keys = parser.getFiles(); - static constexpr iceberg_data_directory = "data"; + static constexpr auto iceberg_data_directory = "data"; auto new_uri = std::filesystem::path(base_configuration.uri.uri.toString()) / iceberg_data_directory / generateQueryFromKeys(keys); - LOG_DEBUG(log, "New uri: {}", new_uri); + LOG_DEBUG(log, "New uri: {}", new_uri.c_str()); LOG_DEBUG(log, "Table path: {}", table_path); // set new url in configuration diff --git a/src/Storages/StorageIceberg.h b/src/Storages/StorageIceberg.h index a139a40fc29..bd20213a5d6 100644 --- a/src/Storages/StorageIceberg.h +++ b/src/Storages/StorageIceberg.h @@ -32,12 +32,13 @@ namespace DB class IcebergMetaParser { public: - IcebergMetaParser(const StorageS3Configuration & configuration_, const String & table_path_, ContextPtr context_); + IcebergMetaParser(const StorageS3::S3Configuration & configuration_, const String & table_path_, ContextPtr context_); std::vector getFiles() const; private: - StorageS3Configuration base_configuration; + static constexpr auto metadata_directory = "metadata"; + StorageS3::S3Configuration base_configuration; String table_path; ContextPtr context; @@ -45,9 +46,9 @@ private: String getNewestMetaFile() const; String getManiFestList(const String & metadata_name) const; std::vector getManifestFiles(const String & manifest_list) const; - std::vector getFilesForRead(const std::vector & manifest_files); + std::vector getFilesForRead(const std::vector & manifest_files) const; - std::shared_ptr createS3ReadBuffer(const String & key, ContextPtr context); + std::shared_ptr createS3ReadBuffer(const String & key) const; }; class StorageIceberg : public IStorage diff --git a/src/Storages/StorageS3.h b/src/Storages/StorageS3.h index 02fcb7d624c..ee52d5184f3 100644 --- a/src/Storages/StorageS3.h +++ b/src/Storages/StorageS3.h @@ -321,6 +321,7 @@ private: friend class TableFunctionS3Cluster; friend class StorageHudi; friend class StorageDeltaLake; + friend class StorageIceberg; S3Configuration s3_configuration; std::vector keys; diff --git a/src/Storages/registerStorages.cpp b/src/Storages/registerStorages.cpp index 164b125a192..2f3703fb6ea 100644 --- a/src/Storages/registerStorages.cpp +++ b/src/Storages/registerStorages.cpp @@ -35,6 +35,7 @@ void registerStorageCOS(StorageFactory & factory); void registerStorageOSS(StorageFactory & factory); void registerStorageHudi(StorageFactory & factory); void registerStorageDeltaLake(StorageFactory & factory); +void registerStorageIceberg(StorageFactory & factory); #endif #if USE_HDFS @@ -124,7 +125,8 @@ void registerStorages() registerStorageOSS(factory); registerStorageHudi(factory); registerStorageDeltaLake(factory); - #endif + registerStorageIceberg(factory); +#endif #if USE_HDFS registerStorageHDFS(factory); diff --git a/src/TableFunctions/TableFunctionIceberg.cpp b/src/TableFunctions/TableFunctionIceberg.cpp new file mode 100644 index 00000000000..7e4ad8b13be --- /dev/null +++ b/src/TableFunctions/TableFunctionIceberg.cpp @@ -0,0 +1,169 @@ +#include "config.h" + +#if USE_AWS_S3 + +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include "registerTableFunctions.h" + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + + +void TableFunctionIceberg::parseArgumentsImpl( + const String & error_message, ASTs & args, ContextPtr context, StorageS3Configuration & base_configuration) +{ + if (args.empty() || args.size() > 6) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, error_message); + + auto * header_it = StorageURL::collectHeaders(args, base_configuration.headers, context); + if (header_it != args.end()) + args.erase(header_it); + + for (auto & arg : args) + arg = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context); + + /// Size -> argument indexes + static auto size_to_args = std::map>{ + {1, {{}}}, + {2, {{"format", 1}}}, + {5, {{"access_key_id", 1}, {"secret_access_key", 2}, {"format", 3}, {"structure", 4}}}, + {6, {{"access_key_id", 1}, {"secret_access_key", 2}, {"format", 3}, {"structure", 4}, {"compression_method", 5}}}}; + + std::map args_to_idx; + /// For 4 arguments we support 2 possible variants: + /// iceberg(source, format, structure, compression_method) and iceberg(source, access_key_id, access_key_id, format) + /// We can distinguish them by looking at the 2-nd argument: check if it's a format name or not. + if (args.size() == 4) + { + auto second_arg = checkAndGetLiteralArgument(args[1], "format/access_key_id"); + if (second_arg == "auto" || FormatFactory::instance().getAllFormats().contains(second_arg)) + args_to_idx = {{"format", 1}, {"structure", 2}, {"compression_method", 3}}; + + else + args_to_idx = {{"access_key_id", 1}, {"secret_access_key", 2}, {"format", 3}}; + } + /// For 3 arguments we support 2 possible variants: + /// iceberg(source, format, structure) and iceberg(source, access_key_id, access_key_id) + /// We can distinguish them by looking at the 2-nd argument: check if it's a format name or not. + else if (args.size() == 3) + { + auto second_arg = checkAndGetLiteralArgument(args[1], "format/access_key_id"); + if (second_arg == "auto" || FormatFactory::instance().getAllFormats().contains(second_arg)) + args_to_idx = {{"format", 1}, {"structure", 2}}; + else + args_to_idx = {{"access_key_id", 1}, {"secret_access_key", 2}}; + } + else + { + args_to_idx = size_to_args[args.size()]; + } + + /// This argument is always the first + base_configuration.url = checkAndGetLiteralArgument(args[0], "url"); + + if (args_to_idx.contains("format")) + base_configuration.format = checkAndGetLiteralArgument(args[args_to_idx["format"]], "format"); + else + base_configuration.format = "Parquet"; + + if (args_to_idx.contains("structure")) + base_configuration.structure = checkAndGetLiteralArgument(args[args_to_idx["structure"]], "structure"); + + if (args_to_idx.contains("compression_method")) + base_configuration.compression_method + = checkAndGetLiteralArgument(args[args_to_idx["compression_method"]], "compression_method"); + + if (args_to_idx.contains("access_key_id")) + base_configuration.auth_settings.access_key_id + = checkAndGetLiteralArgument(args[args_to_idx["access_key_id"]], "access_key_id"); + + if (args_to_idx.contains("secret_access_key")) + base_configuration.auth_settings.secret_access_key + = checkAndGetLiteralArgument(args[args_to_idx["secret_access_key"]], "secret_access_key"); +} + +void TableFunctionIceberg::parseArguments(const ASTPtr & ast_function, ContextPtr context) +{ + /// Parse args + ASTs & args_func = ast_function->children; + + const auto message = fmt::format( + "The signature of table function {} could be the following:\n" \ + " - url\n" \ + " - url, format\n" \ + " - url, format, structure\n" \ + " - url, access_key_id, secret_access_key\n" \ + " - url, format, structure, compression_method\n" \ + " - url, access_key_id, secret_access_key, format\n" \ + " - url, access_key_id, secret_access_key, format, structure\n" \ + " - url, access_key_id, secret_access_key, format, structure, compression_method", + getName()); + + if (args_func.size() != 1) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Table function '{}' must have arguments", getName()); + + auto & args = args_func.at(0)->children; + + parseArgumentsImpl(message, args, context, configuration); +} + +ColumnsDescription TableFunctionIceberg::getActualTableStructure(ContextPtr context) const +{ + if (configuration.structure == "auto") + { + context->checkAccess(getSourceAccessType()); + return StorageIceberg::getTableStructureFromData(configuration, std::nullopt, context); + } + + return parseColumnsListFromString(configuration.structure, context); +} + +StoragePtr TableFunctionIceberg::executeImpl( + const ASTPtr & /*ast_function*/, ContextPtr context, const std::string & table_name, ColumnsDescription /*cached_columns*/) const +{ + S3::URI s3_uri(configuration.url); + + ColumnsDescription columns; + if (configuration.structure != "auto") + columns = parseColumnsListFromString(configuration.structure, context); + + StoragePtr storage = std::make_shared( + configuration, StorageID(getDatabaseName(), table_name), columns, ConstraintsDescription{}, String{}, context, std::nullopt); + + storage->startup(); + + return storage; +} + + +void registerTableFunctionIceberg(TableFunctionFactory & factory) +{ + factory.registerFunction( + {.documentation + = {R"(The table function can be used to read the Iceberg table stored on object store.)", + Documentation::Examples{{"iceberg", "SELECT * FROM iceberg(url, access_key_id, secret_access_key)"}}, + Documentation::Categories{"DataLake"}}, + .allow_readonly = false}); +} + +} + +#endif diff --git a/src/TableFunctions/TableFunctionIceberg.h b/src/TableFunctions/TableFunctionIceberg.h new file mode 100644 index 00000000000..49df80d0e10 --- /dev/null +++ b/src/TableFunctions/TableFunctionIceberg.h @@ -0,0 +1,44 @@ +#pragma once + +#include "config.h" + +#if USE_AWS_S3 + +#include +#include + + +namespace DB +{ + +class Context; +class TableFunctionS3Cluster; + +/* iceberg(source, [access_key_id, secret_access_key,] format, structure[, compression]) - creates a temporary Iceberg table on S3. + */ +class TableFunctionIceberg : public ITableFunction +{ +public: + static constexpr auto name = "iceberg"; + std::string getName() const override + { + return name; + } + +protected: + StoragePtr executeImpl( + const ASTPtr & ast_function, ContextPtr context, const std::string & table_name, ColumnsDescription cached_columns) const override; + + const char * getStorageTypeName() const override { return name; } + + ColumnsDescription getActualTableStructure(ContextPtr context) const override; + void parseArguments(const ASTPtr & ast_function, ContextPtr context) override; + + static void parseArgumentsImpl(const String & error_message, ASTs & args, ContextPtr context, StorageS3Configuration & configuration); + + StorageS3Configuration configuration; +}; + +} + +#endif diff --git a/src/TableFunctions/registerTableFunctions.cpp b/src/TableFunctions/registerTableFunctions.cpp index 95892d36b18..85c0c475bde 100644 --- a/src/TableFunctions/registerTableFunctions.cpp +++ b/src/TableFunctions/registerTableFunctions.cpp @@ -29,6 +29,7 @@ void registerTableFunctions() registerTableFunctionCOS(factory); registerTableFunctionHudi(factory); registerTableFunctionDeltaLake(factory); + registerTableFunctionIceberg(factory); registerTableFunctionOSS(factory); #endif diff --git a/src/TableFunctions/registerTableFunctions.h b/src/TableFunctions/registerTableFunctions.h index 5f91205474e..88dae5233da 100644 --- a/src/TableFunctions/registerTableFunctions.h +++ b/src/TableFunctions/registerTableFunctions.h @@ -26,6 +26,7 @@ void registerTableFunctionS3Cluster(TableFunctionFactory & factory); void registerTableFunctionCOS(TableFunctionFactory & factory); void registerTableFunctionHudi(TableFunctionFactory & factory); void registerTableFunctionDeltaLake(TableFunctionFactory & factory); +void registerTableFunctionIceberg(TableFunctionFactory & factory); void registerTableFunctionOSS(TableFunctionFactory & factory); #endif diff --git a/tests/integration/test_storage_iceberg/__init__.py b/tests/integration/test_storage_iceberg/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_storage_iceberg/taxis/data/vendor_id=1/.00000-0-c070e655-dc44-43d2-a01a-484f107210cb-00001.parquet.crc b/tests/integration/test_storage_iceberg/taxis/data/vendor_id=1/.00000-0-c070e655-dc44-43d2-a01a-484f107210cb-00001.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..4702362b4af08e1c1c6e081288add7ea0427e5fb GIT binary patch literal 24 fcmYc;N@ieSU}BK#xw=NiICh3|V~+IAi;mX-RuTx7 literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/data/vendor_id=1/.00000-3-eea1ad05-da93-4b49-aa8d-3ebbb5c69379-00001.parquet.crc b/tests/integration/test_storage_iceberg/taxis/data/vendor_id=1/.00000-3-eea1ad05-da93-4b49-aa8d-3ebbb5c69379-00001.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..eb1117c22b1db67c3f652ae5cc61d5bceb116b42 GIT binary patch literal 24 gcmYc;N@ieSU}BK%f0;0^#(dK*?aRfJ{loVH0AL9TQ~&?~ literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/data/vendor_id=1/.00000-4-c4940e8e-116f-436e-96ae-5c639106aec0-00001.parquet.crc b/tests/integration/test_storage_iceberg/taxis/data/vendor_id=1/.00000-4-c4940e8e-116f-436e-96ae-5c639106aec0-00001.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..7b202ac11c312d2a449402f4a5dffd34c8ceaa06 GIT binary patch literal 24 fcmYc;N@ieSU}89Hm)-u`M5wLrh~^%fjVe0;Uq%T3 literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/data/vendor_id=1/.00000-5-f80715f7-5243-45ea-bc9f-f087ab7c2a69-00001.parquet.crc b/tests/integration/test_storage_iceberg/taxis/data/vendor_id=1/.00000-5-f80715f7-5243-45ea-bc9f-f087ab7c2a69-00001.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..b5ac1ae17d96c97e231bf742361c05022be65ae8 GIT binary patch literal 24 gcmYc;N@ieSU}A8ynYHcr_49EvLuMSY*{HGu0BEub9smFU literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/data/vendor_id=1/00000-0-c070e655-dc44-43d2-a01a-484f107210cb-00001.parquet b/tests/integration/test_storage_iceberg/taxis/data/vendor_id=1/00000-0-c070e655-dc44-43d2-a01a-484f107210cb-00001.parquet new file mode 100644 index 0000000000000000000000000000000000000000..08cc624f30c081cdea1eda7d7220e9685dbc73d2 GIT binary patch literal 1565 zcma)7T}TvB6uvt>;WTts)L=ZPQ6lgc?s6g*(D%VlCtB1QL|w(UjtY{7K7d0k)JFNyHyJLrOi z8<~CnH`gNvAn6FR%GwXO+@4NJ^z;^h0Ey${Acv#?ne|}3U`}z4Iqt<~0bTamC~%WU zN-QOjq}UHrtU}_ZlQgBJ&3;4c;cTSTGK~zo=V4Y5u{D3M8Cgq7skFz3upiLPx?<9P zWjLKnSv>;6TL7!bT4@HRw0`{*e3Qyx6^i)-0VtF3Qb3A_;?QKNO5i1zFy;wQhRBR3 z6b}i3mDa9U1XVi?UqO)+&Tk*o%{CIYjePatO7fn^ttd3>v87-_#V5)ObrKzXEuv7I zizpO_#bnl^&|%1r^7()JFd>&qSpF$4fCYFI$iKQnd6iaBnc`aP5wqXL{g6FBaQn`k z^?WI0eO-bhC^Qc1>$kJIJ6x#T1V2<>+zruL7*_dO)U9!YX-xQ7<;$S4#nxD7VtJ`U zQ2w&Fr4dzodPPsCJJCIs=#dZ0h)j419}>tbzLKq>xXn;$f|`SotU5%874aiCDqE*B zRBo2DmYGv6IV{_*8bgztYs0c`(4>}?oBKv(hD5`1N*QMH&Qb!!q$4@Qq~M1nG+a!X zOs59rv9QgobC`wY{ZC%~YF=^4H)iX}v|>534G!IP8a5_t=^UIqhaGd+YtPNbNuZSm zd-4-wNIaTR%#j?mYKG0=Nor<|bSfB)MC&5;!RkcLNNPd7NmHp7ZK_in)To}&sM@Hi jiTY?$ELOia5mRcTw2^9!i9J#rPO1BYYTymf1uurby3t{7 literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/data/vendor_id=1/00000-3-eea1ad05-da93-4b49-aa8d-3ebbb5c69379-00001.parquet b/tests/integration/test_storage_iceberg/taxis/data/vendor_id=1/00000-3-eea1ad05-da93-4b49-aa8d-3ebbb5c69379-00001.parquet new file mode 100644 index 0000000000000000000000000000000000000000..984f59f9b0a24de6fd2e0a9ee4af73ba2ae6af65 GIT binary patch literal 1603 zcma)7U1-x#6u!-}F4n&}JRy;QHDH-gi`~{a#?*-lGGPp14{CKylbadn(k@LWGBy+u zM8O9iL{L<8415vAAt(x>q7MrCAP#is!(dN7i6{z!=Qe2;nG{2R&i&5$&Uf!Q=aPY) z`_sTfGo;|>*VA(iJ-`7^$og{v@gChcLCBBKXKpb72u60nw|B2+mn%r)Aqt3xqVu_2 z&NOpTB47G_+ovR9H_UvWztzAfINOQ}N0q3P6fUpEfDaQW2B zw~;z@lfN(-S6fKqx_GG>Rpm9DEC~^`*b_2=lQFI5zPD?2&3)fDLvG$Q7k0=J{zZ3& zU}37XEjD{I)r(0-nbo|fH{_1=%TnZk4=7?5Om+%c5uZlD@y!`7FsB<}(nqIwu$hV6 z!rn$#%NHob2=E&@Y1tUHjiNJT8H1dMG`WsdVr!m&fDtRJYr`tLnyu5pD1jKF%T3Lp zL)u8OV!ML^ffb0gl-(i&ZDYthhTnoVypd#Lih|N2FDr63k;OvG8G(<=A!Xo2UUbEW zkwi8jD2ugSv_#SxRD2O7a3jE6xmWf$N?$RAI|iZW%?Bbm55IoL_Q&iXB*eDS(H{;I?YpOI91k<(h*IX=f+j{ zc!{drYT0!vx~nEt&$Zby)b1@w)wF2AD68Fv##M%-ld7$au=rpyk>b$NisexBn_8hv zleL_MqCKomBt2?dK=nuXe~ImX6Kk_xhi(>%ni~jr1pLsgsKYjj6}*B0JrmHEUZ6Tz zu$~q>{ox5B^_4Vdv_jp@BTjsbI%TV9$J42FTdFX{>V#R_m3Sj;uahyIH)T4CmxQ#(xuw|#2;U$ z58uItH|^R}d>a|`J-5cz0s#qk>};)Vx9*!2i%y9I5{SOwm>!)rXRD2hKP3^^ zf!J`}uQIS=Pdo49w`|VrCfS6h;WcGR)zpGf&_GbLk|3%PWe{XZ_T?L#Q7|NJGq;*l}tr-*R zMf-@IO@J(4l9*T!!)faTwl!fb$$6$2#Eiw5)~-7H4GyAGs|VVvJ_=tlg})?76XW`T z28oVBH2K|-e&=wDaaX{Qy1k=>4aGw0Mf(U|V}hT>Sx6hA20J=}eJb11{gB1AAsS1f zHc3XgQwn#y&!Nlg(7SQau|8O8Ly{j1j7*as^|5( z?=>u6Pw8#f9@p0MdsDjO(z0FG^QRYdhGbHD#hhjP!#+leN9P)@N73(?J)RsbHQchDbaJ*u?M!aWny@lX$)?t%WtGM= hxomd)%~ICfo1v4`o-Dng7I3~7W&tUuU7sbIyj4`S?QGIM7g%qH2% zWJ|j&^ud;Z54JBY_@E2*VGFIaSVT%eEOaTILVJM+fsj|villv+)<=Xqy5IPU0YGqN7A`g( zo)=X#1&9FRfz`!AAzrZJg#7mZ>C-KW012f0{QEzbqkECCA^5$2Z{8;Sy~T0z?&{Jk zvY5gU(V&-0=>eN_ntRjwq_M$iu0Q*$nySwXkolMcO8EOK*ieqgbL4 z!@lq2<>(wOTNQW8u_pO8GT^yRl|2gtL|pMw>)CQ@o>4aGWQ0Hr(Ji*&(kWx6QZIXx zB7s4~hH74gfn{sTejmRjV|pja#8efxDa934$?JI)1SKO15hb7uf)tlL=?6~F>!P|| z+jUDKBYwr#QKBT-ECFtu4Ka@@t5F!1ngg(p=!^WiRyU`X(kGm>%%NNVi9@k&dt|ZO zGbYxH_7OW114+6hGO=bz1+kOZ#)7pZ=a^zMq&GrpyX)-U+>c7F8E8NCQ8>>O9*U46 z`n9kMv5rEp_}!H5aJWRjC16V3*^&K$Vkz~aeFVQ{f}ciNN^AHBGT0I9)7h5phb*tP zh`tz6C&&nQT;`7S3A)UJ-im^T{lQl2qV(s$Fq>R?t7Fn4b*ED`^Aw#i&tkaSWj8#8RZKgURM>1^F`Df(@rRHMmm z%u=O1tt}+m)KQ<>8Te-KE3z2Z4w3R z>9FiiE)erTl^*jdY@RysRsj+<$_Xi;;* kG>c>DY$h|drCW@~qc|6_0525&qI*PBwC!klAMgRZ+ literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/data/vendor_id=2/.00000-0-c070e655-dc44-43d2-a01a-484f107210cb-00002.parquet.crc b/tests/integration/test_storage_iceberg/taxis/data/vendor_id=2/.00000-0-c070e655-dc44-43d2-a01a-484f107210cb-00002.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..32777a2ffb037420b75b082368f2f320b9cd71ed GIT binary patch literal 24 gcmYc;N@ieSU}AWm(V+gw>Xm5k?d#98zy3W50Bn8>pa1{> literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/data/vendor_id=2/.00000-1-3a214094-2325-48e0-b560-d28fc767f054-00001.parquet.crc b/tests/integration/test_storage_iceberg/taxis/data/vendor_id=2/.00000-1-3a214094-2325-48e0-b560-d28fc767f054-00001.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..c296368d00362fe1ce3f8c3479a472714e18edc6 GIT binary patch literal 20 ccmYc;N@ieSU}A7QGDq#}>G=_}Z7|PQGS^|p-wXEwpOC57EMR72uFc4SS@^iCAx^||iB2!So z4HWdrw;?Fviy)#-eDX;_VK5cZF=Y>P`Xa&}eDcYYq|Kt+Kubz*zH`3w-Fv=sY2W^1 zDa_(VoWyrtznft@FoRiywqN6r_tDgg2sO<`cQ^n>kek5Y7T-RPim+rchXD^{57nWA z-EZa?FFB60`193Y^BbIG^mp>7#hF>e4BT!*FB=w^Mvu}6l<6cuh)APngN1CqI0=w0G(FO%aI`U!S;IERNw;Jh?FavtjkA z^`W8VFv>lWPeuOs+`<+{;5nxQ&L^ivv|L~Y9)bKY5S6f!os@48vGm7+%b>ky6sy3tT9I;f16 zDn+}WL%1HWt!2C9fJJ@KI1Ar`GTe+ZF;RqV5>^m}yqpIQg=UTw0>YRjEEypWLULZ_ z#Ff^rtO(laH+%&}Lc*O<*tavc;0Spb#ZBa11a~6&X~>fThiWHsg?fnru@0WA`NDHG zvmBY#D0CR=Be(vGC?;gdtv|(xn1iE0{XaY8s&sG`wxiKo` zq_S;QG+RnYUTC+eOSzVWWSF#|m!;gmxa1(Igj7^UoqT^vfnw3q71N>+haxmlD_JNN zhouV%kJ;)otJD0Cy!4;E+SISlHVP%h_GR0Adfh6=d;kCd literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/data/vendor_id=2/00000-1-3a214094-2325-48e0-b560-d28fc767f054-00001.parquet b/tests/integration/test_storage_iceberg/taxis/data/vendor_id=2/00000-1-3a214094-2325-48e0-b560-d28fc767f054-00001.parquet new file mode 100644 index 0000000000000000000000000000000000000000..176c0ab5f655079968b88d1f66d2845c2eac2f9f GIT binary patch literal 1501 zcma)6O=uHQ5PsQg(rs+5mA*$777PoG71GcoO$?z{rI%u%ltNFfZT9DFUCCcJn~Id8 zC#`rBycF?JMLcT7Lp^yYf>=>hP!Nj;6%Qif&sk@)yUCWO;IcpSX6E~5-pst6I60ZX z2p_<4y!Q0f7As%|mK^2~I&>9Y^-Ga2BGmJ)?Xab&dnfUm&mSJOxAcDhZR-VMknsBZ zhU3rS0sMFM#`VsY{$pcHS{Dj#bU$ZZ`YitQ>&kjZOZVx?{IlBf{hNL?^7z826Cs3? z`11C{-(4*siki>=@T1&%^lSh^@ZAg_<`|Hy%OhfjC$9OyNSAANMUR=)v@gf`aQ)*^ zbGIwULT_)yPXMVadd$9YKI|KJ6awM!41`0xKvbM}q)i{5b;LCWYmT_$w;cdl7!8<` zT%b6FFx!j5?TfUemW?@GonmZFn`!73Yt2F&s>ZqKS1`;$w)r@?EpMmq~z0qO#kxsym F{U3aUP;3AI literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/data/vendor_id=2/00000-3-eea1ad05-da93-4b49-aa8d-3ebbb5c69379-00002.parquet b/tests/integration/test_storage_iceberg/taxis/data/vendor_id=2/00000-3-eea1ad05-da93-4b49-aa8d-3ebbb5c69379-00002.parquet new file mode 100644 index 0000000000000000000000000000000000000000..8ae414f7258ae462247a33afd157daecfa03a2db GIT binary patch literal 1677 zcmbVNU1$_n6uvu|os6p{R&p*gGT3#f(+Ha-o47HCrM38CRjds}5Lb3*XYRz2&2F5X z^)CbzDJ>PGg(B34Abs)&1vP<^haeI0r=lQz=z~~k`yjN1J_PE651#qi%}Bx`4ztX? z-#Op;&OPT`cJF7y8Q@_Jq~WIrPap9);D9IO!?Pj@9;0-Mkgng_KlT71=s5`2uU>o6 zp@4uy0T7RR7mGz}v?!3fy$hE^N)C3y%WL=l@AN2SdCKqqJbFU70~Z(eb+0BpJH+qT z2XY_5Za95r>R+w}_lsQ<)|1;8Uwn7|ywyqW-bokM2IL$R;qlB5zpiXSE({lcsZZXR z+P+S_^yAj)xsGQW?p^td5NFf=vwjXdB%lwzem=9%xwJTDvG`vb8M>(*j=x`d#AZm{7 zk)_z4&|jf>uuzW33Z0L^@o+iI!O3tr-3I4k;rc>6a7J=@N8Hs*Wd;cX{6fdrx}E!|-SeXLrqxFaHgRfw&vxm6FW zm<8(?{>u93Mv_e^3g$@#S&{QvUIAXtio764l!2!Lb?IzO%WI;tSlq=GCEd-EFQP9w7h%YcD z0;O}lz$}iqh==h6-Y$|aGfYovvs^A{v*h<2!~0DScG0lYUf9RgB`uqy1tGJW&KDs(P}j@mr0;N;oSLUm7!Gj&LP@MZ&B|s?9Xc?fdPpXvR`fB?zd5T&aoCZ%?J%@kdb!3@jhN+XWmKI^1=PMK zH5}@9CHB8fY)pR(x>c^~Zc})3lOMS!E!bwYj+e4Y&o=2xuT+aDSWnwK{lQ5h?Ht#g zBX#EXj5*1p%&FPcN-~qq^ricg-KDx+Hj~z%X)rT0*k^1uGFHiC#(-gz`ZI&sZ2y*0 bR`1QQ0cH-AHpzLs!O@p|j$^zR|A^iIYG-+d literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/data/vendor_id=2/00000-4-c4940e8e-116f-436e-96ae-5c639106aec0-00002.parquet b/tests/integration/test_storage_iceberg/taxis/data/vendor_id=2/00000-4-c4940e8e-116f-436e-96ae-5c639106aec0-00002.parquet new file mode 100644 index 0000000000000000000000000000000000000000..bbd7c8ec41f51109fb1a5386b314c386a5257be5 GIT binary patch literal 1841 zcmbtVTWH&66#ga4wyZhJHhhExVl{$l8FjrluCtJpE@LlchAthX6zUS|@+(tO>_nD3 zMkr*grR>5s2D`l!)`vdr0?j(g9`aDepj+um8Es%?BwOhow!t1sp7yXK*;W*6!dQij zkIp&Y`E<_tetY`4mrB6FBT$6%S8rb9_5%wXA-mq86uy2 zn|n&?kmO)LocU<=;?Pbc#*qz<7~gLgLIC1Gf+^T|W98mZAB|a`UN~5-uDn=XvWLmx zmBnQC0eD8?2z<5r=DX=Wo@zD0R?25@-G1$vyGK7zKFqCo$7etLoe*J{x#vMz6JQL^ z{&MO5aQ}L&dj0PNnY^f-NdCVC$&a$K#Ipzvu9yj|ETSU}uRK583t#G*u`p-b_Or5t zy2($)-29n84qtvesZL>0%&={3JRW8a$r5v@o3Ds9u#t|)ig}I9RN!qZQ-mGKIupLj@_nd{`nIphs#!&cdIW1prm~h%;deawj0 z84PPz7$obhN8)=s3&{Gt1IW$`Ac|K)vdx47e7CXf0}Cma8Sw_AZ6>t$5=v+7Qm6r> z3PsR)8bEP5pjgg8k!W|43J6__`1E@{-D5CCTj#K+o_M54Q9)1r-T{Qy1Hvyd!2r*t z@zy+B4}6l{}ou?4(uI}9pJXx)cuHfUxbexr5(^# zvyJCAf>$DV|Eb%d3eM9Bx<7u3NG~kt-eQ~jd*{9UN$RznW+PuJmd1+Z{D{$ZTq|!+ ySSGbf6JzE+vt%0NGmGaX@MISBEacYelPs%ksd@x(G^K&PTPG-~D z*@%RIi2q{9gA@Z&f=G?h7h4OZV3E8OG13zHP-szUg$5Myp%lrJXJ&UN6LtwwhgtUA zbIy0Zd+zxzbM%>GMG)WtD8Ls#uCEDC0S^KpkA5tY#$y_95z=?PYhUO9f|(=m_1X0W zP5}vl89;wByHZ)KSc`y4Awuq^GVN6B*qI%U#=4J^Tlh+Jt0iM4UNI=n+x|cZA6C;&)HXgSpIGCkC4cYE7sWK}AEr^x78?X~p%O-X)`mnD%$aQGxl;AM)AD17cvZ!3JhX(qyg zZQD=F5{f4`mT-&j{_*04N7CvT7R3zP)`pW|?uaaLN1FKxErX48L{{i&9=xWxnTpL@ z>DXN9YUCo;eJ&m7hRYy}D%i_Q-LuTIs{W*7jq?ppSKvD{;aVUdV#e-hZ8^0-_e?gP zA&^DyE?f86q&{7(d%?I!pa*>q)q-m1^sGty4g9(K)Lv4`Dhh6sO0puCwXy<&ToQ$h zoL~llN;IGsIjyXT%GTL#S(Nn0C%)w+avqf|0B$^#G>(LRmW4jLAi_bSo#wZ=T0KQf zpL2Ab!?3o>Az5oYlGvJANY?J{Kz1ezl&(c&n<)jvW@FnAETVjeqt`fXGo^i+QW|5I zA`Ku_B!b4O1I21c@liJvh;}QjK(`-|GgI&{JtDX(D}3>5qsYF=%G_B)n3I*?awhq+cw9i5m{Lr*cUdir#j zAIljbK08r&e1>sbcWW%)s_9m}DRn;IfDXpcrtbeMaA-TQbwPH3+iq14V&VfaK5>nis9|IZigy(o=({P@p&RWKco96>MYnl?dN8hUvsKnu2?7z7KU>DM%{6( zoIPTh%qoryng`6HZCK14HcexwI8rJNJz*{X+IVA?$ literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/data/vendor_id=3/.00000-4-c4940e8e-116f-436e-96ae-5c639106aec0-00003.parquet.crc b/tests/integration/test_storage_iceberg/taxis/data/vendor_id=3/.00000-4-c4940e8e-116f-436e-96ae-5c639106aec0-00003.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..ac23a9742438c1a58a412bceec077302bcaff5e6 GIT binary patch literal 24 fcmYc;N@ieSU}Bi}JW1n5j%<3d{h!LZU;F%8|MiJ_FDcqtZ2DfDD*v%BwYUCCEBn~Icz zNdLf? z5k7^p_}-Jv_i*#G2(XjV1Xd_shaQ4(_T1ri0GhGNL06nfNCYx8P^;XTA;Sufu>`c?D z+hC2(>$l-oQ5O19At8#8CE)~7m`(E{jtTudRIMEt#|THp2q;I}Wo=6Y4Ftt*qi}4P zBZN7i{vF4ClM z%z?$I+g0}s>`sr}U6}Q87bfIu3};b#gA;MWWer!=OI!s#WbRK!Te34hjD}-8vtz8s z?aWuBBJOcxbm)>3szl0QHVP9450E?GFx{3-w-d(_B2PKOrFpU!?X%}EdD76B%FKmq zQ@hEkN^+lDmaIDsCXGl<%WP?ul#yK3ph2aPp^T&(tU{a8$n|B(M&yiCQ>wOoFpl72 zvc;BRGSCAF%lMR)dTl{k$+(`mz|$Ms|Mbfr^z+C2Xm7nzSF9lHaNr)g2S+>8dJFDc z;9Us358s%hNuZR5eexSCD0!`+n2RlD4OY$65;L1dy_S-*axR-s4X7=nLQ{H)YK+RI soHne6*5R(iyX|U|49ZkcMv3#1cx;DpHD+BJ|WsvmdkRmd!^un~Ic5 z58^>DMLZOb{sldVHxZ;Det=Mmhw7oUdhp`GgWyTuZgw|YD#2m1`{up*{eCm=y~)g# zn-WG?z(xG}^ZsF6zzj@dm_z9NLwLDERZ69juIcLty}@MSBsg8nIDWACF53a}<>37X z#GvbEH_myWemsVseBIvRI-s=DOdX-}{f{r>ZKd6v-`^q~KsTb1e6Aby{P=#slOk}Y z-JdW1^mRaK?4!FVdvA{RfD(V2;S*dGEN6sF%<#kyKf8L$5Bp{)%p#P$%qQGW$j#}_ zNEILyn4xfFnomTgJ%>Ozya;lL7f2m1dDdPeFt-_Odfu&QYbij@$u3(~Op4v|(f&jZD3@`BWDX1sQ=Q$MPL&}&X44EQ}DQ>Kt*S1B_V36&$ z7tRgMg)jr$Ph+^BY_fP9rRr?~{~#{?Bw0$;f!;4h+j8iSd&Bh49r{l${T}od=N;f^ zVqBF8P}z%&wF=vM1HD`3)+|i>Hiij#+=VkJwao}P?$L%b>t)7-1+aI&JKC1B_v7BM z$i5@;s>?~uR#d>fp2$B+g1grwJ1$ z4V5a?DrXw%BCW`YLuOU9@6@SS6dShHRBbUWwn9UOCKiX&qHa)AYly|URnhTCX|XC- zob^FZ0>q-rO~axP*JZOo(>|wJtCq#JbPJjdpkDL-FEDo$=y!PsxNg>DI}jcT@S*$A z0j<@VaQOmwK7b#;IvrRbpN4b#TWcsWRhO;hCbfquR&s?}4Wm{~N*O7e$t4GsreSJH yeN&{yuRicJKkb`S$h4uBPB0K5xH46bigQ^R)f&4DQEE-#71un!+o&@fm~? z>--j^)b{2F@r!r6Tg}l&NBvYf6+$DYChoVt`v?^0&Zp=5&BamFT<(`24X=vV1MO{} zJ5F&?#t&qBcuY)jEC7!C*`=lOEYwk^C)AQrCEzz_la4N;TyGWSS;#wE8aj67k91;Y#g?ZDNuP8#@+0C z2%ksd3f0i6=TuYL^nq90@_|AN6}z6eBzE@ zT=etRL$AI;$!US=tcAi)TVajC2_!BDcpRxIcxzwF=%o)$DK937U$~77FZ}`B`E$0@X(m>fROWT5qiZA{V0qD#IC_{iwwIO>bAQ# zdQ?}4Ow8#-Nx4SyQuGToFIYEch%hXaEVHawLR_f2`gBzo?vD$aP7Jjq3}2ZSY$O>M z3R2!)@AfiKOfp;6O#*&RGD;*~k7*PO)51c$N*!>iwWj}HV(K8Vemf4KYerGBT;V~N z@3}{Zu+?H2F0D&XyY%KOcSsbdr*3C|bpb^$&q(HMnOJ>!Gd4%el3pytl8NL%A{FbE z%etY)v>{a?YI0~m8B~&*tP&-oC~_(}luoD4%4w-TNisyu$fvn1+;H!Mbm;4G_!ImG Deo=!g literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/data/vendor_id=3/00000-5-f80715f7-5243-45ea-bc9f-f087ab7c2a69-00003.parquet b/tests/integration/test_storage_iceberg/taxis/data/vendor_id=3/00000-5-f80715f7-5243-45ea-bc9f-f087ab7c2a69-00003.parquet new file mode 100644 index 0000000000000000000000000000000000000000..f567383114495475dbe19895e47f8dc4d8f550a0 GIT binary patch literal 1801 zcmbVNPiWI%6#ts0ZCW>{vM)3cutXLnc4ckXI);eiWjGjPu#=TFO}^Af`e$i6kujJc zC>wa(Vc^A!CofYcj4@<*5D^3!bN<1D(|HiYg9i^Dd`a6hqh&}#n!Nnp_j|wZz3)9b zad9$^X?zUF@U8p1-;SQc6s8e6xx}E_C94YvE$%-0VF4Jy%3-{?zW1t|#|#kt7;x9> zTsE8C&u9H;Ui$vfrdYufc=PScPdzQcKYn@r22sfO;mqUC!!vjgFYRyM4YY)pvlC^6 zV(a`Cq}1`|hw!ua@3vZ_kBxb$R5E}@PfgzIeD@(J&YjOsf43G#RH*6oYUVU?(bO$pVYYw0Z-6x|HujsSwY@ zHd%(_xr~_MG0mkI+RwR^G0ifp$*z0EjL7iK+BPi~^*a@BqKJ#au+SJb4qJy8VqXMt zFS{PV=aINVHMJT!)s;3q;1#z#pw&2?re$fhqHVNxgP3Li*jhVY7~FbZ2kT`k>g6>lO%X;0qk^gtU8x8oSLOr@ ziARN^RIt`Ny$lqC%v3dlfM1pL3W+vi>ZM{{n2*+|35Qy5`u`;+4-y;GaR^=2OOojb z4>^3-938?|N>vzIhn{lits{3x6sV_WWq)-Zg)f&SW2Q>Xfr1g4B}PRn6(jLjJP}Jq z`g2uHS0d`LA`>M(oREj)xSCUloR;NWGCrJ2CC}zk(qNpViIUEp<}xth?i*>)*Awt3 F_z&$MgV_K8 literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/data/vendor_id=4/.00000-1-3a214094-2325-48e0-b560-d28fc767f054-00003.parquet.crc b/tests/integration/test_storage_iceberg/taxis/data/vendor_id=4/.00000-1-3a214094-2325-48e0-b560-d28fc767f054-00003.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..8aead16947eb88f667d79a7c247f5f0bdaf8319b GIT binary patch literal 20 bcmYc;N@ieSU}8A_!##B8lx)Vdfc!cDK~@Ix literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/data/vendor_id=4/.00000-3-eea1ad05-da93-4b49-aa8d-3ebbb5c69379-00004.parquet.crc b/tests/integration/test_storage_iceberg/taxis/data/vendor_id=4/.00000-3-eea1ad05-da93-4b49-aa8d-3ebbb5c69379-00004.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..054f4b7746697e03f13710458a0dc5cf51ea6287 GIT binary patch literal 24 fcmYc;N@ieSU}A`Fm+`wN(|1v0ksDXI7b7PCQKbf@ literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/data/vendor_id=4/.00000-4-c4940e8e-116f-436e-96ae-5c639106aec0-00004.parquet.crc b/tests/integration/test_storage_iceberg/taxis/data/vendor_id=4/.00000-4-c4940e8e-116f-436e-96ae-5c639106aec0-00004.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..f01abccb82ecdf348f5d49d461a08a0cbb454c87 GIT binary patch literal 24 gcmYc;N@ieSU}9Ji`!B*f;l#oNHg;)2A*w;u;aIN?Y0B|Y_s{jB1 literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/data/vendor_id=4/00000-1-3a214094-2325-48e0-b560-d28fc767f054-00003.parquet b/tests/integration/test_storage_iceberg/taxis/data/vendor_id=4/00000-1-3a214094-2325-48e0-b560-d28fc767f054-00003.parquet new file mode 100644 index 0000000000000000000000000000000000000000..9887f3ca5db181b853364d9e271437272dadf08d GIT binary patch literal 1500 zcma)6O=uHQ5PsQg+BH_I(zj$GYFucnkcMv3#1KMNdXW}O6#`OgoBes)uI8uRY${TU z3R+P?i$zdG=s`UyD20Nz;zba}tA&alJm{qd@!&;g|B@|D!DWBu&CK`ByqS4BeBz9N z5#EF2`1Z4xZDCBoiopy*2k*ekx@7SxLW`d!znTimIf3sjzS;0M^Nf-HJ{fy8oMY=`;At>XS!QQ}>H=`&at!=HL0yz}nfrPJ|#% z;G65qzuKEZWTjBp@T2Uj=!F1;;Pz5nh%tkimq)}D$GYa_{&v^wiXJnqs_HQ=WPLnp z=El#{D+dDnaUgX?kJ&fGg?vMfBFr*)1i~RM%u2Z6NUJ_P=7>uaRvmHPZ#w|Acrsv! zQjy>w!gMDJ?U*5DrJ|2(N}jSct%j~mnQIyv2_CzwwW3uGu`H8(3&KIr?o>scjEkj8 zt!(5Ogxf&3uWD3GWmy?lFTqn3Cpu9o7!E_0EX9Sn(I^wfG}p;M)f#~@%~EWR9f>lz zMp+vYi@NH?ZfIfKFi8kgK)n&b9qe~M?n6R**D11|B)PtM(h%tsdJeFYCh=%(CBT97oY57}TK6>aD7ixC(a2 z+#QF8WM;lq>y7ckjIj_fGoOdE)@H|O(j_ZY25L0#&%uPj4dm9>OZV2K`w+l9VxLni z6=m3Ue}_4Lq1~EHB%)8mtMVi%iJ^6BmNza>5kAXT4ZS8Ce2lkE-I~H@dt$t*k)l%N zv*%`cQzOLqvRE?x>*ELzolMs>oq!!kNX(@yR>~9nT+Bu$>PTm7{}Y$ojC03(d$d}t zh({N)66j9i6J9x1N~oNuZRPx%2aLC^Rr7>eDr1beHtV4AHAvr5q9B zLL#1wbV)U>s6^C^A`?Z(B;;OMP$h-PX<3$%LMD|;9+gsJk3iByNlQn#Q8=I88y$v! J+c5ms{{fjdOP~M% literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/data/vendor_id=4/00000-3-eea1ad05-da93-4b49-aa8d-3ebbb5c69379-00004.parquet b/tests/integration/test_storage_iceberg/taxis/data/vendor_id=4/00000-3-eea1ad05-da93-4b49-aa8d-3ebbb5c69379-00004.parquet new file mode 100644 index 0000000000000000000000000000000000000000..420c11a3d860d523e5277a8c250f74e23515945d GIT binary patch literal 1539 zcma)6U1-x#6u!;UF6(|o*}XQ9X$h=M=*H5nbCfcFwuiCFn8JM7#+D{GTSoe`G@Zzp zOi&O9KKP;!B0l>nLP7L#pbREDM4|XHA4K02d{j@;q|KNVLw?RZ=YHRJ?>XmY=E4;T zBP`%F-aUA5mlZGr!w}{WI=u}q=TfpuOG}z6Zz1$nct6nvP6soJUwo=O?r4Ike%swc z4ElI7{lP_b;|#vJy|vfX1a)a@>RBsddlt6Cp42$5{@nl7-UMW9_Eg61&AtwxHga&) zRU@#k{U0y?bT$F1?EM=kachpYgBX9D;UgRimVNn5%<#kuKOZ~hg+0?3W;IPa$48ux z&&}Qbez|fgD4qvW&-8@@(|jZ_?K%X);T4cWyg-WBaII#CR@jEkC14Yeeu z=Qc#!BgMtMoU`ZGdlDcfT`TJ*g}5dgB^viQjY57|+>F_1{Q> KzuXM`{{I3&_F3!z literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/data/vendor_id=4/00000-4-c4940e8e-116f-436e-96ae-5c639106aec0-00004.parquet b/tests/integration/test_storage_iceberg/taxis/data/vendor_id=4/00000-4-c4940e8e-116f-436e-96ae-5c639106aec0-00004.parquet new file mode 100644 index 0000000000000000000000000000000000000000..a947586bac735d5ddd6df3be5786235aa1132c04 GIT binary patch literal 1802 zcmbVNU1-x#6uwQf3U0K_8mLdMI7uj%3VIXs|rpe6~&Dt$ZCo+cm zprV3`Lzt*<`l_HC3>_jOf-n$aOxQpX!3TZW!|-9?^dxQ5jFuq_X>;-!Htd9Hn#S*3O>XZx@Ua){&EDatVyypB2LbIe75-UP0G%x6j|o!|h%bR!k49wTWbV+?3Y zFO^5COhX&7-f4sxSh3B1SG24trDf9VM>qgzjVc*rKpM{E(q=D%a4lfFvS!8t)5?H) z9zMD>*ofi*o`-F+G{k<1zhA-8&WU;8psdx!RT*w|6 z*f?w*Q=sy`8rQRPRrnARr>K%vF(+HeWM6y0E57l7R`GO7mZepWR?u3i#w@#LYb|(r z%oOt4ST9>qXXiZPf>&Ip`23;gzd^}vf&5zqg%wL-mce1j#>eA8LBU=7LPqZ(BcJ$^ zhK!1xQ?jg#%FzmXR|BB;gR!c&=;v{5L9fEO+Olb@kc)nC-N!>?!VN<9vqI>HR_Mpo zSU~JF9Jk1@(^d6W_Xf9XGD#6*Fr1Z#$gmXrPK^uZg%KjO3t7|1$)*qz@~&o05!zcq zf~pZ+$qMbK#{~hN82v>QDd(Ht?GhmD3Y zVq~>Ux*-ydM8mO$rc_STl?FAT$V7=GqH?PoQBw+$+hjQvizMRl*nw1BYKf3GqO_&< SbNw*k?i*?F^VkFbg8u+JT#qjR literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/data/vendor_id=4/00000-5-f80715f7-5243-45ea-bc9f-f087ab7c2a69-00004.parquet b/tests/integration/test_storage_iceberg/taxis/data/vendor_id=4/00000-5-f80715f7-5243-45ea-bc9f-f087ab7c2a69-00004.parquet new file mode 100644 index 0000000000000000000000000000000000000000..7471ddfd1d07deecb3d8e22b22ab36cf700b51fa GIT binary patch literal 1802 zcmbVNT}TvB6uvtfhjVLb8ldDUS|VH%v(*O}inB?_{VNUV8L z+Y9l$Z+i)FyD2WnSlL^T%fuAN0${sm+G`5#g6VS8s;VC4g7)BonQOmJ&F=FF#~^6I zbh$lUT+q{%2Y9v@t00Fso}Ho`bHVGFQ$ELB^yf_A>L+}rl++0hAWYX&!OBsRQqo4h zru0xbq{1|`VQWq!%)p9jd#-31Q%cFC$B%FT&>gB|kbY?>olTiN48m1_?a7#F3rs2f z>P7hI(m*|m1$Z8|$ub8hswnd9Y4Nk?2DB?oU zE;NRX!`5Ly?E6Yw%g$HeBS@U0idu!7Y$cO@?E$a&#sgY~(6mclH9!;p=S$AP?pyY~5v-a$q_ zagK(J3OA={Ss9h0W%RBGKyQ_?s<-IpaaCTg%(>dKDXWl+esROc!^MObgzT|Q=toxQ zCzV)0>@*y=$gtBDwbtzo?$l(GB*s8EBM*`xDY!z73Ff6?BD4t^)5yxE5E62(Mokgg znnHrA5nag$ZD+>>3yFk;lr&_mckVJ!3^J0{3<7>t(laDfh^ePj1HyPHM~ynv{H6b2 zV)JHVp*yyqt9n{89pM&-@9Lv1*h)GJP3zEO4!yMHwul1t)U52!jiX@4uw;y6iP<=0 z)Qu7&qoq@Ik#Hm$Zmw%cW;I=@Q{#$Elt?@(x5yDSsSvqUmXpnqcr4a@C>fKQBBYfl Ut;vI2A9T3;MH>7(cEi8mKdRY}+yDRo literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/data/vendor_id=5/.00000-1-3a214094-2325-48e0-b560-d28fc767f054-00004.parquet.crc b/tests/integration/test_storage_iceberg/taxis/data/vendor_id=5/.00000-1-3a214094-2325-48e0-b560-d28fc767f054-00004.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..e19f70c59e794d87ed5186e8856044ae026a646e GIT binary patch literal 20 bcmYc;N@ieSU}D(ic&?~aqHtqcKz>s`t0Cm&~>i_@% literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/data/vendor_id=5/.00000-4-c4940e8e-116f-436e-96ae-5c639106aec0-00005.parquet.crc b/tests/integration/test_storage_iceberg/taxis/data/vendor_id=5/.00000-4-c4940e8e-116f-436e-96ae-5c639106aec0-00005.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..d4ec50e35e13adfac9fe62425b0194ad221306bc GIT binary patch literal 24 fcmYc;N@ieSU}9)nRky@g?V7XZ8AtZ5IkJlYS_lXn literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/data/vendor_id=5/.00000-5-f80715f7-5243-45ea-bc9f-f087ab7c2a69-00005.parquet.crc b/tests/integration/test_storage_iceberg/taxis/data/vendor_id=5/.00000-5-f80715f7-5243-45ea-bc9f-f087ab7c2a69-00005.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..3c8e61e22fc38637078c11327cd389e6d8a2ce26 GIT binary patch literal 24 fcmYc;N@ieSU}6w5kG!{AFTbLkRb$829N9$xQDg_l literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/data/vendor_id=5/00000-1-3a214094-2325-48e0-b560-d28fc767f054-00004.parquet b/tests/integration/test_storage_iceberg/taxis/data/vendor_id=5/00000-1-3a214094-2325-48e0-b560-d28fc767f054-00004.parquet new file mode 100644 index 0000000000000000000000000000000000000000..ae416be428e2c6f2fcba8f5862c7a9bfcb882832 GIT binary patch literal 1500 zcma)6O=uHA6rOB0aU0uOr88t9U|4Ldn1&{4(okZth#>Y-go4G|X0tP?3;Ag`o2rz8 zhf)MDdh_C?T0sy&5D^a|wntG~P!D1cLJOs!_Ryo?+rMN>Q*hazH{W~T_h#O_*^!|u zag1;Wj^Ss!%gccf=3ppb0il!k;ALJ4yMs{M+J_zkFryRr<-+=kw`uhEt&g7(hi?67 z-?iff+>3X9KY!EGG=6wksq$!zte@f?`3&CM`Py9l?5oYU@2E|yIWZ_0mT|dE%!$NtzrJwHaXY5V%@+#(Au(uvT(B^6 zF7WMf$Hns?>XYv*x|}aE z96*@wLcx|B<@WafVc4%0} z%nzdO950L<_kG697vV6pS~;5Rl9?()auAk<4uc!S&97T-)sXw-!xAD-Il@H*vf=GC z`Y+h7DlE^msaRFH!Af#)gPW7|+tW-MkgB>?Q*+>jhd0N(HYE17bY2jI>RnjRo E0nP18R{#J2 literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/data/vendor_id=5/00000-2-cbd5000d-366b-4dbc-9825-61a5b85c6dba-00001.parquet b/tests/integration/test_storage_iceberg/taxis/data/vendor_id=5/00000-2-cbd5000d-366b-4dbc-9825-61a5b85c6dba-00001.parquet new file mode 100644 index 0000000000000000000000000000000000000000..e276734d08779f6f383b7160647fa37ea9d1e69e GIT binary patch literal 1501 zcma)6O=uHQ5PsQg;x@L{O5c)&1;avPg*0@NCWcU}(o3;WN}(rfo6YXqx{|+cHWevF zPg?ONcq!tcTG5+UJk*m&@uH%l;6W@NR6L0Id)C?CY-tKE`!jE5zHjEu%-iu}6Cy@< zFHYh+KUP;GJf>jDU$E(MEro!@0;WwY&KWuC1{qF0t7l=aq>u>9h zKZE=6-{ospJ6igWjxMO3D7@bFoObDR_|LCPYwa!FPfQe^HWu$)52E2m=SG|e5uCyo zHy`}&YzdK-Lg9N5<=5h;LlA=NqPQqyO6KJgF~za2`TlUHYj#DSnO4=$N4Ti<@u``c zwZoCOH28aVcm!0JJm`GNfXO z;0VHW4~n+UlZsN+XEbG!vNgL5U8|XE8X5^69oAaW>V{O2$z&Me2xxb!l1^r%a@?IB+E{N$DrL+`wi??kL{J8bvcY#_9KmxD854RIAXKliZhif9PfWU(AuJ&FHpNnLhW!xi zHs>$8N0UjB=(EYXe2SE%=q9xw7-wrl$P0BtZ^(v_5NuPQrU?1|grI7qq|}A{i3P#b zhzX%0l}&$d908(}xrU|_umcH6xRj-8WmZ^B*vOQJbjJ2Sap|o%cf5B-tEH-Bcwq-T zxbGb73|6WQICCC4$4Px|__1HYo>sqxE6O&>pnU3`q z8(K+;sX0X^ikM5u1G1Q)w#Tya`Is_alOovX}X-lbEQNbUj2n9bQZ6CD-iF|=}AIXzL{xa01%AqgdcBT%`I1u#zPbk4@Kt& z2M0}aFiPahzwY>yBy5MN`_Jw#V-)gt%E$RvvxK{Ht&J>PnmsRiS_%@MO#eJx*7^sr zcaL7&yJy_2C->H^*|y3fCZP|e9-VqwjhLGJGWB%-^gC@=YC37NpjnAUSY*j8V0KlIIzn>Wq*EwY4v z(XAm^m>AgBF4M<;l& znTg!|&RSQ?xhwBbU@cKZbaD-c^#bPEi$jREs0e)HPU29k*>3QCK-tjO6!78@;R1U@Q;_##T=WO+2Pa4a1-RJmFQYsHJTu$d%gBAymOt;(LtN(V&; zo!yd9IT=Z)92e7BieisZ0K10*ck4hDUzY_YIRymV3={aXNoAScC^N}*d#B8P=eJ<` z@|k<9aN6fA?YStVNaALV0x>^rxXXU+ColkOugr%O3=OM&DH;~}z(gKLS#u_M1y=YX ze>t|YRH#Y=`=^3p;S>nLWec! zGdHff$402yr50VMq`PWT^;}Jsp>{PVRnww*qo{Tr8dn*TPO7#x%;JN|M2bU4OO`{? zZ)*7>O;&Q|3-*wDJn2ze0;)g4|4VHBlUSMcDs(en(A+?{E#QZ4MHRMDDB%?h=$U}N z^a54Mg8j7E?hhX)((Vz>87)z_Y1oO6QKx7X?07nrZb`Mq8*?QqZ^X?`L#IZ%vqf*y r(`L?~db_UYTGO4GOlwCjqcx{#J2l#K8|5tC*YJmQ;CrGEUmw2#w+(LT literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/data/vendor_id=5/00000-4-c4940e8e-116f-436e-96ae-5c639106aec0-00005.parquet b/tests/integration/test_storage_iceberg/taxis/data/vendor_id=5/00000-4-c4940e8e-116f-436e-96ae-5c639106aec0-00005.parquet new file mode 100644 index 0000000000000000000000000000000000000000..d9d03829c0531c92721d3e38da6748193ce85a6a GIT binary patch literal 1818 zcmbtVTWAzl7(O#QnapmD#*lxQfdRKu93pNOH_4_;(5l#iHpPZQpwgA>?97=su-Q#! zXVXiFv_42M*yn;45-&(!ieR;p2PqPST3at5KC};o)XkgVbA9N4_A;4tmr(4mm;ZeK z`ObI#8#8wJqZA0x4N3U!<<+aP{lJ4j$gV|+)E|p}Nl4e@yZsCRf;%&CZe{I$mj)6N zML;~XJDaWtA@b&1JCAhN6?$L< z)*hX|w|y%G+li3U{p-C^#$or*z|U)oXFFRYMn|pk2cvV&4&whkvDygYbtK~OF5J2J zVjWZTuPFBZ@Ox6tLe(}}+h@}@}xw(9PPm7La=W+|S&~uuM?fP{w z_L|vJck-$tbs(EPArp92Mxf@tab!o+eXC)H+=Am&52_0OiHAe5bY=QL?AhhyAv|=3 zS>5j)3c0!Cs=}RU0GgZy2jz&W$xA$(YM9kXVBYQs%vWOFL~h||hi~Q!6k=HR-MkW= zrA52sO}O?rUq?E9&z)v#fq;ZNx@v3NE&FECqT?cg7@}`Drbj2tsZyork4pr$A=Xp& zOAIX96V9jjEtr$LNjj!!cuiSSH8pEwH4xOaB#3HA83b99eR+j5vWBE>(p+T&n z5KewGq$eD1F@6&;q-O6ZK|`^STG2Lw-!s9lqb#IXq6XUB>1{wXg$OTDsND#laE)If4D-j}bq1v~C!Y>V2d%z!myrJTG?tzpZ`52iBd f^x)h1w7D-uhp9cB-=k)6y2Bqr@eMVGZ^eHAspOVj literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/data/vendor_id=5/00000-5-f80715f7-5243-45ea-bc9f-f087ab7c2a69-00005.parquet b/tests/integration/test_storage_iceberg/taxis/data/vendor_id=5/00000-5-f80715f7-5243-45ea-bc9f-f087ab7c2a69-00005.parquet new file mode 100644 index 0000000000000000000000000000000000000000..0d704e74178cd618148a2f029ffb08702c2b5e7e GIT binary patch literal 1818 zcmbtVU1$_n6uvV%yO~|Jt|4cbfdRKu9HMR(H_65&XjN>XZAuJeFwB0-Q^ML`twL2FRwfTAA0WoOeWnW6g%wC zx!*b8{qFg>%-FuS(jY(_QgG(!>C>@Yz=J@@<^_>79+SRD$c7u2`WOHNBeU?${2!l- zDw+aB0P#R{A)k-utvDff{yKWJP7xr1l;6JkeN5>{<~w|4b9;lEfKBrm?SX+N@< zLN^S0d5k6cR~FlUdUWC9+rQp2UgzeQ9usmP^3MDJBbOe`<=oPTTibL@E1#dYgzk6A z*yi6CV$WGEWfQN+Vi?)%37Ehu5(0Ji+5H<^?w^}xz%AHzZI2@3Uu0hZ7Qdg~9lQNq zYAstLdjoE6RFS!ZO+b}$V51ySRB4fiBTcgw^3C&M-+UHnn8+=@7WRxnkwOgH zzJ-^gv$SND-3iAU=Nm|u=Q`7DEf5efVncmxITg<+nRGltAcp8V+i>ZGF;%XXym67h zdc?XbUYUUIn|X;anfZD-TEaC#aiu=#a7Rl zSUcK5?1LCc(zhZLtA$h$n~AM1SY2|KDb_;zd`N9{o&DDKpi+GX8mkTppD~5|BBY3Z zKCD8lsSqrFE2WzphUnJ>OsO?HvOiEPrFOJ~;15jjqbN)1Sww~PO~DSGt?M3_mN^X6 z7bEHr>E#Z}+-QeQm)WLQqo84bu+_RK-RtUMlPkaAn6yCM$yCKWLZ=M*AwQ>iC#I=3 ztW`X>YI<5yYq#i)NF?qt%^2$Xij5DI;oY6DHiX~F;ZMQQ*~U5e%mNkXtK3u zv0R$eP9_`FexKSL`2R{ASWRr*j5X+Xv21w0@Sx8R+@UqtR=J87)~9EDdi!--V;8Ka z!?HholE`mN8}3Y%dVN!FVwSoUr(8;;Q|bQHK%%!$b&6KP&RQn5(%F7<&`jF}i<(2G kSr|xXGns*%g^aNyO^2v8RM@KI@OB43gyI`&4Bv|X06>M2o&W#< literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/data/vendor_id=6/.00000-2-cbd5000d-366b-4dbc-9825-61a5b85c6dba-00002.parquet.crc b/tests/integration/test_storage_iceberg/taxis/data/vendor_id=6/.00000-2-cbd5000d-366b-4dbc-9825-61a5b85c6dba-00002.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..ebc2fe0a310fbbc83abd2a3f93ce32feb00f82de GIT binary patch literal 20 bcmYc;N@ieSU}7kBJy-v#?04ga*_}TDMcWAL literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/data/vendor_id=6/.00000-3-eea1ad05-da93-4b49-aa8d-3ebbb5c69379-00006.parquet.crc b/tests/integration/test_storage_iceberg/taxis/data/vendor_id=6/.00000-3-eea1ad05-da93-4b49-aa8d-3ebbb5c69379-00006.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..bf4b3e7540c95e3fd89ba6088c7c57d2f3add3d1 GIT binary patch literal 20 ccmYc;N@ieSU}CuIv$=BG(}aW0t*gHQ07e!Fl>h($ literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/data/vendor_id=6/.00000-4-c4940e8e-116f-436e-96ae-5c639106aec0-00006.parquet.crc b/tests/integration/test_storage_iceberg/taxis/data/vendor_id=6/.00000-4-c4940e8e-116f-436e-96ae-5c639106aec0-00006.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..22746a11d3be2c4b39bd6c41e30a652abe1e9d49 GIT binary patch literal 24 gcmYc;N@ieSU}Bi6y6bZ29JT`Ce`P{fjUN^P09;21-2eap literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/data/vendor_id=6/.00000-5-f80715f7-5243-45ea-bc9f-f087ab7c2a69-00006.parquet.crc b/tests/integration/test_storage_iceberg/taxis/data/vendor_id=6/.00000-5-f80715f7-5243-45ea-bc9f-f087ab7c2a69-00006.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..6b9711dfe42726b2fa14e4958927b52c4137efec GIT binary patch literal 24 gcmYc;N@ieSU}A_*n0t5Xrd$74&yAgV)%al%0C2AhEdT%j literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/data/vendor_id=6/00000-2-cbd5000d-366b-4dbc-9825-61a5b85c6dba-00002.parquet b/tests/integration/test_storage_iceberg/taxis/data/vendor_id=6/00000-2-cbd5000d-366b-4dbc-9825-61a5b85c6dba-00002.parquet new file mode 100644 index 0000000000000000000000000000000000000000..a6545d948a63ab68778e3bf4e0300db017d78cd8 GIT binary patch literal 1499 zcma)6&1(};5P#Wxbd9Z5>3d{h!LZU;F%8|Mi6NAtcqtZ3DfDD*v%Bx5uH>tmO+`vU zq@Ymf(LbPGJa|%z;Gu`ogGfQ}pr}Pe#AA`-AK>g~wloDFo85W8nfcAUnR%JMJ|km< z&*BU|e1EvrCt?n^JmwL)unsSKY1%`C4nEy_W&yT!7H@xix8L8@`rFUF4~Rq2uUB5V z_B%X_3^mMh?%F|7RGH=D5h(|w)-+cMJ)7^Y>GR)<3J!sLLxNf0wUq@9E3xHNGLA5(ryUPyW%E?RaaaKI|krdors!>T4gwnaAXiAPA#)K zZ5RtWo#h;)$29b&wU0m>!J^OJQ@v#>b&bu&5RLP zb-GZw55KCiIEeCbQG_fBCy2saiWhN2804X9oxnIkI5JK^Io2s_M)YI2vSs6a)6nh{e7e5;5(dw}GCaok}IR@F^h1wCZ$O-4tuGCz(6W4y3pY(%Zh z*TN$1bz*esk{zl@%3wAI69x~E+utDFjzzZ@#S$XVIl`rQvL7C><}YzZ*OjTB<8G%ibSHa530Y zTQ?Z!frOY%ZM{m9l_J#`m5W(z qRFf+zWm-Yg)SO(*=W|!pyfPxQ0;2`>vM>ktGx(rOFtgL}xBd_JuRv@7 literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/data/vendor_id=6/00000-3-eea1ad05-da93-4b49-aa8d-3ebbb5c69379-00006.parquet b/tests/integration/test_storage_iceberg/taxis/data/vendor_id=6/00000-3-eea1ad05-da93-4b49-aa8d-3ebbb5c69379-00006.parquet new file mode 100644 index 0000000000000000000000000000000000000000..c652d5a2ba3156371ee96c0f578da0164446120a GIT binary patch literal 1535 zcma)6O=uHA6rOC-besN&(iyY5U|29#NJBSiVhN%6hm<0v2tBpZ?9XhvW%JX`rXr=% zgLqI6MLZOb-i3n2n+VcV6@*$m6c1AM;KhT7f+u~u+1+fZ1c%M;oA>7XzM1#lWctc2 z5hKjw0^WMO^RkD>3{0b#Md;ikcsWCnOQn*gDr*S6IC~fy0H=c)#}7B&Wja7UAH4s7 z7Q5yx9wfF14=DT*AXf|`1mH)R@&SB{XNtHbTb^vWxG-D&mR|DDFSEO z`}O)ye+QHrd2$bB?$6R*P~uK9T%3)9WsiV~8IE}2t!pQ|uxAFs5rh(#xw!KQxLM5} z$vlL7GY}3);8C7eEeiJgMVF*V+&H<~D;3*Si^REd{7C(Pc@pL2(S>$ayB- zvrMaM&79ZO8K&jwwoJWl&m#!4u%h3Yt9rwdstTQnA{+y6uO^vvUaHiZRcnStxDR}% z8dlAAR@Hg!4*U$Md>-XuJP*4h499bGNvLOp%dsfR1(b1wFl3S}B-vs+uWgH)ETI-K|rhAT%tqsaQfvXoZG#RVa+41Wl)g+7Jq}D}wD2Q$kg$ z*z5hC1c*tOnz~6Lu1Q9NraVrgRxJywsTMTjL*3^6Utsnq(ChLJaLuSmmM=W&;{*4h z16r*$;qv+LoDV;Kbvm#>K6U%_w^vbovM!lRO==BS%)~M^8+xsp5Yu8NolOkMP2Erv y+L)?PRUFGGql&1>Dpm4|B4@?1TrPV_&PgL8%~LfmU*zWCrUoA*LhDY$U;kgkfKD&~ literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/data/vendor_id=6/00000-4-c4940e8e-116f-436e-96ae-5c639106aec0-00006.parquet b/tests/integration/test_storage_iceberg/taxis/data/vendor_id=6/00000-4-c4940e8e-116f-436e-96ae-5c639106aec0-00006.parquet new file mode 100644 index 0000000000000000000000000000000000000000..8441ab330efd50c8bfc1814e8ca41589e2610511 GIT binary patch literal 1801 zcmbVNPiWIn7=KOEHtja2vKJZ%SR#grU0K()mLj5f84kx7>||w4lb0Du+bvBeG6p*c z3ImTrcGSCv=_ZUZWOxu62vRox;X%dYkg==%b@vDwznNk+TzzI^TT=iu2&>+F@&PWF?pT!*DUJoecE7mR+i*8bmm3Nfxx$6*j|*m0YltauIi}0)8RNp9u{yxBS*(H_ z;&^t2a?D+?V=ns~v+A#zz|}{5rj*wS4k1jRqr$;CQdCODl%`BlH6&mf+Ke@)5oTaT zk3CnkvMCj1GU-P+1ZbBk8DvT-lqyAYl0mo|u+wF;WPwFxO1%p|U79|Jk|CalZL&1S zb6GJ94bCMQ+RwR^G0oDf$!>VWtjO@q+BPi~^*a@BqKJ#y*9A5XTgMcrybj_%b|Zi< zBXNalYBh4ID{XneD{gy0t8qF_%hGB^+h~0VVwU}FYwdb@%+&KbS}$8sXE#0Ku2)?2 z^0h;+y+O%tf!eH%!oH=j&fpjlmwh}A)fL>euV?hi11rQ$8Zv4$QPZ+AYDL@V-4B7@ z55}rq)z9PZx?Y=ewPlMe6te0U|M+-lOn5-Z{n>nQgK>Xh}Nh>4z=F&|4U39B{puyF?3ZgNv0z_?C@Rp z=oq$As=%dn=t+m(dgYFZ0`=6a>~Afg@bwwVn5__VpkPGih*8!`#Yj9BABrU+{rQTf zD-kuV$V7>!hvZ>7uI3dYr(`*wh^Ldu#KnA48jO<^QBwKyTo!J)`#~D?^*H{vM)3cutXLnc4ckXI);eiWjGvTu#=TFO}^Af+HPq&kulgo zP#k#NVMjf9pKd}KLxu+#f*_;*#e~A#uyPnbs_t%g^Oymm9|P`Moy%sk zd-<#%El59~*%T{y0&l*5^R=ff_?Pc*-ysV5K3{l#{O}AO#7ldd4+CxCzK1emjue%WF|8?6R2>PJhBjk;(+D%L zqTBvfw6ZA`WisVQI0WbkRWiu5R47%7<`jc)7hq?~X2}AJ%CveHe!7(JL#YtY!!}ul zfP%W)yP7S3^4|v5b4`?+{r)61M?Pv$Bk3r0`e{8Lq7Y4VH*Wr5EiaNXD5o=y? z(aYBlz5WCxy9MfB9TfI0g>?qUkhtvQaj2o-u6-k;R~}d)ZqSfXvx!=kl~Fs|LGOMD z^nNf_^>+L`?rP|DI9FS?$U-4^{Ni694~+>o2-)urp;xTXPl8xL>^dB`$grz{UaNb< zCpDSmh>?$#<=dnng?FiW!Mry^gb|@^8Wq_TqC(v@pee%0U{p{wqAO)#LdPy=J;US0b znxiAwN~r=v>(EmUy?x}4hywN0tn6$(XGWbD&^E=7>?&O2tS#7Ei>Ik^Wpo z)0K!itjI)(4=3ayIj-guBBy0Jmy8dmQptmfz%OQX3q#0^o-bw5X;~(2#2^3E90CkEqU;`EzVO|vBg=hOoTMIC%71UA5l8UhlEBaC%ouY%GxJ$EYwcEy7G0``O0zj|>xT)08_4TlH{DBQQAr(;MX#l@#SF zpE)tjn;Id?m&KCl-xx=L=wz~{>ICdSLZS|3Q7aetnW%+~H<0$&{wFT67Uzuj#%QIe ziAE!AUjyzs2OEQBtp;bVflfBi>rc$aBv4A#-1*5F6dbCE`ecn5JtaLnMf9qwmBT_z zh{qD)?tD!x%3&ofOGFmZaj8!dl)Ox&lqBU7LOPjD?9V5~UV)^DoXYRx#^8LqZ?qSF Jq@(a-{|C@5O#J`= literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/data/vendor_id=7/00000-3-eea1ad05-da93-4b49-aa8d-3ebbb5c69379-00007.parquet b/tests/integration/test_storage_iceberg/taxis/data/vendor_id=7/00000-3-eea1ad05-da93-4b49-aa8d-3ebbb5c69379-00007.parquet new file mode 100644 index 0000000000000000000000000000000000000000..d77b2da658939fe2734e2190b4dd7c551b7062cc GIT binary patch literal 1539 zcma)6U1-x#6u!;UF6(|o*}XQH=?JV$=*qILbCfcFwg=f{OrgGPvgPMy%gEX;O(!xY z6BLAj556cO;XmY=KK{2 zBP`$yez5oEQHy{X7)CIM(5WqWxtFReEG+2S;wD0=6LZ^L;B+x#_~oa{la3~s>bLhh zh(UYDGao%vH_qZ)TbnywO;DGnrk=MVw&&Vb)R&sT)t|e++M9sPjh@Q*z1-UnY9sqs zJv9RR+WqnBPiGUL#@@e)hHlT&b`ax_GJK3{1zPIs&sC0ov`G#5cQ3f^{Iw&=XP zQYh!`9EWfR_>PtAg5%6<^ZIpoOnI>nrK5rXt0W9B@N;oaz$~BUP=pUCW0o*vk}So! zY$LA?i=h5`wi{m9H#8W+3~)b-;BNAi#bYR5YzX)p@#xKX5$OG5G%Sbyus2Bm+M)mK z(pN(Q=DZCYOpL2?fh@zgvkjG)1MDcA#z)_Djm|1V>7EFM( z`z_J1oV6eJ21UL)BJaXZYASGYJ3Ntpl*-kbfGSj4ve0I51H1bT^8I%B?nbbP$QOn% zagID>yPYPCbr~vEsI{0ZsY`T4j_ot+qJ5)C#f(_8t+Hy139%L$Ff=hUm=JY?np#QB z%&v=$M@oo!dBvGu?@54IbhT_)6ymyUmT1D~GzatHaZE_~E0|gaz_xIJ>{Lfnt+I*;*}Adtk-tU87dXDCB#k zq%@RF_4X@e!_<29QB9?qG&-aXtCFs0R2@-OB_)lf)2TB`S{{_>2-QZE)BGHq)ZmRI K7NHsV{r?3FNm*q8 literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/data/vendor_id=7/00000-4-c4940e8e-116f-436e-96ae-5c639106aec0-00007.parquet b/tests/integration/test_storage_iceberg/taxis/data/vendor_id=7/00000-4-c4940e8e-116f-436e-96ae-5c639106aec0-00007.parquet new file mode 100644 index 0000000000000000000000000000000000000000..c21ece7eaf595a44db5879390685af9a749f42b1 GIT binary patch literal 1803 zcmbVNU1-x#6uwQf3U0JtvmLdMI7uj%3VIWvp)8uAGvvy0`Ik_W~S8e00Cu>g!<0%W@-QE2( zfR66D^#fAc^ycIE@R$ulK$5pstk@e>cDXAe86whpDyNG6h*lCRe?v z~}@Wno?ROeSU<4fL5!LLHeb^OfGHqF$mWHwj*n1EHJI~ ztLNaSO9Krk7UX%@CQEZXmk<-s;#`cO{hUh~(=5%J?5syjhzwtn zFtBmhI;KG7MHQ}N=L7fv5+|sVRxzi7(!2+};(`aXilt!qI?3_nj@`~eLK7Z)>e^9bpApcfD;k%_U%isuP$&|2^GGEfXMoYM>fepS-5BwUQCXVL@0SU68@aj1o+|6gM3 zdSY=pHlV9|Mlv1YHiz$;qYc_7&|?n0eB?HW0`=6a>`#rMP}h)T4Cjd1G-%Y1 z5F@K)()H0uv?bD7-B#k6`3f}c#GU7N7a-<f3U0JtvmLdMI7uj%3VIWvp)8wW`vvy0#&Dt!11I`2L_|c;leA4UT81p7z2|=Ce&@UA+ z;p0(E<83&C-%b5`5ZI3?Oe3`WI)e(2oSr~vcyTpr0T{u`IKDQw^16=43=sVoaM$Wo zGMW6GPWsWfwEEblSi#%y%{$N5ycNNhzrJ~kD0Kh*#S5Dc&)`;k>+0{jo{I3jy}eHZ z=;)r?KOv<}Z$6GcEj+wORYX5KzH_$w#=t8N>RCSVck}xXLU}HKoc>W?p&%>Cy`HVThAY2RBj+~jbz>G4e zo`at*rJGPJ$n&sGmf?6VAtrcCb1{bYb1r2}vkYspa~?4vGJLtVWs614PQ}Y8;=<4_ zG=`1C)?q;G%NpFkJ`dmnNSvg~TBV$dO3yvu6&F09RXUxrWocESRkS|TV3z%3Yb|=A zb&Gkeua~W;v-2Kt(JM}P`NE+W{z1vU0)<%>h3}Ta9D^f}jgQB{qJq2j#f)A+Mm}+# zhKx!#r)*gnRiah&t^`5v3u9Gp$QIZf|I-CX*B~(vh5enhZ&y6>3Z{&kqx!OURi> z7F3PsN>1oHIVM<0G%RGKA#1&Jmw{rCk-TOQ@T-!ZBjHj^J)21joDxvy$lucQ|}kA8o)^vUzA)haPk2l`XeH6sV_WWq)cMg?ff1V8pmW?m$j&M(P!asFGfxPQiTT| literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/data/vendor_id=8/.00000-4-c4940e8e-116f-436e-96ae-5c639106aec0-00008.parquet.crc b/tests/integration/test_storage_iceberg/taxis/data/vendor_id=8/.00000-4-c4940e8e-116f-436e-96ae-5c639106aec0-00008.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..a24b34e378470c2b6d393f16a5bd9fca22d21e9a GIT binary patch literal 24 gcmYc;N@ieSU}EUV{=ZT>>cC}HPvdD5Tr2(p0Ab(>UjP6A literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/data/vendor_id=8/.00000-5-f80715f7-5243-45ea-bc9f-f087ab7c2a69-00008.parquet.crc b/tests/integration/test_storage_iceberg/taxis/data/vendor_id=8/.00000-5-f80715f7-5243-45ea-bc9f-f087ab7c2a69-00008.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..cff7baa4f7e2439717fb02db7f65d6b03c32fc32 GIT binary patch literal 24 gcmYc;N@ieSU}7-zGi&O9^=QrSsl^Mp2XFs+^0K9A{Lqk6=FuuyJI>qkDZI1&sk!>87aOl$UwZS=h3;+$KWhnm zIEJ@YRv&Op6DTy3xy7TueLGto$RHl!M89CK&0%6rB({6$>|qFnkzcS)hnvskyn~|O z{5W7?`n2!!!}fD$fMuHww`)}NyGCn3fCzXD(jhTGD9+TZstb?T%ta2s%2d+Ds04yun*9KIay;9 za-mcy>LUWet$-aZ>m>s$(uv$Pc=GaOCrbDN0hlG>#DF*!5(1bPI|V39BQfR)M+OKu z(~Yt=EP}e~#crUmY-l8eIgtMC!5w7NjR#PO*7faW)YLN}3hMS&G%Q1XFX~jE_k#K( z-cYZ)0$8x7zq=}Ck|Y_2h_tn<)*IB#J+?+p)%#vd$afw`P-qdl&S$AQt7>Yd2?amQ z+-`@4Wz2jp>df)f$Z^MG%zOb3L#vge$u60xQX~UmY3MMxLEQX0<*J6ZndCU! zzdS1$NHi=J<$@7kZ%3eLY`UUq4E!L%!nS0-RGgIN!WJ`DXV%*GKY8)pJiER3XXo-I zS+A$KVZMwqr?t&JaVOm)&6@$@8 zG!}^myRsEEPlLG>RTzz?VoI+P&1ET5l8TazM^lMJyf2%Od!j7KXfk_J9E0=eywM5x Jj}F6s{U3m=O%VV9 literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/data/vendor_id=8/00000-3-eea1ad05-da93-4b49-aa8d-3ebbb5c69379-00008.parquet b/tests/integration/test_storage_iceberg/taxis/data/vendor_id=8/00000-3-eea1ad05-da93-4b49-aa8d-3ebbb5c69379-00008.parquet new file mode 100644 index 0000000000000000000000000000000000000000..239ac0a4f928aa8fd60443d5555ed2022d6395e4 GIT binary patch literal 1539 zcma)6PiWIn7=O*OnsrmB?1cs*D}j|MU0K$3j#`J)i84J5VX)z3O_P@`BmJ`^oyZ(` z84Nt>#eu>EZ-OWPFcgNT9Yho-DoF95b9m6BJ9rp=Ns~5XQY>xqz2AGk-|v0j`(8%| zFNqjo9;fj8?Zp=ik0}@iF@sS1O?WsTIh)C3)XmHdgjSlaw;uq|;SAu#H}4*`)Zvu3 zUcW^YI??)c%w@IVH2(AJI8=nraP!`(;Uew1!GcEMyU$2&oZ>4J*E+6V8woCtZ1esb)aO3=V6yD#qr!k6q<+Tk_-xRK4(m` z6g$jLN11f3uC<6ooz-gBP{l* z-K`t20QT;+L@l!Sz8CeI{IE@4HQJ?FZQyZ>Yx0kya=eD%6jmG3&}Q%jb?5EZ+p+bQ zf>=Q8CdE=whFzxH>?RB!&}5P&##G#tr%6rspZrDZ#Xil57bvp)%E_DMG3{ zCa4IHj0t%uXRojJBv1@8Tha^y^QxqqB<5-Ah5VGT5UWsos#Ldm|CgB9 zP4v3F4qep?l2tYCt@3^Mp$=Oql;HAJ>B%a6@71ZJ0`=7F?5`}K@NiKwW=q8C${CS4 zVwhSX9}(kXPdpLn%$77=iKzXGOcb%dNA8tHHLDQ0PnNR@u|Jti9M2}DZjtm6r7wGw Rn}D0@|00n_XaxTL{{oQJS)~8~ literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/data/vendor_id=8/00000-4-c4940e8e-116f-436e-96ae-5c639106aec0-00008.parquet b/tests/integration/test_storage_iceberg/taxis/data/vendor_id=8/00000-4-c4940e8e-116f-436e-96ae-5c639106aec0-00008.parquet new file mode 100644 index 0000000000000000000000000000000000000000..ef788a375ffeb6c11974026b972aa7b7e6cc7690 GIT binary patch literal 1802 zcmbVNU1-x#6uwQ_K?pfAG2leA4cT81p7&B=G}cfNDaJ%(SGv?T0XfS%mi9;85jS%UnU|@17v84eAHFQTK@7U|spIGN!*jS7uYZ3q7ibE9`m|DH z(K1=t2V2^9@+tg#z1*Dpf!WXVi)Wv`^Probxo_J~KM2+N`eOMm)0DoVW-`}Vv@!7e zt8aVx^Sc>7#M!yoSA&Teo&eyupB&#+bJxrUH>+v>V|>VYG{D@%;o$1M&J#x=Xw7VJ zdq((>XQT=UB#kwYL%cw4Gp21I>Zo=R zeug~Ojgmn@fHVoq3w&BiLx=N8j`i~m%9tfAvB2%9YuV^nHNYL zQpXIayb9nhve1T)An7(!*Q)KQs>V|uE%dGhLGKG^SFh|BaC=p+#ko4Nc>;wj`=!4=0U8r-5OSU^Lf^4N-w$9B zky$uyi6gUZU3T||I&_6*sW}!cDi>%@4y`d$qIG$kii2X&GE0gjM#PHipstF8y%AB< zsi79d!E;lhjl?2iUe4L;-ChQYNheCWNx`qlMv+EpF^xigOq`BXsBxECZTkNuCN>jm z)3F6zGYYch3irAEhB?}Ttrki!v@SjA(wj$aizrY}-QNBAX%sp&E}Ii2Y8}j(;Yn&1 z^+G-zi^k&7M7Sqg(hW7NrBsEgu~b~?Q({_HrAoh|WD~JeGMO02Cgt83?Wbygc0Zqn P32(fS4&R0$_zM05$exfV literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/data/vendor_id=8/00000-5-f80715f7-5243-45ea-bc9f-f087ab7c2a69-00008.parquet b/tests/integration/test_storage_iceberg/taxis/data/vendor_id=8/00000-5-f80715f7-5243-45ea-bc9f-f087ab7c2a69-00008.parquet new file mode 100644 index 0000000000000000000000000000000000000000..881936e184443dbd9e7343697cc6d87fd0d5cc24 GIT binary patch literal 1802 zcmbVNU1-x#6uwQ~vU_PD;0P`#c4h6>S%!*31kslv3>;S0G`X39tliRdDl+gv zMD)oAVGMmz@Wls(F=T8*Hc=2kHn4&}Ps7O=`19b4pm>tDX-CVDhP3zG@7(Wv=bn2H z85}qp#Vl^a5q$T_;$rg=%wQIweYZJOeAZLf5c>4u)tn7r1S=Ex*8KYG76EfW^kcv) ztCe&rmD(6h`O%d8^^rrdgU9gG>uW3Cn&2Nd-n>N&y8OBIXXD{H+>N)sKYkXd34i8{ zl4H>-S=$d=YB>1>{1Ko0Q&nPFVB;C0PupKBKUMH9HjDW4^$4T^&Zvz<&RFiA6N)*R8* zA*P5lS*AX2&sl^ySkdClRXu0P8HEn{5e@>nOOs7HB9CSB8Ec3`xEZj$IV)>}8FfUv z1V2L_?L_gQAV8Xg;{`q`B?Zj#agO!#Rmzwp9I?o}M@mYZP^oRjBItly@d}FgFmwxx zA#q3@21H&4a64IO!pD#_!&J1&dnzd{dB7_zdqAsvIu*;-szvK)eF$JeemhzPFLZ8c zU)$^DC>mtJBNe>THLp-S^x{7#Iai=KtE2G4R(QtY2$H6K0uGiGs%!7WH_5UGR!9ph z?5KQmDwe&YTC|SdjUec4aCY?yegQX^^y-|eBby;m$bw(`>l5H&!V5yqSSR#7JM_Z< z77>|+1@22a4{+j2WZfqLrp?k`NC(CKm6oXAtFYs~DJ zq-IXfW;&vgXe`pxaUh-74Yfl{s0vl1iI~!>M76X^l|DsD_e2x%c+cTSfjGk4)fjWb>)7w!esTmIVw0FxXHZvX%Q literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/metadata/.00001-8fffa3b5-2209-404a-907a-9e33b85a3eee.metadata.json.crc b/tests/integration/test_storage_iceberg/taxis/metadata/.00001-8fffa3b5-2209-404a-907a-9e33b85a3eee.metadata.json.crc new file mode 100644 index 0000000000000000000000000000000000000000..3afa2c7aead48561273acfb7a9e95ff9e3bf570c GIT binary patch literal 36 scmYc;N@ieSU}8wSn054`XYRs}8fUzwB};VPIiRgjb?Nt=%Jvz%0SW^U?*IS* literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/metadata/.00002-c55cb6e3-46c4-480b-86e8-8483b4343fdf.metadata.json.crc b/tests/integration/test_storage_iceberg/taxis/metadata/.00002-c55cb6e3-46c4-480b-86e8-8483b4343fdf.metadata.json.crc new file mode 100644 index 0000000000000000000000000000000000000000..53fa3d8a7d362f68f2ca7846cc6f6d390ec39fe3 GIT binary patch literal 40 wcmYc;N@ieSU}E@?aLMB}^L8No}F=D9lPEX4KKQhCo9Jyihmsq;SEPE2ZkC26zfW OF|idG#lXWSI9fba%ocM1 literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/metadata/.00005-d0b691c7-70b3-4eba-ac85-7a8b10a00ca1.metadata.json.crc b/tests/integration/test_storage_iceberg/taxis/metadata/.00005-d0b691c7-70b3-4eba-ac85-7a8b10a00ca1.metadata.json.crc new file mode 100644 index 0000000000000000000000000000000000000000..203440f6f4f22c39afa24fa048bb1f18825cd768 GIT binary patch literal 64 zcmV-G0KflZa$^7h00ICzAVulXNo}F=D9lP|W4(f^&Eo`xZUeSW2ON*$0bZ}&pc4F9 W`9W>z2*~Id&$XYw;#toxuE{TAydEO} literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/metadata/.00006-366f138d-535b-4fca-be66-0fca929084ea.metadata.json.crc b/tests/integration/test_storage_iceberg/taxis/metadata/.00006-366f138d-535b-4fca-be66-0fca929084ea.metadata.json.crc new file mode 100644 index 0000000000000000000000000000000000000000..16ccc3379c104fa329933cd1d05ac3e32d87d781 GIT binary patch literal 72 zcmV-O0Jr~Ra$^7h00IDz9XXQGNo}F=D9lQH*P&pF<>*fdyihmsq;SEPE2ZkC26zhN eT&{i!Ok7?95KBJ=-t5-|1qkUx#4bWv(g4Lz&>ph@ literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/metadata/.08acf54b-b53c-41d4-a71e-06bf618bad7b-m0.avro.crc b/tests/integration/test_storage_iceberg/taxis/metadata/.08acf54b-b53c-41d4-a71e-06bf618bad7b-m0.avro.crc new file mode 100644 index 0000000000000000000000000000000000000000..2cbb5940c9f75d6e9e9d8b6b5b0e1dcae49d5b3d GIT binary patch literal 60 zcmV-C0K@-da$^7h00IEo!;Lcj)s}KOo3#g#i30{bhA$yCT7MoumOCB+SR_m?OHt&l%JJ literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/metadata/.7ae325bd-fe20-4a55-917c-36cb8f6a488c-m0.avro.crc b/tests/integration/test_storage_iceberg/taxis/metadata/.7ae325bd-fe20-4a55-917c-36cb8f6a488c-m0.avro.crc new file mode 100644 index 0000000000000000000000000000000000000000..078f749127e97917bd2e323d5ac43b035b24f00e GIT binary patch literal 60 zcmV-C0K@-da$^7h00IEo!;Lcj)s}KOo3#g#i30{bhA$yCT7MoumOCB+S^?}rx=?6 literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/metadata/.f9e891e9-fbd3-4411-a5c6-0cc14a2f1392-m0.avro.crc b/tests/integration/test_storage_iceberg/taxis/metadata/.f9e891e9-fbd3-4411-a5c6-0cc14a2f1392-m0.avro.crc new file mode 100644 index 0000000000000000000000000000000000000000..039ec44364ca756d805b7218b88074a192b634c1 GIT binary patch literal 60 zcmV-C0K@-da$^7h00IEo!;Lcj)s}KOo3#g#i30{bhA$yCT7MoumOCB+Sul03v4)egFUf literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/metadata/.snap-5277285803176961576-1-e25db3dd-c8eb-4a35-a17f-75b68ee25005.avro.crc b/tests/integration/test_storage_iceberg/taxis/metadata/.snap-5277285803176961576-1-e25db3dd-c8eb-4a35-a17f-75b68ee25005.avro.crc new file mode 100644 index 0000000000000000000000000000000000000000..83b6ea3ee0206a09674844eef4f3b236825ce420 GIT binary patch literal 40 wcmYc;N@ieSU}89?XB?({Uj9_j&L^vL1W)NX3d`S!cU|UGZ~tOKNZ!>w02q1@0ssI2 literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/metadata/.snap-5735460159761889536-1-a51dd31d-ea86-42dd-82d1-1981332a0f6d.avro.crc b/tests/integration/test_storage_iceberg/taxis/metadata/.snap-5735460159761889536-1-a51dd31d-ea86-42dd-82d1-1981332a0f6d.avro.crc new file mode 100644 index 0000000000000000000000000000000000000000..f5f9b510b4208ad77c009dae6720ca6170d1523d GIT binary patch literal 40 wcmYc;N@ieSU}89?XB?({Uj9_j&L^vXMK`!Q3d`S!cU|UGZ~tO~%CgNL0VO;T!TeZj3l?#&_}*Pd-zyO5;?Rx)x-vGhG?(AI`#pN}-rei{ zDeIsinN!#>Y@td76r6DqI*n##E`xR1lo6rJ=73@GPjMYOvzj@=R7za%`}^Iy`~BYW z-sAPrq;1l>-}m3=`{(!h-rDr0n{J)usNFb=j(>EE==SHZ*eW`%?WwL96a6<$x+b>9 zV`4_fhUSQ^4{Q<1lSE83QI?$V!=`503OtO7HtzLw8^ikyG8`NWbQ+eqQ5=Z*qEaa8 z+InsZf^wmtraLY&Rb0ky#;}kZ8g35dGswmY%37XD0xjjWW_f7?hvIjH;*M((c~es| zo8i|$8_Q?`JarO%*T8M4&$br(unv7G1l-1|WfS!BfM$`YXRzZc*mUiFc+NqeKLUZf zo&&G)In!wz^qikBf^~4#L^-G1a($qEfd@p>GYpWYiRp@o8qOfkaK%>gJa3U|N^qJq zl0$BHsDYFlfd^c$RH&v7OGU13nZmvIvK;zFVf8i#P<`HBjiF#KGWKlA`A8I$s<##{s@;ylfiVf|U{c!;YYvg4+$q z_<7}HmpRoGVMl)mE4rzq`(4acUx9&jmWvFkO(4gK`rtUPyP4CSHY~MADX*NDy@Qig zEyK&2RG4`bly{gOWZQ^zlJ4Rx%&_ozwDo!ti58_Vkw^wr;19ffC-q>WqeRPrB;|G^~xfii~ys=*^-W%!wdXt5J^R$&XObHNmyuhoDP=g&u(k*aIGw zZr*q@>XTm_(*sosppxR%K$VKR#c<6R6PItvFZ$}BqP-ANElEgC!>c zEICz0waM2sLAh#^Q&oZ$%vGo>fua_UR=&ibGmF^WPTbsIYW09E0b}sTY2>c zHzYQ27HF?eJ!P3)hMh#Mi-3>AofzmYJ&t6a7CDv;9@!ey3*Ay8yo-?M5QI)R3v3Y1 z3hiGh8+CJ(i;*cihf;ji(y)3fcxNCNH~5^hsSn!@aL1a281l#_Cx=ytwI<%6s~C9M z#{710;qxIyscapC1udZa!D_&^8d&JqDn_CuOtvioK&fVaNhq!^O6|)n4PmTtwz5(S;kr(&XS=&cAB-30`%SfeLM&??kdiQzC7rHOAYYggvXLlvs7efd95mT1tF*EG; ziZZ$qcEagsz;{^~j}a;tK7bLE-|>SJOkqJri%`DMy=FEdqU?@oE68GgHv1S9vOM5z z@Y4|9(v+6%xe2~>=i8URZ(SRi4&e@DRBFIgb(ee zyX(I-sn3iq-*xVC-HWxuXI@|R_KG>R@9(|nZJpJ2sdmSXT1%h1|M>d4+fPkAv-mx< zc9pU4{<>{vu1u`I*&1&r?)=>|-<@B7Ze(ovgBQmS{<8fKXVXI+u@C0` zM!UQ_e(=Kb#8IP_Udu?>)-aX&OsRL-?JY8$JZ|>n224B5@=C-|SKizirv0of{ z>e)|hn14BWNq^5V41;{O&tXcaHyS!?N2)x7VK-`SpT>Z39iWyqo&+Qa$c9_|3zD-)%>w*{|W1)wP>uf_jLS?6>rbobmx{gAMu(mZXF!n{`}K#jdqV* zNKFoZZ-KU8=aKp|-#`7ezMkFRF?Y|}n@bw<^H2V7-}vi4ojD`7qVD~nc4guxZ?>!% zI`Gf4$99?N?HzTy_BGzOdT`&7neOE)KfF5oS?#&T!~bYJ{KTU#Z93F{Pv;L7CjQp{ zsfiQIdVjn>`MKFYKRWi-$>HB_Ikj~58><`dZ2U;BL3;G}OFuk0eER(0So>F7+P8On Z@J|0JY4R!a;k)NOd0?wFSsTsx{{ag+ub2P; literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/metadata/5e3c62a9-1537-455f-98e5-0a067af5752a-m0.avro b/tests/integration/test_storage_iceberg/taxis/metadata/5e3c62a9-1537-455f-98e5-0a067af5752a-m0.avro new file mode 100644 index 0000000000000000000000000000000000000000..a3385f4b23b70343e857f7047ce25cfd744e3661 GIT binary patch literal 6324 zcmb_gTWl0n7;f28Owg(nft1D!BdwxS-I*!$@*)aUNf3=dxHiMt*|Y7?*_rLkY@32j ztj0=a$k$?BdciT;}wBPn*qB{5}SGwF(*`;Um^ zi7bg0N|W;g*h<=t1`j3C!Tnjo!SFtXOczUmPSdt}#9_%7RYFnEF*5TIRPzOshU+0q z$0h8hOdENj;nq+-g&eG*w4Jp`pvAmS+S!DOL-F;Yxa-+O-b!kzLHIRMPYEr6r$M6c z9$tY49Q$z})(cOBfIC>X9fDpC&@{4)6m~rgTb?rn&pGIGM<8%7>%yyC&I}p{J?H0( zU>%&bP{!@GJs)UaV83W(O%vp4V!91`1s+3XAtK?8^z<791MOXX?m`%bHzb(9N@b>n@(Uyuv3D6*cA**@Ol9mKd*f3DyNzv>>7u$ zW>{Kc$irOqcd1wk(L8V&aDAtW?7r~ZI+iext(Ux zw0jI4nLGW_n?@O!6G5t;Lm84MKSIGs3Z4x=1YHU&^axDAKJch`^TykwKKaG5Fi>>> zR7$)Os5+uSJ3)|85(%A7%5-E))bpZiX zWCCE}IZ?XS6n-q-w$>D-qFSR!QRPsB{!LlD(qw)dTgX~)u+(+{OYNwj+SEBsP^#M0 zjtapF<|@>cKv4@vtKY#Ti%sQ+hiMODM@se5c za0gNAA>iY1CkDEUk0X_*MXv3DM@|yzg>ESk-bKiB2*M6H3v3Y13R^dqjJh?-#mJOh zPAR@_C$U}!-kHe5jXvjW8NiMU+;NdChCH&x&0rm3T@-ILbPT+#C$}B^+7wch^42j} z&;q(2tOjhWfrXB(VkBD9)cH%Co3|t|(X!S!elK96yw$nU(EFWRC*v#vN;Pv!LSc1L zYF}z;2xE=2m6ciu*M;(Ibd|Y2Fnesz4HFIpRoGfN05OF4C4%WIl&kJx<_}w+BYOf< z!RIV!Nq;XZy*Dr;+>T9ly(p~j!I1Hx(yF)26OiO#%G?6WWiRWP&CW-KL zNrWNk&ndZ${sY^qzlYO1AomJv_*X(_fT#bKnIM?MPUpj1Cg*1{bc8Wby^NhQ1Leg4 ziU{W{5Q74Rw=gl3CgWjHGOP4LynCLJDbV-DB7+va)5zwq z6^TNH)ZKX$x23OPTng*H2vePBOv;9$F)4su5L3di5d98DAG9zdx9uFBSTgrn+ebJg z(_B`|D8*YwWhA-W@ z_T}$e&q0<;xC0rLN#Lr6N4^TVcGl6cKlJlH^w4j1Rhw`1{-?J6{`r;TGwc3N9(|+c z+ZsLb&tJE`|7!OiKR^7Z{^ga^Cnw%IG^M?_=j}5mw0-YhIJ=GNnf8fF#t#mc~}cmc&-ZV)aXpRxhhRn0&cv#mM#V{yn&2^)vf^QjgS)G%f!0 zz>h0`>&VDA&&~WYIyrv*#HRP}n|$HQ1JlZ~-bd<}#OmK%+qUu8v-jTj(6K6n_O2Ow zPXD@VnD&fkQmfI{7-$}_j>ofy|tw)OSUDu zd!Mi0^YQ+k+Z2Dky0PArhfzxUX5kLrnn)wQn>Q^ZBU`+mx8JlY3BEh%=aULj6DHri zbq9~1g#5fFrSSPNq$PAC1`qwbfkrcmf#7{oQcdJ{bgH^G#83KdQNb0p3?)4WK`~!2 zp_rDW$*6?gq^e7nYq-OePf7-gNhv*};XsRdozOFJ6}jT8U2)UWvAmXuCCA}cm4-@a z0X!8PeROiUG-l|J*sy*WZ~-@vtQ#1;9H1#lQEU5q_ zj^Uy)7(zk6v2ZLcS;MXdTy7X1a6y-$nj*R^S&FW4Pe0YmjVsnLw_$)IkYTGd!cZ{C zct-Rlbt47Yc6{?{-T<~a`wWMn>E>`c{G4vc_z#zsjL{6j@~jm$emG$%*g4lm@%Tf* z!U(Y4GUr?bA?WC2zJme-gI!56Iak-Y;vhM8@a@l};>h6iBxfHsIYr~FVL-;tD;vAW zs3r)T$_pr_XtDT&g_!EgGcc%Il1j93_&8SY9A|ZRFuLQaE|0`YD`#a_GqSR-W>Ojv zrXD%v?WRXE3<-CVVxbhwF!#B)^+rOW&e&Ke6n3h>9(cJ<8bMe`k(L8V#;q|)&CoWn z+bl0{p)kp$st+l$qz>AnHzlQEPB^K0QcB}I*%5L|g0pn^!RQiTp+{f>j(|tSn>QHt z`eYZ!{6N(OPzk|Gpz89vMRCm)6O(VjF8a!#BE8^;DwNes0tO9-2OI~Zo9mbQOJm16 ziKSvBsq+Y+5W)Z!o)e{OO<>2;9cxVxDylVt6j2T}Xy25@D=oy1V@=ZL9V{^nV2NE7 zRGT=f2})I)*i|7|&Rm7M;s|Q)Xtg`IWU-0t@KE(}WW?ftIS?*q;M~Pb6V~85>#P7e zVNLNyEil!E&Wf-j@p2^Ik{}7};LoJf^9vH5W@07NOkf9vrx^=?Bvv-gNWA5M#8x{@ zkyr{@5@v3*Dv7t9bMKqSG})L)6Y>ReoFm7<^&Bsm7m=MtW?EIOn4V0U$SS{jof{IV zC){=ma!=32pE1n*SILajFEY#&303EXi(hld<*-&mf zxbdJ%QOaA#U_o=}zOx$8tp*l4x{BdwaZ~3nac16y+W}OVu z2nf~8EeVCyd8vJ=rNNCgNLN;3!Ce>1v$3Jf^?}->d#;;sD5%2L$^nQX#4Zt3U!hz@ zC#gSlefI1LR0W$erzP#Ztn}VMjW9bl-u1ksz8q6UHwbEluHrD93hWN0%up?1w`n|E z1QpN2)J+oR>3$xDq&=rXo#Y?bUhO@c+yS{)V8gx=x*a_2x6}m29CkVz<}x`yjiJkp zf#{{|lo}{622g}KXMq?5D7^WJp)?tHg9_0~FT}g&7?}WlUo0|c(YuXo7E5-+5TPt` zL%9GUo?{du6jG5WL`dD;M{!5`>c%C|?(;C!ImRTcDH@Xi*f}x96-^@F!N`MlYUGZc z!xBsCK5csshoqXzY8jz;%ZNZFyPD&1M^ee3l2i4SrhP zTbj_a{nhZLJJ-JKeQOz#W@7GuM`Z%Is$$`zZr&nxk!U+d#B=&tT2f2L{BN zt1WGNK1-P5-yJmr&C$KjuKRN5>G!_QURZXr>f3`~ZN9Yhq2-68zce&7G)A^XLjJ|- zXsfbqaeYtY=ACc6S~F8M>f7SG>AJSHmk)gL?#quaJ9hn4{XaLC9}nE; zYiSMFW_xI1xFmEhfS&J4_KXWk9*1LIe JL(?pj{|7z+O921? literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/metadata/a51dd31d-ea86-42dd-82d1-1981332a0f6d-m0.avro b/tests/integration/test_storage_iceberg/taxis/metadata/a51dd31d-ea86-42dd-82d1-1981332a0f6d-m0.avro new file mode 100644 index 0000000000000000000000000000000000000000..790e8a66bd952a3ce82510c7da91c91329632ef9 GIT binary patch literal 6363 zcmb_gTWl0n8198`3z3!xRv#K1r$R)xWp}3ZrVn7Cp@0<&3P^A`JA1aBc6Mf&nJpk` zA|XIR6hl>@ctHseY_L&6VjzZqO%yK=MiZZmQ9^uCiJ}-ueDFVKcIKR&?k?M1EK}tdcNIb)N zW3Z&0eiLCOg`9q011UEG54d2eP)%!06*-z|2+#I*3)?lPU+7vVP{@c?7GX3J;XI>y z6Q-4fY1HFtq+!961oXgURwj<8@_D)}R%khLw1i9BzGj~`Ar8gb6Ikw5-uBsT(l zw@f(~5eRzAaJGXI2ZLKlikhkGOmWa0d-(RGlQC=wWy=9Q0K=2TOJ zZS6&@XoeEo>0qw8e%^>osp!ptMD zy#4eb%R;1+GzTYPhWXEftv48rwkbo=XuDSh?!e1*(jX=}^0XXCa&8SFJ34aw8P9xZs%ZL(rwbLXW@%90ZT@H*cgp=#yI< z(*sopKqW;=fvO|u7Q;1DOkBPtx9CfQiuOVns!-L_aTqiL8E^uOZlMSD6vmEsl1M2u zsnZCc6eR%Wo)d*@P2$JW9cxXJN~$%A6jcs2=-!n1D=o^8V*?q}4wl>wV96aNRGU1h z2?|x4+)*M}-du&c5-4i^XmvZdV6n;k@X*ce*ivGEIS?*y;M~PblUhlgwUq#!)SACh zb4+!qtt9MdyaJ84AV?BF_|vJ>^nygDnOw>=llVaqXvQ2M$)$}m8gDTm@zoAfL`h-? z5#~0jl0?@z|GrsasMgLDC0`&Xcya<<&#{7ek@;z4r*zFx%tXS*PVv?2-H=$vNua$h z^^|3H8LlI09Rz$F?nF;_{&6Jpw8%Cs@W_foz0fTsz`F=}_CZ(&XMqjES)qG*!Khn; zT#QWFd6eR-W*n=v;GK>f+~9J~rXg(Ez#S{1qR%6n>=afZ){01jrefe_eVOgx`U0P# z6t|ASg67eEZ#7_B4J>qQ6(iA-rp{jC?5rh$i59iS@p}Oi<*iPRhTiYoIvHURP^y_( z5^}2xQu{(ngCA>zt*q37zb+JKqpQgEf!SkwuAguysNB|y0f-^QFA+>%u3T*yX8y4C zIj|=%6@1RTmUQ>B!g~WV!tK~(*9(&RVoc@TAea@lioh9t64#>O$8}60R>EY?VWhMybu+zCP7s>fq3>|(9R4-$v z%s_E5fFi;c?8iRV^*AiHDQva*<&%`OJnEDv}a{51HtG^J&S z%i&9RrhWPQ*0GRb6YfApWgNJw=8&&Kwwbn6><;~`4?XmoUD=O|X5D(@=B<16+iU*2 z@avPm)Euo+V*@*+zplJ~-{XB>_kD8t=;6`V22LsOY<&ILA?5kE#*S|oJ$q#2q%yMc z(5AD;M>dX}Q4SwFGNQ~oJ{lVwGs`M2c3u3Ud(r8o&m9>5Xmr!ac=xK_bK|G>9(sR# z6rIpU_G;%gt?E6|yz+zhURV};25AE`m6z1DTh9#E%)S4+=0ErC3snn^`xfkOI@tKi zW4p_*q8N&)*UF@&4War_Yp6EV911P0pHunp!pmQlm4_?deEas5aO=VI=llL?s8~Hq zzc9Y|d;6!@&BkR5D(B9vuKcF++{DDofBzBrw5D$U>ExN84=vYse0XK}Le-7cp-@$I z?e3O2Yvn!rtG7M6sJ71jc4Fdz#wY&0+VkLVDaBZE;8x4lvmalo{o$d{$7a;kU0dJV zFu(rf{{1t%cGN_EUpo8Md9jH-l{dahuDletug`g82CgXI@b2Qq?zJ;#*G+==f9Nbr ACjbBd literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/metadata/e25db3dd-c8eb-4a35-a17f-75b68ee25005-m0.avro b/tests/integration/test_storage_iceberg/taxis/metadata/e25db3dd-c8eb-4a35-a17f-75b68ee25005-m0.avro new file mode 100644 index 0000000000000000000000000000000000000000..4b55eabcd68e72ebbe32d8f6fe1b3a8dfb07029d GIT binary patch literal 6501 zcmb_gd2AF_7zYxmC!(UFX>=T|!MLS6Go{Dkfrv#Yrv*HqnY^8yr6aR5+nL#7ttE&T zg@92D34(~V8X+wSCJJ8BpjNABJZhr;0R=Jfz&j{v41Vv;&b+t7?y_B)v`sqmzU%iL z@Au8z=-ng7oNX&jY6A70*2X)nNtKWAwqvCfhY#`Yn{qA2N5UaKuBm#=<|B*Sc>F{N z@dirZ^DU|oGc6e&hImVDPHC13@8d|f)sUxCH;tuyd&m_PeNo5Kl0y)bG6iFr?I1%@ z^Vp5+CUShkmA-r&S*nZ@X3D^U=JGmbrlPv)i%;^!ZO6p&Mof+`hhH5n&7%eI)Nu5T z?H8dI%be)Kx}(Ae+)@?O!sw*|O&~*ytF|MnhGVtDa|ZhK5eVE#+3+fzGmXSS&bj%* zSbJv;l(d^n#|7FISj-zKT?cuRn8pwvQ{yP5JA4E`Pg^9K1V&Q~IY^Fc0EuTfXAG8@ z)o&~;Cy~?SYry4(;Q<#+1*&P5sUSx)4Q}CWGq~lN)5O&^a0D`Jl}0Fp!;EJ{Z``yJ zkZsSmpyoQT&ADed3{8Z?=?HPArQknY5?ReD70a_$*!bauqhaTK8@b~b!r2jEyJg6^ z2t&|~?U@dW3=D21$x6De)5Sq@?BP2nm58bqXT~}Au+3=(=QIH_ZeH2gB}O$t*w*e< zWzCSIt&Yl6UxtANrh{~%jl;*Wdha-^yOPlz)lFrYoL@OBJBg81Og)t_h%oiYD{ntN z$g&XbB+XG1FvI-k!PZ+Q2-R|nAk=tO;10ZWCoNO4jvOrql8jp|NKer=aoa2-Z$b^p zq;4+N6r?Y3M{fcpVNQ6dnvIe;Pi};q7ULWfelWTOSm+U$fXl$6+|3)V3Hsy~$M8T^ z3s8ySVxX!GxR15|UhX))7qnn$9=H$nYbrMU-B&ov) zpeSGfv(JhAwI;G->7=zLibd5LL5e7c8gy^U+?6J<gYeTXhPrUhjrfbu|IB*QK74 zL@&bztkyxm$Kg)&bmtyN5=)D0(*lpI7}N{dQUbgSlV=}<1~?0B5YBQlrsa*gGRVd7 zlpRMXzGB8yWej+yBS$TDIcIr`YT3XYM+v;oBg^fisz9ux!ljy`0xw&d-VP3z`4pwF zbqp3XkM4V`0o`g~p`)u9jutm{<`QRTED2P!pf!%&3#ce-^~7k%{m!hDVHyFUn&~AW zySgB?&$l%Av4-i&N-X&6LSZ)Q3S1wkJ-X-m35SBpZmkf2C_?NKLG@+JRbxB#hpx|o zJ%OrVbLO?AyO-tP8>kUx$Hu!}kkl7qD(414t+hm$)X{R(WjS3<KMV=VB0ZF|5aJ<5 zAwePKh(dzYoq7}}rLTTmBJI8aQypSVVpYzVM8M9ADXJNWd

B%Bhi)b`DD{sr$6; z100fSE~sV1+$|$9EmOSvEamgvm*_PH^}w>b9PabJ1MY~)OY^81dV2*KT@E|`bkyOy zER4qhmGd9Kh)M7G-U+HOBcoX;pYL8J9T7oxN3~^SF+H1I3^G|B@HY4<^>1lH%QlXL zFWu?(W$#ZsqbiUWDAK!NV^dmD~U;F-5 zpP{imC2tH|U;p-$@g={$_Q$<9ly+6e$DUWRyRyB$;j~@+&aSQpp6E`DIqOh;uetc* zH&?FyuIIp87f+iut$XlleoyV1ZI|!u{`#(28`eIwf9%AAa|c)SAHU-8isOSn|NQs& zR>=WnlbjsA?wr>K_f_=vM*0pds@n45_*>efH}5`tq$|-gdF?CD?Ok$1^|(p={?BGN zKF=>2=-g=Zo!@*tioCqFf6T3ob+>JNYS)Xx{NU+Ft2T_NdwT7fZ$91m z!TS3m zUy;s|eU}NZM^`Mo>C+ebDnHrt=e1{QtNF&xpLX~Ev2~t&&bO0W@2ckqHjlpIjW3?= zxoUG=N6GY#f9H)qGVsh2)Y5VA)itBWjUSObcHwiqEdx*7xPHf0{nE1YO3M!3Jo3xl zt}VCEH@=wP^uR9%j?JB4U$rhedb-d)Vab-tC$HJPMk?#Ru3>8Lrfd2KC(O8f=5+fg z`rs()-Tvr>rSJa!jI(#^xn)z@zuxfU=lh@hr+d}2-S6S1W7OB{b-QuBD|go|bq+9y znbQn2a50cz)Cu7uexN2Q@|6W55osF;Fr^kbfsTyq;H`90=4*JzPB(hpu4Y=GeJm7*ZK{YMVC1feO#yz%ZKG&{TEnMAVjzEU3(g=fr0OJ|a z8`q5lWZUu0tGNzrbM_eyLsP@yboe>lknkTaiHub#8OyU)*!baurC{e=8~Ni8267|7 zcFUl15rCkZIr^Gl*haZeC0Ty}$Cg4i&D1Y+?LSCQj z;uspJ!T^;JC^RntHsoN5AplDZmr!lu zWld11+Qe{)U^#OY>WU+%xuezY;DW^_vcp5w+hrpX1xF<9f_AC@fHM0U@H7)knPvhzC_K%W10=DuaYo`T1|+uHVTwc& zvV|~nyR4FU+d22XnWsrcI!VYE$Z?Jw2iJ47U|vLa8ktE|u_AgrZpv2i)$81lvMMKl z_S)1_6zOHS7^}4q@Nu{k9o_lIk;u{_Q#ZgPBL?+CwiFNV!sOWnVKJNqHV9|A`41F~ zy2{JN@RYrkP<%;`$bmq%8ZNm+tehX;I$Bm*yN&TI!i zm%9|DxOEH`G>7gxs{!3=V4*Vq2oh>WW++s6D#px(SDZ%5AL}fG9%j5<&Il%2iV* z^@py{o;`u8U~}fQq`j9F-W#Y9X2-_6o|n`YV=C_kL9NhL9EMYl-Jy^fswLz$jc1FX z;#ruwNy0qc&%=mV-f&6C#I;PA@UuJJgA^XuGl#& zv83+Pw)b#Is=26^5%RZ;$h1u9?z5E7bzh{{7}NvH?((?LyAIeRrXbCuX6WtZWpsJ$ zxYJRE@3Jr+Jygzp03#-|<2xs)!mNzupnR@-Rdhsn*&WrEmBq|#wlT|=i%15s}yY!(C z-fd02R9RCK3UzI%Jiedf$E+TH=+Jboa&=j7`vQ5(wgoFx1MQz`E>UmI}hzz{(aT4BRlsksXFpg>w--5??-?%ipuIbJZJ!Si6 zx4u(*(_L-TpP#vQ*szh;t?B-ukMEnj=kLA)6K=S%dfxtxkAHsG+}iN8#@Fi>y?V55 zUe$m6=K2xmI`rE9`<6brzVC_q|6Ke0`L64`4}?D6+_2)(ss6uXdwrw-^Nqfv?C`&x z4L!XBsN)x|Vt@LN0|SkFntJ-z^z^^5ZseRfbGM$@@W_M5H*8w7VbifUW}SO~?XSu+ uJqzl;ne*T`hsTY+rvjZgb@SOz-;Z51W6OQUlVg5YMtm}5qHmHnxBmmcMU>hA literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/metadata/snap-1793608066486471262-1-f9e891e9-fbd3-4411-a5c6-0cc14a2f1392.avro b/tests/integration/test_storage_iceberg/taxis/metadata/snap-1793608066486471262-1-f9e891e9-fbd3-4411-a5c6-0cc14a2f1392.avro new file mode 100644 index 0000000000000000000000000000000000000000..c5246c3dd644df61578f5e089980ddf45bcb3aa8 GIT binary patch literal 3948 zcmbW3ZD<@t7{_a)wS*9ADb?E4j7yakl6aeYKMNXZ8k+_hO9dDXk4s(zHZ~3KB!5lG;Eq1dXL1Y@|pit+Q`4yY~|J^6hSS z=J`F(%=3T#yEoDKNX16jrO0(EmqKZ{w7HkjdNPQKF$&^jg=cCQ38wY`X($Ovq-yb` zAR_!YDIk$onb^VJycHR!rSJ+Xkg6$y)Peu=GKb@|$Q;xJSpxfC-wnD2Ed_S90fSDW z)(Sim3;KM0HTkhcBy~8Wi?Sp3K3Rhz&=Ulza0CHC0*M|CsdlFtcrM`$)H;iwA_5_9bqBQBR;N5Fr%hq%^CI}BV|Pq2R1Tq zw~wXGTB<{|6x4tb0sln6K6yPEl!RGPV`6#?>kh1<{CRX9YS6eXHO|RdqZso_ND6v| zB!dP$Wq>42=>hioWq1Jx4;>a%4PSD+L}a72&pS2fpz=sW8b0L`p*ziO-kBi>leCN8 zQ|{9w%7Kakd+=-AA!#fS@;p~jO3)X<%kqW9$&v;INsU{bVhJfB%OZl3-Ufz}_99d;h=6Qyt&n3YtuQ7`Fr*3e z30>rvh*(7YMS+*)j7##sVXH(|4kI;NJW9~iFgiTk7Fsf_xR<)zlE&5V3)`rf^fI1F zbEQ+LfIZT{?kjQ4h&-F5!RZ7dYIRwZr}#vu4zHpJI&*BDmQJuG>IAg+ZN-YzzeVh^ zsasqy><a2S7Mz3AI_r$lCV!c&2tM_g{JCyox>Z{4=Uk8t@-+y~)Quuez zn&!W%&Q$K`eC&Ai@z0{=zodSQPSk9BgWVNk#ugS9r|0kd(Ae@(^}vntrn^rz4V?17 z6^!$UaJaCyr&52L65IKT#8er~Ga)lFAY nFFZTZGXE4`x9PyZrwp?=I&=8?c(iJgPtV7{ADP>dduaawj&58J literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/metadata/snap-5277285803176961576-1-e25db3dd-c8eb-4a35-a17f-75b68ee25005.avro b/tests/integration/test_storage_iceberg/taxis/metadata/snap-5277285803176961576-1-e25db3dd-c8eb-4a35-a17f-75b68ee25005.avro new file mode 100644 index 0000000000000000000000000000000000000000..ada0e6cd3241efd578a84e3cbfe6f54d56880a1d GIT binary patch literal 4042 zcmbW3ZEO@p7{`^AA_Qu|Qo$BFZWfY|TyT?(R zOA#@_0u@S75rqV8Vn`aqSZe{b1*K{b%ZmmRL@ggsFhWsOBt>W6W_NqNdhNHn-I?e2 z%skKk`R@|n(|3=}gzE%;rbq`-2>z2*M@h9|L={s4qItnjVav=2rrRPNL>B zeyZ4Ev&Hh_W3z~>F-K>4Q|x8D1X&;_2t?*N1Q-tZY9%D<>1u)Prodq+NQ|71R}3TC zi^(K$hLcFvmG~tnp9>-)@&kznQGt>A=UZS33;llN_lg`0i$Pv8hpnQx2+QKYO6Jx2 zwNz?LWr!4n5>O)Gp9ttTFDHWnFv3byu^dCIfp)0EIJzH7P^p&arTK_*7_}rM1^I*| zgEBc~fH+C01^W6ycts{2TF;0QF0yxs$a&hnn992Ni zMi>EMcf_HPK7gnpIIP?(D#ad8A>NvRlbqKYhysEj?C5u?H^+6H%^&qdZ)s&37`rj`%`>wX zRhOlw{wTh3PutPvdf&q(M@o*=-FW7WK5FCq?mY(|KXZNocr`sEw;b+}Ry2UjvhQm;cYAk9X|%ER z_*bqaZTn(57r)%T)3$I=Y)ku#6Pi-#^yjCq-frIymoIHQP{T&I_C@#PoVe!Q`%}x- zU&<#fey!<5%I=RwzIFKa`d#Vo-h8Lkw~DZ`YOPV722uCX&W zt!vecF{2l*c<*)^6+At!+nMUk%eqEibN`y1GwZ3o^Q$HwoLU{-yr}Ekzn+J#1}C;{ xczepE{UchwPHAe6toeLL&&A&SWj#HYZ*|_f^4sq(tiI6RJG1BTrThEWZRU&u%>~C(Hd+(`yjzWTJs>|kG66aw}gjm*8@aQ95U{J>M37m&Imd$)pMjnYN zLZ*q!DyCWYkUv3JCny+50$2lsx{OQnZS#bMl7ys`p~Ae8)v!10eoieY3JGjw-s>Jq zkF(T-;4H+zia@?1;GVph49dWQg1L4xhI0o_QQt#{4&7SuG+yuvA5X5k><-0jX9!C z6lq~Sg6xVb#!g8s^a}a$CkXgK7ir{?|h?*4-TVZ56bgS zG*|1vajwjR(jw$~8Z0XmH3^C9Ned|uqegdzlN;LX3!-T$w%1xM!29&?_+Gi^I*zKN{(`I{;*KN~ zat9DsgTtE5VpD{23iZ}1oM2wd5N#Ae$C1Znt3dwWbr~Q`0e0uRcV+u617x6Lu;cQT z5yW>HFj4}nU_I5S-nz=ImLZ}PK{d8a3Kg8!(-|m6L9V!#$#Iod855@%+63l=u5wIN zEDCv}Aj?X|C4JzrRVpiwk=iYuq-bVx_PZCO`-J4W-Eth39;}e4fVI4pQT+`W6kEY# z8I?>D=S|&jr&Xb{O6lZkL77Mkw(bMn%cQ%9b!Klbd0;nVM}`FZ8>*)O)Q+}g0_;@nin%*5Gm zFI@X>b}`oX%M*1Y9UuSx!+7_pec3m6)Q^Gm$o8wNkA&8(>$>+uq%-#NPW@=&@2m0o z&E03tA8MJE8+JWC(%clM#qbu~UR*gS* q!;5dV-US~YZThBf`ugJVX9v!``u*D literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg/taxis/metadata/snap-608017560956539142-1-08acf54b-b53c-41d4-a71e-06bf618bad7b.avro b/tests/integration/test_storage_iceberg/taxis/metadata/snap-608017560956539142-1-08acf54b-b53c-41d4-a71e-06bf618bad7b.avro new file mode 100644 index 0000000000000000000000000000000000000000..9af6977d28bc2125bb8928ba32f1a2d97e7ea218 GIT binary patch literal 3990 zcmbW3Yitx%6vwSCYUl>4(TZ5*c4eKO%GSQL6$vQC@CrSUCt_5C)Ykx6L=hnIpj%+1)O_8RurLxqk+8_Ygc#!`YuHvNC$lULtY+SB z980slRDnn_C;>GB{)vEb@=7u&3T0MuRw^;{JJ5^rq|qZ#g6eIFVValKVzetEDJT_^ z4Av37_%;p*7mJ?3sQGleCM{ zQ})v&%7KCad+=-AA#p4a@;p^hg3lGi%hH8J^PB`3P7Ldvq6?{m=UD_fr43XiwFok9 z@hE^?kJj@sC;|wSO@QPHl8}{CJ2hmf$b_k(?KL?KCh3w)P{_@}%-lXy(2@encOxO( zfqT?cAL&s|z=XfS&?``P0d*IM@&bUkY(@IY&CCO~P8B(kSOi$zwQw-i-`-#D#LXAS z8q-A^>?ega%kenG(o;%7kdB$hId46#faM^ysUz^+85M6II-2B_^;syMvd%Z6`0Qas z?Ah|H6V1?i_Bcc4+0wlDdLC7+kme(ZUWQwUg%T<>W;njF62B|xq5?j@E9CS0JR!>K zF2dIphf+ln2_iO;`>7&^Mjc4#)D88kl|=s72W47u6wPYY4shAzuiIX_5?=@pGarBjHtX5eIJwI;$%5M&xzOg01L|4r8fcnCmieB-J#yEOq( z$cpB1*~-Yqc1@r+1vrQEWTIN@N_Scl;Ufqlu|-md-mI2xg8T$zXls!iLuhkj0t7>v zKpD^(j){nc#UB)SS<18|4=k=qWM%PDt;ItGZF-#c>1EPwLNd&5F$PNxmI}^+Rjt*e z+8Z)k>>L(rQt>3wKh*7Rni0xy*liSRQNJa!u~idh4iCG7E*O^CN^NFo##QbMyQr4* zJeFzZYEGd!+|dkdzY^PwNOMUuxH*A{T5S%MReT~;i&dEjbLJR2T{yvzs1?xOUxF2> ze2N%j6Fbr~R3PM0T+Z+J2L0Xu<@USp%l-4E^273#&M!~4H8)+U@GO0Q$)WS#H{>=I zJaBb#q-t#6(2w8zHPqREFcfgC$gTfD?s%^*@6ETXH{|F4y|Fg0zBp%9RbzgB{>d*M zuGl@cvwC=Z_J+qb`JmXc^VC1CgKKEdgs-r*xP6Q4`Tc0T?f9t+y|Ir*{YNhi?rM1D z^M>G_iN@i9C4;}zO%Bu^*?R2a6K!37uDaS^clyljd(-s(mU{P8-=@Z&M)!1;KKt@a zvT~+$_4LTK@@oYr9tWlL8)MAi6t!cmd+)97=T~{A*7t7yZP%ga@2sY0o^!NJ^o^{| z^R6tp`?TY(V+(hnpoqCNw9V00efjzw?(27dPA+fw_~zNB!&i^Zp4|%z+O{sI5=$T1 zzx@NW?5*VGn%x&xY!24ZqqoQJUEFr!*KcR9gCpviw zh1+`cfM~I=5C#Jv5(E)`9|0VK_JE-%Pd5u}X@UEYGA#13qdszMPL_$|kjbpq_@ChN zMG!~G1&n}QirIL+OIldyy2!Po5cVUVFg@(15ynBl11Fhp<;Sv?Efo;vLk5xv{38PS z$%|x=2jhSlt0G2r2U$_I65WLiCbt#Y4vCXu(kqcEhzex}uZWZZ3K<=MeEkW$y2eA- zam4r~+cP3x&GuENHZ-b>0>tApwG+XoEyaS7^*z+ygKiq&@G$2tXh<0huS*fk8ap zsViDV3YLT})@0;N*(KAUFr2`g-ab}KOCcb;h}v!5qe^|UM^gby{u{YoL9z=ZyMRY1 zfV^xWeQ`4o_xq;+C%a|`Tf?p4g29`w$2FFTwl$Q6t;M|_GMr0zB){by(&+iXt~y_<8ql-rPcZMJd;?VLtNxM z&08pd44uzsIKQzMmRk+8)@(LfD`u@#t6R%Ut$BW5J;)Y!5RJHtoVDh$gW8C0XO8;) zvr_-u2xCWBO6s*q3z!A{XTDeV+{{radN0Nlad+gQP(FZ+85~wLOG>fI6zQ!~IMuvX zA(kXUjw8)wr$GMSbrql{0cq#wca{3B0(dZp)NvKcsN%Z{SRVsUu%2mDy{@v=Dnvsf zq{dE3k%H4bU4fP*C>Pf$Il0m%W112pn?OwHDaRzm67UxVUp6)_;#WhR6a?vmwM4or$TYa?M8S@@|dXN zs|u!uN6|uOhE4a7R~`sf}x>f8yu1OJ<@ z-wD3$wmU~(ckzDbe)m)VUGMIp*9kiZ_ilH;xViq;!N11dhYx@K``SP6{6LO>{A7Rc bnafO1 z)%(3y_5SbQPn4fud&@AKvdnF^Qbjd*?Vec)H|t0ml`N!~mMX<01J>y88Z@*DvT>=R z>4-e8Xh>IWX>|W=$BH_{RkFelWMfM+CdqHrR7jjXrUJ2M8sO=@1u(7QD%e#7E}cc) zs;SawT9#X}&e%FKCdKHwDa0N(G1LJ~5ZKxw1T+Jbn;6+ax_;oe5V#I4tkHZ*b{#1t zLS~6;24-0o$)BJw1{@oy0GnXi(r|me2_Y<0Riu_|1J>=TiN&z{CBxBm64=drA~=?! zzm!676=L8(w zfDCLkL9l)WUP|Dh8O_Gzl1qIecl!IHQ_})fMLNRdl>3B^dE25hGXj&f3+<`!X)@(N z%YsevH|dZX5eR$Ut|%)l=g6{7At|PTp=Q`6zf*i6l}%Ge(4cMLDrwwuNQ+khG@4$| zSD_6cpql{86HGzfY46m$P!Wf@p~E#v0h4t}FDNv+FnhO8JSU`}o72cDl}V3s^|2oH z1kCtP26_eVF5vD071IKclue~iZ)OFsI8~f_eHCDLSO4I6YoZmCNb}W*#(dG{TCA|5 zQ>#HDJz5HaN|hhyiuK4@;UK-KW0})E6(1ftp5*EJ2#Sx3^NlG!I*f@uDo>nff!3qr z0+~mprO5T%?pmQ@sz_N+T1bZ&4F@xv+}H!zIQgXHbRwRTWjUWqrZStzebpe|I)yBo zxX3_!lcu0DvL<~;{ct1GKQTf_p_bx#t=j@}NdLa?b$YJvs1|BCnuUlvl29lZKvEwL zOPj@`#5*a>TYGS#c`ZjI7=n!>!DV|u{@-;uAjtq;=LdJ?`7H-jplGYK~`%dndAAj-ABs!7hi^Lj_xd*rM?Tp;oLYS85B>cw zs;3SWV%D?o7LQ+g`?jwOyT3bmv@x)2K6CZun~#s&G4sN!j}E=I`S4(3{;@y5x_bJq z&-Puv-uU#wf!NQ#F1>Nh$7{o`kfm;YU!yWyj`fw!dkw str + + logging.info("Running query '{}'...".format(query)) + result = instance.query(query, stdin=stdin, settings=settings) + logging.info("Query finished") + + return result + + +def test_create_query(started_cluster): + instance = started_cluster.instances["main_server"] + bucket = started_cluster.minio_bucket + + create_query = f"""CREATE TABLE iceberg ENGINE=Iceberg('http://{started_cluster.minio_ip}:{started_cluster.minio_port}/{bucket}/taxis/', 'minio', 'minio123')""" + + run_query(instance, create_query) + + +def test_select_query(started_cluster): + instance = started_cluster.instances["main_server"] + bucket = started_cluster.minio_bucket + columns = [ + "vendor_id", + "trip_id", + "trip_distance", + "fare_amount", + "store_and_fwd_flag", + ] + + # create query in case table doesn't exist + create_query = f"""CREATE TABLE IF NOT EXISTS iceberg ENGINE=Iceberg('http://{started_cluster.minio_ip}:{started_cluster.minio_port}/{bucket}/taxis/', 'minio', 'minio123')""" + + run_query(instance, create_query) + + select_query = "SELECT {} FROM iceberg FORMAT TSV" + select_table_function_query = "SELECT {col} FROM iceberg('http://{ip}:{port}/{bucket}/taxis/', 'minio', 'minio123') FORMAT TSV" + + for column_name in columns: + result = run_query(instance, select_query.format(column_name)).splitlines() + assert len(result) > 0 + + for column_name in columns: + result = run_query( + instance, + select_table_function_query.format( + col=column_name, + ip=started_cluster.minio_ip, + port=started_cluster.minio_port, + bucket=bucket, + ), + ).splitlines() + assert len(result) > 0 + + +def test_describe_query(started_cluster): + instance = started_cluster.instances["main_server"] + bucket = started_cluster.minio_bucket + result = instance.query( + f"DESCRIBE iceberg('http://{started_cluster.minio_ip}:{started_cluster.minio_port}/{bucket}/taxis/', 'minio', 'minio123') FORMAT TSV", + ) + + assert result == TSV( + [ + ["vendor_id", "Nullable(Int64)"], + ["trip_id", "Int64"], + ["trip_distance", "Nullable(Float32)"], + ["fare_amount", "Nullable(Float64)"], + ["store_and_fwd_flag", "Nullable(String)"], + ] + ) From 4dcd3cc4f676c0f7aca181a18e8e9fcc2f41b901 Mon Sep 17 00:00:00 2001 From: flynn Date: Wed, 18 Jan 2023 08:46:06 +0000 Subject: [PATCH 039/717] fix style --- src/Processors/Formats/Impl/AvroRowInputFormat.h | 4 ++-- src/Storages/StorageIceberg.cpp | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/Processors/Formats/Impl/AvroRowInputFormat.h b/src/Processors/Formats/Impl/AvroRowInputFormat.h index cef792da546..d2b4b924b04 100644 --- a/src/Processors/Formats/Impl/AvroRowInputFormat.h +++ b/src/Processors/Formats/Impl/AvroRowInputFormat.h @@ -37,9 +37,9 @@ public: void backup(size_t len) override; - void skip(size_t len) override; + void skip(size_t len) override; - size_t byteCount() const override; + size_t byteCount() const override; private: ReadBuffer & in; diff --git a/src/Storages/StorageIceberg.cpp b/src/Storages/StorageIceberg.cpp index aa02e920abf..8a71f2c13a5 100644 --- a/src/Storages/StorageIceberg.cpp +++ b/src/Storages/StorageIceberg.cpp @@ -42,7 +42,6 @@ namespace ErrorCodes { extern const int S3_ERROR; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; - extern const int INCORRECT_DATA; extern const int FILE_DOESNT_EXIST; extern const int ILLEGAL_COLUMN; } @@ -230,7 +229,7 @@ std::vector IcebergMetaParser::getFilesForRead(const std::vector for (size_t i = 0; i < col_size; ++i) { auto file_path = str_col->getDataAt(i).toView(); - /// We just obtain the parition/file name + /// We just obtain the partition/file name std::filesystem::path path(file_path); keys.emplace_back(path.parent_path().filename() / path.filename()); } From 01ae8f584b912cc3dd55d21495f2bb4b1502f809 Mon Sep 17 00:00:00 2001 From: flynn Date: Wed, 18 Jan 2023 11:34:52 +0000 Subject: [PATCH 040/717] fix test --- tests/integration/test_storage_iceberg/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_storage_iceberg/test.py b/tests/integration/test_storage_iceberg/test.py index fab07787331..c610bdc49d6 100644 --- a/tests/integration/test_storage_iceberg/test.py +++ b/tests/integration/test_storage_iceberg/test.py @@ -150,7 +150,7 @@ def test_describe_query(started_cluster): assert result == TSV( [ ["vendor_id", "Nullable(Int64)"], - ["trip_id", "Int64"], + ["trip_id", "Nullable(Int64)"], ["trip_distance", "Nullable(Float32)"], ["fare_amount", "Nullable(Float64)"], ["store_and_fwd_flag", "Nullable(String)"], From c5ea27446ee15362ba16158511a6e501c7d087ac Mon Sep 17 00:00:00 2001 From: Pradeep Chhetri Date: Sat, 21 Jan 2023 01:38:59 +0800 Subject: [PATCH 041/717] Fix query in stress script Signed-off-by: Pradeep Chhetri --- docker/test/stress/stress | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/stress/stress b/docker/test/stress/stress index 4afd2745526..0539ba2ce83 100755 --- a/docker/test/stress/stress +++ b/docker/test/stress/stress @@ -200,7 +200,7 @@ def prepare_for_hung_check(drop_databases): # Even if all clickhouse-test processes are finished, there are probably some sh scripts, # which still run some new queries. Let's ignore them. try: - query = """clickhouse client -q "SELECT count() FROM system.processes where where elapsed > 300" """ + query = """clickhouse client -q "SELECT count() FROM system.processes where elapsed > 300" """ output = ( check_output(query, shell=True, stderr=STDOUT, timeout=30) .decode("utf-8") From d07b7ce1eee020c1c1ae118a0b044d8ece950eb4 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sun, 22 Jan 2023 16:23:35 +0000 Subject: [PATCH 042/717] Some fixups --- src/Common/getNumberOfPhysicalCPUCores.cpp | 51 ++++++++++++---------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/src/Common/getNumberOfPhysicalCPUCores.cpp b/src/Common/getNumberOfPhysicalCPUCores.cpp index 86fff6732e0..ee02e7b7bfd 100644 --- a/src/Common/getNumberOfPhysicalCPUCores.cpp +++ b/src/Common/getNumberOfPhysicalCPUCores.cpp @@ -37,21 +37,25 @@ static uint32_t getCGroupLimitedCPUCores(unsigned default_cpu_count) return std::min(default_cpu_count, quota_count); } +#endif -/// Returns number of physical cores. If Intel hyper-threading (2-way SMP) is enabled, then getPhysicalConcurrency() returns half of of -/// std::thread::hardware_concurrency(), otherwise both values are the same. -static uint32_t physical_concurrency() +/// Returns number of physical cores, unlike std::thread::hardware_concurrency() which returns the logical core count. With 2-way SMT +/// (HyperThreading) enabled, physical_concurrency() returns half of of std::thread::hardware_concurrency(), otherwise return the same. +static unsigned physical_concurrency() +#if defined(OS_LINUX) try { - /// Derive physical cores from /proc/cpuinfo. Unfortunately, the CPUID instruction isn't reliable accross different vendors / CPU - /// models. Implementation based on boost::thread::physical_concurrency(). See - /// https://doc.callmematthi.eu/static/webArticles/Understanding%20Linux%20_proc_cpuinfo.pdf for semantics of /proc/cpuinfo in the - /// presence of multiple cores per socket, multiple sockets and multiple hyperthreading cores per physical core. + /// The CPUID instruction isn't reliable across different vendors and CPU models. The best option to get the physical core count is + /// to parse /proc/cpuinfo. boost::thread::physical_concurrency() does the same, so use their implementation. + /// + /// See https://doc.callmematthi.eu/static/webArticles/Understanding%20Linux%20_proc_cpuinfo.pdf std::ifstream proc_cpuinfo("/proc/cpuinfo"); - using CoreEntry = std::pair; + using CoreEntry = std::pair; /// physical id, core id + using CoreEntrySet = std::set; + + CoreEntrySet core_entries; - std::set core_entries; CoreEntry cur_core_entry; std::string line; @@ -64,45 +68,44 @@ try std::string key = line.substr(0, pos); std::string val = line.substr(pos + 1); - boost::trim(key); - boost::trim(val); - - if (key == "physical id") + if (key.find("physical id") != std::string::npos) { cur_core_entry.first = std::stoi(val); continue; } - if (key == "core id") + if (key.find("core id") != std::string::npos) { cur_core_entry.second = std::stoi(val); core_entries.insert(cur_core_entry); continue; } } - return core_entries.empty() ? /*unexpected format*/ std::thread::hardware_concurrency() : static_cast(core_entries.size()); + return core_entries.empty() ? /*unexpected format*/ std::thread::hardware_concurrency() : static_cast(core_entries.size()); } catch (...) { - /// parsing error + return std::thread::hardware_concurrency(); /// parsing error +} +#else +{ return std::thread::hardware_concurrency(); } #endif static unsigned getNumberOfPhysicalCPUCoresImpl() { - unsigned cpu_count = std::thread::hardware_concurrency(); /// logical cores + unsigned cpu_count = std::thread::hardware_concurrency(); /// logical cores (with SMT/HyperThreading) - /// Most x86_64 CPUs have 2-way Hyper-Threading + /// Most x86_64 CPUs have 2-way SMT (Hyper-Threading). /// Aarch64 and RISC-V don't have SMT so far. - /// POWER has SMT and it can be multiple way (like 8-way), but we don't know how ClickHouse really behaves, so use all of them. + /// POWER has SMT and it can be multi-way (e.g. 8-way), but we don't know how ClickHouse really behaves, so use all of them. #if defined(__x86_64__) - /// On big machines, Hyper-Threading is detrimental to performance (+ ~5% overhead in ClickBench). - /// Let's limit ourself to of physical cores. - /// For few cores - maybe it is a small machine, or runs in a VM or is a limited cloud instance --> it is reasonable to use all the cores. - if (cpu_count >= 32) - cpu_count = physical_concurrency(); + /// On really big machines, SMT is detrimental to performance (+ ~5% overhead in ClickBench). On such machines, we limit ourself to the physical cores. + /// Few cores indicate it is a small machine, runs in a VM or is a limited cloud instance --> it is reasonable to use all the cores. + if (cpu_count >= 32) + cpu_count = physical_concurrency(); #endif #if defined(OS_LINUX) From 1a1fa4a40ad354d55a0f42e68418310181d2f035 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Mon, 23 Jan 2023 19:48:09 +0000 Subject: [PATCH 043/717] Fix ARM build --- src/Common/getNumberOfPhysicalCPUCores.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/getNumberOfPhysicalCPUCores.cpp b/src/Common/getNumberOfPhysicalCPUCores.cpp index ee02e7b7bfd..2c7a7efa418 100644 --- a/src/Common/getNumberOfPhysicalCPUCores.cpp +++ b/src/Common/getNumberOfPhysicalCPUCores.cpp @@ -101,7 +101,7 @@ static unsigned getNumberOfPhysicalCPUCoresImpl() /// Aarch64 and RISC-V don't have SMT so far. /// POWER has SMT and it can be multi-way (e.g. 8-way), but we don't know how ClickHouse really behaves, so use all of them. -#if defined(__x86_64__) +#if defined(__x86_64__) && defined(OS_LINUX) /// On really big machines, SMT is detrimental to performance (+ ~5% overhead in ClickBench). On such machines, we limit ourself to the physical cores. /// Few cores indicate it is a small machine, runs in a VM or is a limited cloud instance --> it is reasonable to use all the cores. if (cpu_count >= 32) From 36d6af0e04cc05ebcaf661b493b0322941cc8e94 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 24 Jan 2023 09:57:12 +0000 Subject: [PATCH 044/717] Yet another build fix --- src/Common/getNumberOfPhysicalCPUCores.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Common/getNumberOfPhysicalCPUCores.cpp b/src/Common/getNumberOfPhysicalCPUCores.cpp index 2c7a7efa418..60f0e5a1a42 100644 --- a/src/Common/getNumberOfPhysicalCPUCores.cpp +++ b/src/Common/getNumberOfPhysicalCPUCores.cpp @@ -11,8 +11,10 @@ #include #include +namespace { + #if defined(OS_LINUX) -static int32_t readFrom(const char * filename, int default_value) +int32_t readFrom(const char * filename, int default_value) { std::ifstream infile(filename); if (!infile.is_open()) @@ -25,7 +27,7 @@ static int32_t readFrom(const char * filename, int default_value) } /// Try to look at cgroups limit if it is available. -static uint32_t getCGroupLimitedCPUCores(unsigned default_cpu_count) +uint32_t getCGroupLimitedCPUCores(unsigned default_cpu_count) { uint32_t quota_count = default_cpu_count; /// Return the number of milliseconds per period process is guaranteed to run. @@ -41,8 +43,8 @@ static uint32_t getCGroupLimitedCPUCores(unsigned default_cpu_count) /// Returns number of physical cores, unlike std::thread::hardware_concurrency() which returns the logical core count. With 2-way SMT /// (HyperThreading) enabled, physical_concurrency() returns half of of std::thread::hardware_concurrency(), otherwise return the same. -static unsigned physical_concurrency() -#if defined(OS_LINUX) +#if defined(__x86_64__) && defined(OS_LINUX) +unsigned physical_concurrency() try { /// The CPUID instruction isn't reliable across different vendors and CPU models. The best option to get the physical core count is @@ -87,13 +89,9 @@ catch (...) { return std::thread::hardware_concurrency(); /// parsing error } -#else -{ - return std::thread::hardware_concurrency(); -} #endif -static unsigned getNumberOfPhysicalCPUCoresImpl() +unsigned getNumberOfPhysicalCPUCoresImpl() { unsigned cpu_count = std::thread::hardware_concurrency(); /// logical cores (with SMT/HyperThreading) @@ -115,6 +113,8 @@ static unsigned getNumberOfPhysicalCPUCoresImpl() return cpu_count; } +} + unsigned getNumberOfPhysicalCPUCores() { /// Calculate once. From 094e42d1dc24dacad84526d442092c8ee41cc940 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 24 Jan 2023 12:57:56 +0000 Subject: [PATCH 045/717] Fix style --- src/Common/getNumberOfPhysicalCPUCores.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Common/getNumberOfPhysicalCPUCores.cpp b/src/Common/getNumberOfPhysicalCPUCores.cpp index 60f0e5a1a42..5c6640dc509 100644 --- a/src/Common/getNumberOfPhysicalCPUCores.cpp +++ b/src/Common/getNumberOfPhysicalCPUCores.cpp @@ -11,7 +11,8 @@ #include #include -namespace { +namespace +{ #if defined(OS_LINUX) int32_t readFrom(const char * filename, int default_value) From 1202f12d01bc9268be93b2f5f6c9d624962c183c Mon Sep 17 00:00:00 2001 From: Pradeep Chhetri Date: Wed, 25 Jan 2023 22:29:23 +0800 Subject: [PATCH 046/717] Add join_algorithm='grace_hash' to stress tests --- docker/test/stress/stress | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docker/test/stress/stress b/docker/test/stress/stress index 4afd2745526..49454b86e64 100755 --- a/docker/test/stress/stress +++ b/docker/test/stress/stress @@ -30,13 +30,15 @@ def get_options(i, backward_compatibility_check): if i % 2 == 1: join_alg_num = i // 2 - if join_alg_num % 4 == 0: + if join_alg_num % 5 == 0: client_options.append("join_algorithm='parallel_hash'") - if join_alg_num % 4 == 1: + if join_alg_num % 5 == 1: client_options.append("join_algorithm='partial_merge'") - if join_alg_num % 4 == 2: + if join_alg_num % 5 == 2: client_options.append("join_algorithm='full_sorting_merge'") - if join_alg_num % 4 == 3: + if join_alg_num % 5 == 3: + client_options.append("join_algorithm='grace_hash'") + if join_alg_num % 5 == 4: client_options.append("join_algorithm='auto'") client_options.append('max_rows_in_join=1000') From dd5acf9caf5cf64262bac5ef16dd86e711abef51 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Wed, 25 Jan 2023 17:09:26 +0000 Subject: [PATCH 047/717] Update --- .../Optimizations/removeRedundantDistinct.cpp | 30 ++++++++++++--- .../02500_remove_redundant_distinct.reference | 38 ++++++++----------- .../02500_remove_redundant_distinct.sql | 6 +-- 3 files changed, 43 insertions(+), 31 deletions(-) diff --git a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp index 5976d8238e0..45491052734 100644 --- a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp +++ b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp @@ -1,14 +1,25 @@ #include #include -#include #include -#include #include -#include "Processors/QueryPlan/ExpressionStep.h" +#include namespace DB::QueryPlanOptimizations { +// template +// FMT_NODISCARD FMT_INLINE auto format(format_string fmt, T&&... args) +constexpr bool debug_logging_enabled = true; + +template +void logDebug(const char * format, Args &&... args) +{ + if constexpr (debug_logging_enabled) + { + LOG_DEBUG(&Poco::Logger::get("redundantDistinct"), format, args...); + } +} + static std::set getDistinctColumns(const DistinctStep * distinct) { /// find non-const columns in DISTINCT @@ -30,7 +41,7 @@ size_t tryRemoveRedundantDistinct(QueryPlan::Node * parent_node, QueryPlan::Node /// check if it is preliminary distinct node QueryPlan::Node * distinct_node = nullptr; DistinctStep * distinct_step = typeid_cast(parent_node->children.front()->step.get()); - if (!distinct_step || !distinct_step->isPreliminary()) + if (!distinct_step) return 0; distinct_node = parent_node->children.front(); @@ -47,7 +58,16 @@ size_t tryRemoveRedundantDistinct(QueryPlan::Node * parent_node, QueryPlan::Node if (!inner_distinct_step) return 0; - if (getDistinctColumns(inner_distinct_step) != getDistinctColumns(inner_distinct_step)) + /// possible cases (outer distinct -> inner distinct): + /// final -> preliminary => do nothing + /// preliminary -> final => try remove preliminary + /// final -> final => try remove final + /// preliminary -> preliminary => logical error? + if (inner_distinct_step->isPreliminary()) + return 0; + + /// try to remove outer distinct step + if (getDistinctColumns(distinct_step) != getDistinctColumns(inner_distinct_step)) return 0; chassert(!distinct_node->children.empty()); diff --git a/tests/queries/0_stateless/02500_remove_redundant_distinct.reference b/tests/queries/0_stateless/02500_remove_redundant_distinct.reference index 71cd9d2e51d..dae4f737580 100644 --- a/tests/queries/0_stateless/02500_remove_redundant_distinct.reference +++ b/tests/queries/0_stateless/02500_remove_redundant_distinct.reference @@ -1,24 +1,16 @@ -Expression (Project names) -Header: number UInt64 - Distinct (Preliminary DISTINCT) - Header: number_0 UInt64 - Expression ((Projection + (Change column names to column identifiers + Project names))) - Header: number_0 UInt64 - Distinct (Preliminary DISTINCT) - Header: number_1 UInt64 - Expression ((Projection + (Change column names to column identifiers + Project names))) - Header: number_1 UInt64 +Expression (Projection) + Distinct + Distinct (Preliminary DISTINCT) + Expression ((Before ORDER BY + Projection)) + Distinct Distinct (Preliminary DISTINCT) - Header: number_2 UInt64 - Expression ((Projection + Change column names to column identifiers)) - Header: number_2 UInt64 - ReadFromStorage (SystemNumbers) - Header: number UInt64 -Expression ((Project names + (Projection + (Change column names to column identifiers + (Project names + (Projection + (Change column names to column identifiers + Project names))))))) -Header: number UInt64 - Distinct (Preliminary DISTINCT) - Header: number_2 UInt64 - Expression ((Projection + Change column names to column identifiers)) - Header: number_2 UInt64 - ReadFromStorage (SystemNumbers) - Header: number UInt64 + Expression ((Before ORDER BY + Projection)) + Distinct + Distinct (Preliminary DISTINCT) + Expression (Before ORDER BY) + ReadFromStorage (SystemNumbers) +Expression ((Projection + (Before ORDER BY + (Projection + (Before ORDER BY + Projection))))) + Distinct + Distinct (Preliminary DISTINCT) + Expression (Before ORDER BY) + ReadFromStorage (SystemNumbers) diff --git a/tests/queries/0_stateless/02500_remove_redundant_distinct.sql b/tests/queries/0_stateless/02500_remove_redundant_distinct.sql index 1e9f8bca29e..766241292be 100644 --- a/tests/queries/0_stateless/02500_remove_redundant_distinct.sql +++ b/tests/queries/0_stateless/02500_remove_redundant_distinct.sql @@ -1,9 +1,9 @@ -set allow_experimental_analyzer=1; +-- set allow_experimental_analyzer=1; set optimize_duplicate_order_by_and_distinct = 0; set query_plan_remove_redundant_distinct=0; -EXPLAIN header=1 +EXPLAIN --header=1 SELECT DISTINCT * FROM ( @@ -16,7 +16,7 @@ FROM ); set query_plan_remove_redundant_distinct=1; -EXPLAIN header=1 +EXPLAIN --header=1 SELECT DISTINCT * FROM ( From 60a69d4689938818ce28b93f8f0e1270c7480bad Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Thu, 26 Jan 2023 21:53:08 +0000 Subject: [PATCH 048/717] Fix distinct after JOIN or UNION --- .../Optimizations/removeRedundantDistinct.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp index 45491052734..bed30c0fc21 100644 --- a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp +++ b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp @@ -1,8 +1,10 @@ #include #include +#include #include -#include +#include #include +#include namespace DB::QueryPlanOptimizations { @@ -46,10 +48,16 @@ size_t tryRemoveRedundantDistinct(QueryPlan::Node * parent_node, QueryPlan::Node distinct_node = parent_node->children.front(); - DistinctStep * inner_distinct_step = nullptr; + const DistinctStep * inner_distinct_step = nullptr; QueryPlan::Node * node = distinct_node; while (!node->children.empty()) { + const IQueryPlanStep* current_step = node->step.get(); + + /// don't try to remove DISTINCT after union or join + if (typeid_cast(current_step) || typeid_cast(current_step)) + break; + node = node->children.front(); inner_distinct_step = typeid_cast(node->step.get()); if (inner_distinct_step) From 095a7488c461ecb3b388a3dde78b4e0611629eb9 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Thu, 26 Jan 2023 22:15:02 +0000 Subject: [PATCH 049/717] Change sql test to sh --- .../02500_remove_redundant_distinct.reference | 78 +++++++++++++++++++ .../02500_remove_redundant_distinct.sh | 59 ++++++++++++++ .../02500_remove_redundant_distinct.sql | 29 ------- 3 files changed, 137 insertions(+), 29 deletions(-) create mode 100755 tests/queries/0_stateless/02500_remove_redundant_distinct.sh delete mode 100644 tests/queries/0_stateless/02500_remove_redundant_distinct.sql diff --git a/tests/queries/0_stateless/02500_remove_redundant_distinct.reference b/tests/queries/0_stateless/02500_remove_redundant_distinct.reference index dae4f737580..bb1480153db 100644 --- a/tests/queries/0_stateless/02500_remove_redundant_distinct.reference +++ b/tests/queries/0_stateless/02500_remove_redundant_distinct.reference @@ -1,3 +1,4 @@ +-- Disabled query_plan_remove_redundant_distinct Expression (Projection) Distinct Distinct (Preliminary DISTINCT) @@ -9,8 +10,85 @@ Expression (Projection) Distinct (Preliminary DISTINCT) Expression (Before ORDER BY) ReadFromStorage (SystemNumbers) +-- Enabled query_plan_remove_redundant_distinct +-- DISTINCT is only in most inner subquery +-- query +SELECT DISTINCT * +FROM +( + SELECT DISTINCT * + FROM + ( + SELECT DISTINCT * + FROM numbers(3) + ) +) +-- explain Expression ((Projection + (Before ORDER BY + (Projection + (Before ORDER BY + Projection))))) Distinct Distinct (Preliminary DISTINCT) Expression (Before ORDER BY) ReadFromStorage (SystemNumbers) +-- execute +0 +1 +2 +-- do _not_ remove DISTINCT after UNION +-- query +SELECT DISTINCT number FROM +( + (SELECT DISTINCT number FROM numbers(1)) + UNION ALL + (SELECT DISTINCT number FROM numbers(2)) +) +ORDER BY number +-- explain +Expression (Projection) + Distinct + Sorting (Sorting for ORDER BY) + Distinct (Preliminary DISTINCT) + Union + Expression ((Before ORDER BY + Projection)) + Distinct + Distinct (Preliminary DISTINCT) + Expression (Before ORDER BY) + ReadFromStorage (SystemNumbers) + Expression (( + Projection)) + Distinct + Distinct (Preliminary DISTINCT) + Expression (Before ORDER BY) + ReadFromStorage (SystemNumbers) +-- execute +0 +1 +-- do _not_ remove DISTINCT after JOIN +-- query +SELECT DISTINCT * +FROM +( + SELECT DISTINCT number AS n + FROM numbers(2) +), +( + SELECT DISTINCT number AS n + FROM numbers(2) +) SETTINGS joined_subquery_requires_alias=0 +-- explain +Expression (Projection) + Distinct + Distinct (Preliminary DISTINCT) + Expression (Before ORDER BY) + Join (JOIN FillRightFirst) + Expression ((Before JOIN + Projection)) + Distinct + Distinct (Preliminary DISTINCT) + Expression (Before ORDER BY) + ReadFromStorage (SystemNumbers) + Expression ((Joined actions + (Rename joined columns + Projection))) + Distinct + Distinct (Preliminary DISTINCT) + Expression (Before ORDER BY) + ReadFromStorage (SystemNumbers) +-- execute +0 0 +1 1 diff --git a/tests/queries/0_stateless/02500_remove_redundant_distinct.sh b/tests/queries/0_stateless/02500_remove_redundant_distinct.sh new file mode 100755 index 00000000000..2788ded9e2d --- /dev/null +++ b/tests/queries/0_stateless/02500_remove_redundant_distinct.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +OPTIMIZATION_SETTING="query_plan_remove_redundant_distinct" +DISABLE_OPTIMIZATION="SET $OPTIMIZATION_SETTING=0;SET optimize_duplicate_order_by_and_distinct=0" +ENABLE_OPTIMIZATION="SET $OPTIMIZATION_SETTING=1;SET optimize_duplicate_order_by_and_distinct=0" + +echo "-- Disabled $OPTIMIZATION_SETTING" +query="SELECT DISTINCT * +FROM +( + SELECT DISTINCT * + FROM + ( + SELECT DISTINCT * + FROM numbers(3) + ) +)" + +$CLICKHOUSE_CLIENT -nq "$DISABLE_OPTIMIZATION;EXPLAIN $query" + +function run_query { + echo "-- query" + echo "$1" + echo "-- explain" + $CLICKHOUSE_CLIENT -nq "$ENABLE_OPTIMIZATION;EXPLAIN $1" + echo "-- execute" + $CLICKHOUSE_CLIENT -nq "$ENABLE_OPTIMIZATION;$1" +} + +echo "-- Enabled $OPTIMIZATION_SETTING" +echo "-- DISTINCT is only in most inner subquery" +run_query "$query" + +echo "-- do _not_ remove DISTINCT after UNION" +query="SELECT DISTINCT number FROM +( + (SELECT DISTINCT number FROM numbers(1)) + UNION ALL + (SELECT DISTINCT number FROM numbers(2)) +) +ORDER BY number" +run_query "$query" + +echo "-- do _not_ remove DISTINCT after JOIN" +query="SELECT DISTINCT * +FROM +( + SELECT DISTINCT number AS n + FROM numbers(2) +), +( + SELECT DISTINCT number AS n + FROM numbers(2) +) SETTINGS joined_subquery_requires_alias=0" +run_query "$query" diff --git a/tests/queries/0_stateless/02500_remove_redundant_distinct.sql b/tests/queries/0_stateless/02500_remove_redundant_distinct.sql deleted file mode 100644 index 766241292be..00000000000 --- a/tests/queries/0_stateless/02500_remove_redundant_distinct.sql +++ /dev/null @@ -1,29 +0,0 @@ --- set allow_experimental_analyzer=1; -set optimize_duplicate_order_by_and_distinct = 0; - -set query_plan_remove_redundant_distinct=0; - -EXPLAIN --header=1 -SELECT DISTINCT * -FROM -( - SELECT DISTINCT * - FROM - ( - SELECT DISTINCT * - FROM numbers(3) - ) -); - -set query_plan_remove_redundant_distinct=1; -EXPLAIN --header=1 -SELECT DISTINCT * -FROM -( - SELECT DISTINCT * - FROM - ( - SELECT DISTINCT * - FROM numbers(3) - ) -); From 4c08561430ca621b8f1f2ff02750551c9e0ebee4 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Thu, 26 Jan 2023 22:32:22 +0000 Subject: [PATCH 050/717] Fix test --- ...ate_order_by_and_distinct_optimize_for_distributed_table.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01306_disable_duplicate_order_by_and_distinct_optimize_for_distributed_table.sql b/tests/queries/0_stateless/01306_disable_duplicate_order_by_and_distinct_optimize_for_distributed_table.sql index dffcf24684e..06aa8672831 100644 --- a/tests/queries/0_stateless/01306_disable_duplicate_order_by_and_distinct_optimize_for_distributed_table.sql +++ b/tests/queries/0_stateless/01306_disable_duplicate_order_by_and_distinct_optimize_for_distributed_table.sql @@ -1,5 +1,6 @@ -- Tags: distributed +set query_plan_remove_redundant_distinct = 0; set optimize_duplicate_order_by_and_distinct = 1; SET distributed_group_by_no_merge = 0; @@ -35,7 +36,6 @@ FROM ); SET distributed_group_by_no_merge = 1; - set optimize_duplicate_order_by_and_distinct = 0; SELECT DISTINCT number FROM From c5d07ddfeceaa54e5e9254cda2a6478fe6d1665e Mon Sep 17 00:00:00 2001 From: flynn Date: Sun, 29 Jan 2023 07:03:00 +0000 Subject: [PATCH 051/717] fix --- src/IO/S3Common.cpp | 50 +++++++++++++++++++++++++--- src/IO/S3Common.h | 2 ++ src/Storages/StorageDeltaLake.cpp | 55 ++++--------------------------- src/Storages/StorageIceberg.cpp | 47 ++++---------------------- 4 files changed, 62 insertions(+), 92 deletions(-) diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp index a18fcf70566..d944ac06993 100644 --- a/src/IO/S3Common.cpp +++ b/src/IO/S3Common.cpp @@ -18,18 +18,19 @@ # include # include # include +# include # include # include +# include +# include # include # include # include -# include -# include -# include # include # include # include # include +# include # include # include @@ -1062,8 +1063,49 @@ namespace S3 "Failed to get metadata of key {} in bucket {}: {}", key, bucket, error.GetMessage()); } -} + std::vector + listFiles(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & prefix, const String & extension) + { + std::vector res; + Aws::S3::Model::ListObjectsV2Request request; + Aws::S3::Model::ListObjectsV2Outcome outcome; + + bool is_finished{false}; + + request.SetBucket(bucket); + + request.SetPrefix(prefix); + + while (!is_finished) + { + outcome = client.ListObjectsV2(request); + if (!outcome.IsSuccess()) + throw Exception( + ErrorCodes::S3_ERROR, + "Could not list objects in bucket {} with key {}, S3 exception: {}, message: {}", + quoteString(bucket), + quoteString(key), + backQuote(outcome.GetError().GetExceptionName()), + quoteString(outcome.GetError().GetMessage())); + + const auto & result_batch = outcome.GetResult().GetContents(); + for (const auto & obj : result_batch) + { + const auto & filename = obj.GetKey(); + + if (std::filesystem::path(filename).extension() == extension) + res.push_back(filename); + } + + request.SetContinuationToken(outcome.GetResult().GetNextContinuationToken()); + + is_finished = !outcome.GetResult().GetIsTruncated(); + } + + return res; + } +} } #endif diff --git a/src/IO/S3Common.h b/src/IO/S3Common.h index 69ae1cbb4f4..37073696823 100644 --- a/src/IO/S3Common.h +++ b/src/IO/S3Common.h @@ -144,6 +144,8 @@ bool isNotFoundError(Aws::S3::S3Errors error); /// Returns the object's metadata. std::map getObjectMetadata(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id = "", bool for_disk_s3 = false, bool throw_on_error = true); +std::vector +listFiles(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & prefix, const String & extension); } #endif diff --git a/src/Storages/StorageDeltaLake.cpp b/src/Storages/StorageDeltaLake.cpp index 479a11b5eb4..7df4829ad13 100644 --- a/src/Storages/StorageDeltaLake.cpp +++ b/src/Storages/StorageDeltaLake.cpp @@ -97,62 +97,21 @@ void JsonMetadataGetter::init(ContextPtr context) std::vector JsonMetadataGetter::getJsonLogFiles() { - std::vector keys; - - const auto & client = base_configuration.client; - - Aws::S3::Model::ListObjectsV2Request request; - Aws::S3::Model::ListObjectsV2Outcome outcome; - - bool is_finished{false}; - const auto bucket{base_configuration.uri.bucket}; - - request.SetBucket(bucket); - /// DeltaLake format stores all metadata json files in _delta_log directory static constexpr auto deltalake_metadata_directory = "_delta_log"; - request.SetPrefix(std::filesystem::path(table_path) / deltalake_metadata_directory); - while (!is_finished) - { - outcome = client->ListObjectsV2(request); - if (!outcome.IsSuccess()) - throw Exception( - ErrorCodes::S3_ERROR, - "Could not list objects in bucket {} with key {}, S3 exception: {}, message: {}", - quoteString(bucket), - quoteString(table_path), - backQuote(outcome.GetError().GetExceptionName()), - quoteString(outcome.GetError().GetMessage())); - - const auto & result_batch = outcome.GetResult().GetContents(); - for (const auto & obj : result_batch) - { - const auto & filename = obj.GetKey(); - - // DeltaLake metadata files have json extension - if (std::filesystem::path(filename).extension() == ".json") - keys.push_back(filename); - } - - /// Needed in case any more results are available - /// if so, we will continue reading, and not read keys that were already read - request.SetContinuationToken(outcome.GetResult().GetNextContinuationToken()); - - /// Set to false if all of the results were returned. Set to true if more keys - /// are available to return. If the number of results exceeds that specified by - /// MaxKeys, all of the results might not be returned - is_finished = !outcome.GetResult().GetIsTruncated(); - } - - return keys; + return S3::listFiles( + *base_configuration.client, + base_configuration.uri.bucket, + table_path, + std::filesystem::path(table_path) / deltalake_metadata_directory, + ".json"); } std::shared_ptr JsonMetadataGetter::createS3ReadBuffer(const String & key, ContextPtr context) { - /// TODO: add parallel downloads S3Settings::RequestSettings request_settings; - request_settings.max_single_read_retries = 10; + request_settings.max_single_read_retries = context->getSettingsRef().s3_max_single_read_retries; return std::make_shared( base_configuration.client, base_configuration.uri.bucket, diff --git a/src/Storages/StorageIceberg.cpp b/src/Storages/StorageIceberg.cpp index 8a71f2c13a5..482443ef783 100644 --- a/src/Storages/StorageIceberg.cpp +++ b/src/Storages/StorageIceberg.cpp @@ -71,45 +71,12 @@ String IcebergMetaParser::getNewestMetaFile() const /// Iceberg stores all the metadata.json in metadata directory, and the /// newest version has the max version name, so we should list all of them /// then find the newest metadata. - std::vector metadata_files; - - const auto & client = base_configuration.client; - - Aws::S3::Model::ListObjectsV2Request request; - Aws::S3::Model::ListObjectsV2Outcome outcome; - - bool is_finished{false}; - const auto bucket{base_configuration.uri.bucket}; - - request.SetBucket(bucket); - - request.SetPrefix(std::filesystem::path(table_path) / metadata_directory); - - while (!is_finished) - { - outcome = client->ListObjectsV2(request); - if (!outcome.IsSuccess()) - throw Exception( - ErrorCodes::S3_ERROR, - "Could not list objects in bucket {} with key {}, S3 exception: {}, message: {}", - quoteString(bucket), - quoteString(table_path), - backQuote(outcome.GetError().GetExceptionName()), - quoteString(outcome.GetError().GetMessage())); - - const auto & result_batch = outcome.GetResult().GetContents(); - for (const auto & obj : result_batch) - { - const auto & filename = obj.GetKey(); - - if (std::filesystem::path(filename).extension() == ".json") - metadata_files.push_back(filename); - } - - request.SetContinuationToken(outcome.GetResult().GetNextContinuationToken()); - - is_finished = !outcome.GetResult().GetIsTruncated(); - } + std::vector metadata_files = S3::listFiles( + *base_configuration.client, + base_configuration.uri.bucket, + table_path, + std::filesystem::path(table_path) / metadata_directory, + ".json"); if (metadata_files.empty()) throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "The metadata file for Iceberg table with path {} doesn't exist", table_path); @@ -257,7 +224,7 @@ std::vector IcebergMetaParser::getFilesForRead(const std::vector std::shared_ptr IcebergMetaParser::createS3ReadBuffer(const String & key) const { S3Settings::RequestSettings request_settings; - request_settings.max_single_read_retries = 10; + request_settings.max_single_read_retries = context->getSettingsRef().s3_max_single_read_retries; return std::make_shared( base_configuration.client, base_configuration.uri.bucket, From ed6fcfe0d1a6385d4e9d69de04e4391c7c0e21c3 Mon Sep 17 00:00:00 2001 From: flynn Date: Sun, 29 Jan 2023 08:28:48 +0000 Subject: [PATCH 052/717] refactor --- src/TableFunctions/ITableFunctionDataLake.h | 178 ++++++++++++++++++ src/TableFunctions/TableFunctionDeltaLake.cpp | 147 +-------------- src/TableFunctions/TableFunctionDeltaLake.h | 44 ----- src/TableFunctions/TableFunctionHudi.cpp | 147 +-------------- src/TableFunctions/TableFunctionIceberg.cpp | 146 +------------- src/TableFunctions/TableFunctionIceberg.h | 44 ----- 6 files changed, 193 insertions(+), 513 deletions(-) create mode 100644 src/TableFunctions/ITableFunctionDataLake.h delete mode 100644 src/TableFunctions/TableFunctionDeltaLake.h delete mode 100644 src/TableFunctions/TableFunctionIceberg.h diff --git a/src/TableFunctions/ITableFunctionDataLake.h b/src/TableFunctions/ITableFunctionDataLake.h new file mode 100644 index 00000000000..0f3f36e64ff --- /dev/null +++ b/src/TableFunctions/ITableFunctionDataLake.h @@ -0,0 +1,178 @@ +#pragma once + +#include "config.h" + +#if USE_AWS_S3 + +#include +#include + +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + +class Context; +class TableFunctionS3Cluster; + +template +class ITableFunctionDataLake : public ITableFunction +{ +public: + static constexpr auto name = Name::name; + std::string getName() const override + { + return name; + } + +protected: + StoragePtr + executeImpl(const ASTPtr & /*ast_function*/, ContextPtr context, const std::string & table_name, ColumnsDescription /*cached_columns*/) + const override + { + S3::URI s3_uri(configuration.url); + + ColumnsDescription columns; + if (configuration.structure != "auto") + columns = parseColumnsListFromString(configuration.structure, context); + + StoragePtr storage = std::make_shared( + configuration, StorageID(getDatabaseName(), table_name), columns, ConstraintsDescription{}, String{}, context, std::nullopt); + + storage->startup(); + + return storage; + } + + const char * getStorageTypeName() const override { return name; } + + ColumnsDescription getActualTableStructure(ContextPtr context) const override + { + if (configuration.structure == "auto") + { + context->checkAccess(getSourceAccessType()); + return Storage::getTableStructureFromData(configuration, std::nullopt, context); + } + + return parseColumnsListFromString(configuration.structure, context); + } + + void parseArguments(const ASTPtr & ast_function, ContextPtr context) override + { + /// Parse args + ASTs & args_func = ast_function->children; + + const auto message = fmt::format( + "The signature of table function {} could be the following:\n" + " - url\n" + " - url, format\n" + " - url, format, structure\n" + " - url, access_key_id, secret_access_key\n" + " - url, format, structure, compression_method\n" + " - url, access_key_id, secret_access_key, format\n" + " - url, access_key_id, secret_access_key, format, structure\n" + " - url, access_key_id, secret_access_key, format, structure, compression_method", + getName()); + + if (args_func.size() != 1) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Table function '{}' must have arguments", getName()); + + auto & args = args_func.at(0)->children; + + parseArgumentsImpl(message, args, context, configuration); + } + + + static void + parseArgumentsImpl(const String & error_message, ASTs & args, ContextPtr context, StorageS3Configuration & base_configuration) + { + if (args.empty() || args.size() > 6) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, error_message); + + auto * header_it = StorageURL::collectHeaders(args, base_configuration.headers, context); + if (header_it != args.end()) + args.erase(header_it); + + for (auto & arg : args) + arg = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context); + + /// Size -> argument indexes + static auto size_to_args = std::map>{ + {1, {{}}}, + {2, {{"format", 1}}}, + {5, {{"access_key_id", 1}, {"secret_access_key", 2}, {"format", 3}, {"structure", 4}}}, + {6, {{"access_key_id", 1}, {"secret_access_key", 2}, {"format", 3}, {"structure", 4}, {"compression_method", 5}}}}; + + std::map args_to_idx; + /// For 4 arguments we support 2 possible variants: + /// dataLake(source, format, structure, compression_method) and iceberg(source, access_key_id, access_key_id, format) + /// We can distinguish them by looking at the 2-nd argument: check if it's a format name or not. + if (args.size() == 4) + { + auto second_arg = checkAndGetLiteralArgument(args[1], "format/access_key_id"); + if (second_arg == "auto" || FormatFactory::instance().getAllFormats().contains(second_arg)) + args_to_idx = {{"format", 1}, {"structure", 2}, {"compression_method", 3}}; + + else + args_to_idx = {{"access_key_id", 1}, {"secret_access_key", 2}, {"format", 3}}; + } + /// For 3 arguments we support 2 possible variants: + /// dataLake(source, format, structure) and iceberg(source, access_key_id, access_key_id) + /// We can distinguish them by looking at the 2-nd argument: check if it's a format name or not. + else if (args.size() == 3) + { + auto second_arg = checkAndGetLiteralArgument(args[1], "format/access_key_id"); + if (second_arg == "auto" || FormatFactory::instance().getAllFormats().contains(second_arg)) + args_to_idx = {{"format", 1}, {"structure", 2}}; + else + args_to_idx = {{"access_key_id", 1}, {"secret_access_key", 2}}; + } + else + { + args_to_idx = size_to_args[args.size()]; + } + + /// This argument is always the first + base_configuration.url = checkAndGetLiteralArgument(args[0], "url"); + + if (args_to_idx.contains("format")) + base_configuration.format = checkAndGetLiteralArgument(args[args_to_idx["format"]], "format"); + else + base_configuration.format = "Parquet"; + + if (args_to_idx.contains("structure")) + base_configuration.structure = checkAndGetLiteralArgument(args[args_to_idx["structure"]], "structure"); + + if (args_to_idx.contains("compression_method")) + base_configuration.compression_method + = checkAndGetLiteralArgument(args[args_to_idx["compression_method"]], "compression_method"); + + if (args_to_idx.contains("access_key_id")) + base_configuration.auth_settings.access_key_id + = checkAndGetLiteralArgument(args[args_to_idx["access_key_id"]], "access_key_id"); + + if (args_to_idx.contains("secret_access_key")) + base_configuration.auth_settings.secret_access_key + = checkAndGetLiteralArgument(args[args_to_idx["secret_access_key"]], "secret_access_key"); + } + + StorageS3Configuration configuration; + }; +} + +#endif diff --git a/src/TableFunctions/TableFunctionDeltaLake.cpp b/src/TableFunctions/TableFunctionDeltaLake.cpp index e72de953858..d968c58bf94 100644 --- a/src/TableFunctions/TableFunctionDeltaLake.cpp +++ b/src/TableFunctions/TableFunctionDeltaLake.cpp @@ -2,157 +2,20 @@ #if USE_AWS_S3 -# include -# include -# include -# include -# include -# include -# include -# include # include -# include -# include -# include +# include # include # include "registerTableFunctions.h" - namespace DB { -namespace ErrorCodes +struct TableFunctionDeltaLakeName { - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; -} - - -void TableFunctionDeltaLake::parseArgumentsImpl( - const String & error_message, ASTs & args, ContextPtr context, StorageS3Configuration & base_configuration) -{ - if (args.empty() || args.size() > 6) - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, error_message); - - auto * header_it = StorageURL::collectHeaders(args, base_configuration.headers, context); - if (header_it != args.end()) - args.erase(header_it); - - for (auto & arg : args) - arg = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context); - - /// Size -> argument indexes - static auto size_to_args = std::map>{ - {1, {{}}}, - {2, {{"format", 1}}}, - {5, {{"access_key_id", 1}, {"secret_access_key", 2}, {"format", 3}, {"structure", 4}}}, - {6, {{"access_key_id", 1}, {"secret_access_key", 2}, {"format", 3}, {"structure", 4}, {"compression_method", 5}}}}; - - std::map args_to_idx; - /// For 4 arguments we support 2 possible variants: - /// deltaLake(source, format, structure, compression_method) and deltaLake(source, access_key_id, access_key_id, format) - /// We can distinguish them by looking at the 2-nd argument: check if it's a format name or not. - if (args.size() == 4) - { - auto second_arg = checkAndGetLiteralArgument(args[1], "format/access_key_id"); - if (second_arg == "auto" || FormatFactory::instance().getAllFormats().contains(second_arg)) - args_to_idx = {{"format", 1}, {"structure", 2}, {"compression_method", 3}}; - - else - args_to_idx = {{"access_key_id", 1}, {"secret_access_key", 2}, {"format", 3}}; - } - /// For 3 arguments we support 2 possible variants: - /// deltaLake(source, format, structure) and deltaLake(source, access_key_id, access_key_id) - /// We can distinguish them by looking at the 2-nd argument: check if it's a format name or not. - else if (args.size() == 3) - { - auto second_arg = checkAndGetLiteralArgument(args[1], "format/access_key_id"); - if (second_arg == "auto" || FormatFactory::instance().getAllFormats().contains(second_arg)) - args_to_idx = {{"format", 1}, {"structure", 2}}; - else - args_to_idx = {{"access_key_id", 1}, {"secret_access_key", 2}}; - } - else - { - args_to_idx = size_to_args[args.size()]; - } - - /// This argument is always the first - base_configuration.url = checkAndGetLiteralArgument(args[0], "url"); - - if (args_to_idx.contains("format")) - base_configuration.format = checkAndGetLiteralArgument(args[args_to_idx["format"]], "format"); - else - base_configuration.format = "Parquet"; - - if (args_to_idx.contains("structure")) - base_configuration.structure = checkAndGetLiteralArgument(args[args_to_idx["structure"]], "structure"); - - if (args_to_idx.contains("compression_method")) - base_configuration.compression_method - = checkAndGetLiteralArgument(args[args_to_idx["compression_method"]], "compression_method"); - - if (args_to_idx.contains("access_key_id")) - base_configuration.auth_settings.access_key_id - = checkAndGetLiteralArgument(args[args_to_idx["access_key_id"]], "access_key_id"); - - if (args_to_idx.contains("secret_access_key")) - base_configuration.auth_settings.secret_access_key - = checkAndGetLiteralArgument(args[args_to_idx["secret_access_key"]], "secret_access_key"); -} - -void TableFunctionDeltaLake::parseArguments(const ASTPtr & ast_function, ContextPtr context) -{ - /// Parse args - ASTs & args_func = ast_function->children; - - const auto message = fmt::format( - "The signature of table function {} could be the following:\n" \ - " - url\n" \ - " - url, format\n" \ - " - url, format, structure\n" \ - " - url, access_key_id, secret_access_key\n" \ - " - url, format, structure, compression_method\n" \ - " - url, access_key_id, secret_access_key, format\n" \ - " - url, access_key_id, secret_access_key, format, structure\n" \ - " - url, access_key_id, secret_access_key, format, structure, compression_method", - getName()); - - if (args_func.size() != 1) - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Table function '{}' must have arguments", getName()); - - auto & args = args_func.at(0)->children; - - parseArgumentsImpl(message, args, context, configuration); -} - -ColumnsDescription TableFunctionDeltaLake::getActualTableStructure(ContextPtr context) const -{ - if (configuration.structure == "auto") - { - context->checkAccess(getSourceAccessType()); - return StorageDeltaLake::getTableStructureFromData(configuration, std::nullopt, context); - } - - return parseColumnsListFromString(configuration.structure, context); -} - -StoragePtr TableFunctionDeltaLake::executeImpl( - const ASTPtr & /*ast_function*/, ContextPtr context, const std::string & table_name, ColumnsDescription /*cached_columns*/) const -{ - S3::URI s3_uri(configuration.url); - - ColumnsDescription columns; - if (configuration.structure != "auto") - columns = parseColumnsListFromString(configuration.structure, context); - - StoragePtr storage = std::make_shared( - configuration, StorageID(getDatabaseName(), table_name), columns, ConstraintsDescription{}, String{}, context, std::nullopt); - - storage->startup(); - - return storage; -} + static constexpr auto name = "deltaLake"; +}; +using TableFunctionDeltaLake = ITableFunctionDataLake; void registerTableFunctionDeltaLake(TableFunctionFactory & factory) { diff --git a/src/TableFunctions/TableFunctionDeltaLake.h b/src/TableFunctions/TableFunctionDeltaLake.h deleted file mode 100644 index e36ffc3847f..00000000000 --- a/src/TableFunctions/TableFunctionDeltaLake.h +++ /dev/null @@ -1,44 +0,0 @@ -#pragma once - -#include "config.h" - -#if USE_AWS_S3 - -#include -#include - - -namespace DB -{ - -class Context; -class TableFunctionS3Cluster; - -/* deltaLake(source, [access_key_id, secret_access_key,] format, structure[, compression]) - creates a temporary DeltaLake table on S3. - */ -class TableFunctionDeltaLake : public ITableFunction -{ -public: - static constexpr auto name = "deltaLake"; - std::string getName() const override - { - return name; - } - -protected: - StoragePtr executeImpl( - const ASTPtr & ast_function, ContextPtr context, const std::string & table_name, ColumnsDescription cached_columns) const override; - - const char * getStorageTypeName() const override { return name; } - - ColumnsDescription getActualTableStructure(ContextPtr context) const override; - void parseArguments(const ASTPtr & ast_function, ContextPtr context) override; - - static void parseArgumentsImpl(const String & error_message, ASTs & args, ContextPtr context, StorageS3Configuration & configuration); - - StorageS3Configuration configuration; -}; - -} - -#endif diff --git a/src/TableFunctions/TableFunctionHudi.cpp b/src/TableFunctions/TableFunctionHudi.cpp index e7b085d0d4e..b6b2e072035 100644 --- a/src/TableFunctions/TableFunctionHudi.cpp +++ b/src/TableFunctions/TableFunctionHudi.cpp @@ -2,157 +2,20 @@ #if USE_AWS_S3 -# include -# include -# include -# include -# include -# include -# include -# include # include -# include -# include +# include # include -# include # include "registerTableFunctions.h" - namespace DB { -namespace ErrorCodes +struct TableFunctionHudiName { - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; -} - - -void TableFunctionHudi::parseArgumentsImpl( - const String & error_message, ASTs & args, ContextPtr context, StorageS3Configuration & base_configuration) -{ - if (args.empty() || args.size() > 6) - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, error_message); - - auto * header_it = StorageURL::collectHeaders(args, base_configuration.headers, context); - if (header_it != args.end()) - args.erase(header_it); - - for (auto & arg : args) - arg = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context); - - /// Size -> argument indexes - static auto size_to_args = std::map>{ - {1, {{}}}, - {2, {{"format", 1}}}, - {5, {{"access_key_id", 1}, {"secret_access_key", 2}, {"format", 3}, {"structure", 4}}}, - {6, {{"access_key_id", 1}, {"secret_access_key", 2}, {"format", 3}, {"structure", 4}, {"compression_method", 5}}}}; - - std::map args_to_idx; - /// For 4 arguments we support 2 possible variants: - /// hudi(source, format, structure, compression_method) and hudi(source, access_key_id, access_key_id, format) - /// We can distinguish them by looking at the 2-nd argument: check if it's a format name or not. - if (args.size() == 4) - { - auto second_arg = checkAndGetLiteralArgument(args[1], "format/access_key_id"); - if (second_arg == "auto" || FormatFactory::instance().getAllFormats().contains(second_arg)) - args_to_idx = {{"format", 1}, {"structure", 2}, {"compression_method", 3}}; - - else - args_to_idx = {{"access_key_id", 1}, {"secret_access_key", 2}, {"format", 3}}; - } - /// For 3 arguments we support 2 possible variants: - /// hudi(source, format, structure) and hudi(source, access_key_id, access_key_id) - /// We can distinguish them by looking at the 2-nd argument: check if it's a format name or not. - else if (args.size() == 3) - { - auto second_arg = checkAndGetLiteralArgument(args[1], "format/access_key_id"); - if (second_arg == "auto" || FormatFactory::instance().getAllFormats().contains(second_arg)) - args_to_idx = {{"format", 1}, {"structure", 2}}; - else - args_to_idx = {{"access_key_id", 1}, {"secret_access_key", 2}}; - } - else - { - args_to_idx = size_to_args[args.size()]; - } - - /// This argument is always the first - base_configuration.url = checkAndGetLiteralArgument(args[0], "url"); - - if (args_to_idx.contains("format")) - base_configuration.format = checkAndGetLiteralArgument(args[args_to_idx["format"]], "format"); - else - base_configuration.format = "Parquet"; - - if (args_to_idx.contains("structure")) - base_configuration.structure = checkAndGetLiteralArgument(args[args_to_idx["structure"]], "structure"); - - if (args_to_idx.contains("compression_method")) - base_configuration.compression_method - = checkAndGetLiteralArgument(args[args_to_idx["compression_method"]], "compression_method"); - - if (args_to_idx.contains("access_key_id")) - base_configuration.auth_settings.access_key_id - = checkAndGetLiteralArgument(args[args_to_idx["access_key_id"]], "access_key_id"); - - if (args_to_idx.contains("secret_access_key")) - base_configuration.auth_settings.secret_access_key - = checkAndGetLiteralArgument(args[args_to_idx["secret_access_key"]], "secret_access_key"); -} - -void TableFunctionHudi::parseArguments(const ASTPtr & ast_function, ContextPtr context) -{ - /// Parse args - ASTs & args_func = ast_function->children; - - const auto message = fmt::format( - "The signature of table function {} could be the following:\n" \ - " - url\n" \ - " - url, format\n" \ - " - url, format, structure\n" \ - " - url, access_key_id, secret_access_key\n" \ - " - url, format, structure, compression_method\n" \ - " - url, access_key_id, secret_access_key, format\n" \ - " - url, access_key_id, secret_access_key, format, structure\n" \ - " - url, access_key_id, secret_access_key, format, structure, compression_method", - getName()); - - if (args_func.size() != 1) - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Table function '{}' must have arguments", getName()); - - auto & args = args_func.at(0)->children; - - parseArgumentsImpl(message, args, context, configuration); -} - -ColumnsDescription TableFunctionHudi::getActualTableStructure(ContextPtr context) const -{ - if (configuration.structure == "auto") - { - context->checkAccess(getSourceAccessType()); - return StorageHudi::getTableStructureFromData(configuration, std::nullopt, context); - } - - return parseColumnsListFromString(configuration.structure, context); -} - -StoragePtr TableFunctionHudi::executeImpl( - const ASTPtr & /*ast_function*/, ContextPtr context, const std::string & table_name, ColumnsDescription /*cached_columns*/) const -{ - S3::URI s3_uri(configuration.url); - - ColumnsDescription columns; - if (configuration.structure != "auto") - columns = parseColumnsListFromString(configuration.structure, context); - - StoragePtr storage = std::make_shared( - configuration, StorageID(getDatabaseName(), table_name), columns, ConstraintsDescription{}, String{}, context, std::nullopt); - - storage->startup(); - - return storage; -} + static constexpr auto name = "hudi"; +}; +using TableFunctionHudi = ITableFunctionDataLake; void registerTableFunctionHudi(TableFunctionFactory & factory) { diff --git a/src/TableFunctions/TableFunctionIceberg.cpp b/src/TableFunctions/TableFunctionIceberg.cpp index 7e4ad8b13be..2555b197d30 100644 --- a/src/TableFunctions/TableFunctionIceberg.cpp +++ b/src/TableFunctions/TableFunctionIceberg.cpp @@ -2,157 +2,21 @@ #if USE_AWS_S3 -# include -# include -# include -# include -# include -# include -# include -# include # include -# include -# include +# include # include -# include # include "registerTableFunctions.h" namespace DB { -namespace ErrorCodes +struct TableFunctionIcebergName { - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; -} - - -void TableFunctionIceberg::parseArgumentsImpl( - const String & error_message, ASTs & args, ContextPtr context, StorageS3Configuration & base_configuration) -{ - if (args.empty() || args.size() > 6) - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, error_message); - - auto * header_it = StorageURL::collectHeaders(args, base_configuration.headers, context); - if (header_it != args.end()) - args.erase(header_it); - - for (auto & arg : args) - arg = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context); - - /// Size -> argument indexes - static auto size_to_args = std::map>{ - {1, {{}}}, - {2, {{"format", 1}}}, - {5, {{"access_key_id", 1}, {"secret_access_key", 2}, {"format", 3}, {"structure", 4}}}, - {6, {{"access_key_id", 1}, {"secret_access_key", 2}, {"format", 3}, {"structure", 4}, {"compression_method", 5}}}}; - - std::map args_to_idx; - /// For 4 arguments we support 2 possible variants: - /// iceberg(source, format, structure, compression_method) and iceberg(source, access_key_id, access_key_id, format) - /// We can distinguish them by looking at the 2-nd argument: check if it's a format name or not. - if (args.size() == 4) - { - auto second_arg = checkAndGetLiteralArgument(args[1], "format/access_key_id"); - if (second_arg == "auto" || FormatFactory::instance().getAllFormats().contains(second_arg)) - args_to_idx = {{"format", 1}, {"structure", 2}, {"compression_method", 3}}; - - else - args_to_idx = {{"access_key_id", 1}, {"secret_access_key", 2}, {"format", 3}}; - } - /// For 3 arguments we support 2 possible variants: - /// iceberg(source, format, structure) and iceberg(source, access_key_id, access_key_id) - /// We can distinguish them by looking at the 2-nd argument: check if it's a format name or not. - else if (args.size() == 3) - { - auto second_arg = checkAndGetLiteralArgument(args[1], "format/access_key_id"); - if (second_arg == "auto" || FormatFactory::instance().getAllFormats().contains(second_arg)) - args_to_idx = {{"format", 1}, {"structure", 2}}; - else - args_to_idx = {{"access_key_id", 1}, {"secret_access_key", 2}}; - } - else - { - args_to_idx = size_to_args[args.size()]; - } - - /// This argument is always the first - base_configuration.url = checkAndGetLiteralArgument(args[0], "url"); - - if (args_to_idx.contains("format")) - base_configuration.format = checkAndGetLiteralArgument(args[args_to_idx["format"]], "format"); - else - base_configuration.format = "Parquet"; - - if (args_to_idx.contains("structure")) - base_configuration.structure = checkAndGetLiteralArgument(args[args_to_idx["structure"]], "structure"); - - if (args_to_idx.contains("compression_method")) - base_configuration.compression_method - = checkAndGetLiteralArgument(args[args_to_idx["compression_method"]], "compression_method"); - - if (args_to_idx.contains("access_key_id")) - base_configuration.auth_settings.access_key_id - = checkAndGetLiteralArgument(args[args_to_idx["access_key_id"]], "access_key_id"); - - if (args_to_idx.contains("secret_access_key")) - base_configuration.auth_settings.secret_access_key - = checkAndGetLiteralArgument(args[args_to_idx["secret_access_key"]], "secret_access_key"); -} - -void TableFunctionIceberg::parseArguments(const ASTPtr & ast_function, ContextPtr context) -{ - /// Parse args - ASTs & args_func = ast_function->children; - - const auto message = fmt::format( - "The signature of table function {} could be the following:\n" \ - " - url\n" \ - " - url, format\n" \ - " - url, format, structure\n" \ - " - url, access_key_id, secret_access_key\n" \ - " - url, format, structure, compression_method\n" \ - " - url, access_key_id, secret_access_key, format\n" \ - " - url, access_key_id, secret_access_key, format, structure\n" \ - " - url, access_key_id, secret_access_key, format, structure, compression_method", - getName()); - - if (args_func.size() != 1) - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Table function '{}' must have arguments", getName()); - - auto & args = args_func.at(0)->children; - - parseArgumentsImpl(message, args, context, configuration); -} - -ColumnsDescription TableFunctionIceberg::getActualTableStructure(ContextPtr context) const -{ - if (configuration.structure == "auto") - { - context->checkAccess(getSourceAccessType()); - return StorageIceberg::getTableStructureFromData(configuration, std::nullopt, context); - } - - return parseColumnsListFromString(configuration.structure, context); -} - -StoragePtr TableFunctionIceberg::executeImpl( - const ASTPtr & /*ast_function*/, ContextPtr context, const std::string & table_name, ColumnsDescription /*cached_columns*/) const -{ - S3::URI s3_uri(configuration.url); - - ColumnsDescription columns; - if (configuration.structure != "auto") - columns = parseColumnsListFromString(configuration.structure, context); - - StoragePtr storage = std::make_shared( - configuration, StorageID(getDatabaseName(), table_name), columns, ConstraintsDescription{}, String{}, context, std::nullopt); - - storage->startup(); - - return storage; -} + static constexpr auto name = "iceberg"; +}; +using TableFunctionIceberg = ITableFunctionDataLake; void registerTableFunctionIceberg(TableFunctionFactory & factory) { diff --git a/src/TableFunctions/TableFunctionIceberg.h b/src/TableFunctions/TableFunctionIceberg.h deleted file mode 100644 index 49df80d0e10..00000000000 --- a/src/TableFunctions/TableFunctionIceberg.h +++ /dev/null @@ -1,44 +0,0 @@ -#pragma once - -#include "config.h" - -#if USE_AWS_S3 - -#include -#include - - -namespace DB -{ - -class Context; -class TableFunctionS3Cluster; - -/* iceberg(source, [access_key_id, secret_access_key,] format, structure[, compression]) - creates a temporary Iceberg table on S3. - */ -class TableFunctionIceberg : public ITableFunction -{ -public: - static constexpr auto name = "iceberg"; - std::string getName() const override - { - return name; - } - -protected: - StoragePtr executeImpl( - const ASTPtr & ast_function, ContextPtr context, const std::string & table_name, ColumnsDescription cached_columns) const override; - - const char * getStorageTypeName() const override { return name; } - - ColumnsDescription getActualTableStructure(ContextPtr context) const override; - void parseArguments(const ASTPtr & ast_function, ContextPtr context) override; - - static void parseArgumentsImpl(const String & error_message, ASTs & args, ContextPtr context, StorageS3Configuration & configuration); - - StorageS3Configuration configuration; -}; - -} - -#endif From 6d9dc8306448f3e8176f8a52e69e549c1be725b9 Mon Sep 17 00:00:00 2001 From: flynn Date: Sun, 29 Jan 2023 10:15:13 +0000 Subject: [PATCH 053/717] fix --- src/TableFunctions/ITableFunctionDataLake.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/TableFunctions/ITableFunctionDataLake.h b/src/TableFunctions/ITableFunctionDataLake.h index 622c0f8f60d..5277d27aa7b 100644 --- a/src/TableFunctions/ITableFunctionDataLake.h +++ b/src/TableFunctions/ITableFunctionDataLake.h @@ -15,7 +15,6 @@ # include # include # include -# include # include # include @@ -51,7 +50,7 @@ protected: if (configuration.structure != "auto") columns = parseColumnsListFromString(configuration.structure, context); - StoragePtr storage = std::make_shared( + StoragePtr storage = std::make_shared( configuration, StorageID(getDatabaseName(), table_name), columns, ConstraintsDescription{}, String{}, context, std::nullopt); storage->startup(); From fc2ce9e8e210613d18fe6bce52095e864fce3314 Mon Sep 17 00:00:00 2001 From: flynn Date: Sun, 29 Jan 2023 14:53:56 +0000 Subject: [PATCH 054/717] refactor and unify storage data lake --- src/Storages/IStorageDataLake.h | 261 ++++++++++++++++++++++++++++++ src/Storages/StorageDeltaLake.cpp | 139 ++-------------- src/Storages/StorageDeltaLake.h | 52 +----- src/Storages/StorageHudi.cpp | 126 +-------------- src/Storages/StorageHudi.h | 49 ++---- src/Storages/StorageIceberg.cpp | 130 +-------------- src/Storages/StorageIceberg.h | 44 +---- src/Storages/StorageS3.h | 30 ++-- 8 files changed, 331 insertions(+), 500 deletions(-) create mode 100644 src/Storages/IStorageDataLake.h diff --git a/src/Storages/IStorageDataLake.h b/src/Storages/IStorageDataLake.h new file mode 100644 index 00000000000..f97f949e540 --- /dev/null +++ b/src/Storages/IStorageDataLake.h @@ -0,0 +1,261 @@ +#pragma once + +#include "config.h" + +#if USE_AWS_S3 + +# include + +# include +# include +# include + +# include +# include +# include +# include +# include +# include + +# include +# include +# include +# include + +# include + +# include +# include +# include + +# include + +# include +# include + +# include +# include + +namespace Aws::S3 +{ +class S3Client; +} + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + +static const std::unordered_set required_configuration_keys = { + "url", +}; +static const std::unordered_set optional_configuration_keys = { + "format", + "compression", + "compression_method", + "structure", + "access_key_id", + "secret_access_key", +}; + +template +class IStorageDataLake : public IStorage +{ +public: + // 1. Parses internal file structure of table + // 2. Finds out parts with latest version + // 3. Creates url for underlying StorageS3 enigne to handle reads + IStorageDataLake( + const StorageS3Configuration & configuration_, + const StorageID & table_id_, + ColumnsDescription columns_, + const ConstraintsDescription & constraints_, + const String & comment, + ContextPtr context_, + std::optional format_settings_) + : IStorage(table_id_) + , base_configuration{getBaseConfiguration(configuration_)} + , log(&Poco::Logger::get("Storage" + String(name) + "(" + table_id_.table_name + ")")) + , table_path(base_configuration.uri.key) + { + StorageInMemoryMetadata storage_metadata; + StorageS3::updateS3Configuration(context_, base_configuration); + + auto new_configuration = getAdjustedS3Configuration(context_, base_configuration, configuration_, table_path, log); + + if (columns_.empty()) + { + columns_ = StorageS3::getTableStructureFromData( + new_configuration, /*distributed processing*/ false, format_settings_, context_, nullptr); + storage_metadata.setColumns(columns_); + } + else + storage_metadata.setColumns(columns_); + + storage_metadata.setConstraints(constraints_); + storage_metadata.setComment(comment); + setInMemoryMetadata(storage_metadata); + + s3engine = std::make_shared( + new_configuration, + table_id_, + columns_, + constraints_, + comment, + context_, + format_settings_, + /* distributed_processing_ */ false, + nullptr); + } + + static constexpr auto name = Name::name; + String getName() const override { return name; } + + // Reads latest version of Iceberg table + Pipe read( + const Names & column_names, + const StorageSnapshotPtr & storage_snapshot, + SelectQueryInfo & query_info, + ContextPtr context, + QueryProcessingStage::Enum processed_stage, + size_t max_block_size, + size_t num_streams) override + { + StorageS3::updateS3Configuration(context, base_configuration); + + return s3engine->read(column_names, storage_snapshot, query_info, context, processed_stage, max_block_size, num_streams); + } + + static ColumnsDescription getTableStructureFromData( + const StorageS3Configuration & configuration, const std::optional & format_settings, ContextPtr ctx) + { + auto base_configuration = getBaseConfiguration(configuration); + StorageS3::updateS3Configuration(ctx, base_configuration); + auto new_configuration = getAdjustedS3Configuration( + ctx, base_configuration, configuration, base_configuration.uri.key, &Poco::Logger::get("Storage" + String(name))); + return StorageS3::getTableStructureFromData( + new_configuration, /*distributed processing*/ false, format_settings, ctx, /*object_infos*/ nullptr); + } + + static StorageS3::S3Configuration getBaseConfiguration(const StorageS3Configuration & configuration) + { + return {configuration.url, configuration.auth_settings, configuration.request_settings, configuration.headers}; +} + + static StorageS3Configuration getAdjustedS3Configuration( + const ContextPtr & context, + StorageS3::S3Configuration & base_configuration, + const StorageS3Configuration & configuration, + const std::string & table_path, + Poco::Logger * log) + { + MetaParser parser{base_configuration, table_path, context}; + + auto keys = parser.getFiles(); + static constexpr auto iceberg_data_directory = "data"; + auto new_uri = std::filesystem::path(base_configuration.uri.uri.toString()) / iceberg_data_directory + / MetaParser::generateQueryFromKeys(keys, configuration.format); + + LOG_DEBUG(log, "New uri: {}", new_uri.c_str()); + LOG_DEBUG(log, "Table path: {}", table_path); + + // set new url in configuration + StorageS3Configuration new_configuration; + new_configuration.url = new_uri; + new_configuration.auth_settings.access_key_id = configuration.auth_settings.access_key_id; + new_configuration.auth_settings.secret_access_key = configuration.auth_settings.secret_access_key; + new_configuration.format = configuration.format; + + return new_configuration; + } + + static void processNamedCollectionResult(StorageS3Configuration & configuration, const NamedCollection & collection) + { + validateNamedCollection(collection, required_configuration_keys, optional_configuration_keys); + + configuration.url = collection.get("url"); + + configuration.auth_settings.access_key_id = collection.getOrDefault("access_key_id", ""); + configuration.auth_settings.secret_access_key = collection.getOrDefault("secret_access_key", ""); + + configuration.format = collection.getOrDefault("format", "Parquet"); + + configuration.compression_method + = collection.getOrDefault("compression_method", collection.getOrDefault("compression", "auto")); + + configuration.structure = collection.getOrDefault("structure", "auto"); + + configuration.request_settings = S3Settings::RequestSettings(collection); + } + + static StorageS3Configuration getConfiguration(ASTs & engine_args, ContextPtr local_context) + { + StorageS3Configuration configuration; + + if (auto named_collection = tryGetNamedCollectionWithOverrides(engine_args)) + { + processNamedCollectionResult(configuration, *named_collection); + } + else + { + /// Supported signatures: + /// + /// xx('url') + /// xx('url', 'format') + /// xx('url', 'format', 'compression') + /// xx('url', 'aws_access_key_id', 'aws_secret_access_key', 'format') + /// xx('url', 'aws_access_key_id', 'aws_secret_access_key', 'format', 'compression') + + if (engine_args.empty() || engine_args.size() > 5) + throw Exception( + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Storage {} requires 1 to 5 arguments: " + "url, [access_key_id, secret_access_key], name of used format and [compression_method]", + name); + + auto * header_it = StorageURL::collectHeaders(engine_args, configuration.headers, local_context); + if (header_it != engine_args.end()) + engine_args.erase(header_it); + + for (auto & engine_arg : engine_args) + engine_arg = evaluateConstantExpressionOrIdentifierAsLiteral(engine_arg, local_context); + + configuration.url = checkAndGetLiteralArgument(engine_args[0], "url"); + if (engine_args.size() >= 4) + { + configuration.auth_settings.access_key_id = checkAndGetLiteralArgument(engine_args[1], "access_key_id"); + configuration.auth_settings.secret_access_key = checkAndGetLiteralArgument(engine_args[2], "secret_access_key"); + } + + if (engine_args.size() == 3 || engine_args.size() == 5) + { + configuration.compression_method = checkAndGetLiteralArgument(engine_args.back(), "compression_method"); + configuration.format = checkAndGetLiteralArgument(engine_args[engine_args.size() - 2], "format"); + } + else if (engine_args.size() != 1) + { + configuration.compression_method = "auto"; + configuration.format = checkAndGetLiteralArgument(engine_args.back(), "format"); + } + } + + if (configuration.format == "auto") + configuration.format = "Parquet"; + + return configuration; + } + +private: + + StorageS3::S3Configuration base_configuration; + std::shared_ptr s3engine; + Poco::Logger * log; + String table_path; +}; + +} + +#endif diff --git a/src/Storages/StorageDeltaLake.cpp b/src/Storages/StorageDeltaLake.cpp index 7df4829ad13..08f262f8071 100644 --- a/src/Storages/StorageDeltaLake.cpp +++ b/src/Storages/StorageDeltaLake.cpp @@ -58,13 +58,13 @@ std::vector DeltaLakeMetadata::listCurrentFiles() && return keys; } -JsonMetadataGetter::JsonMetadataGetter(StorageS3::S3Configuration & configuration_, const String & table_path_, ContextPtr context) +DeltaLakeMetaParser::DeltaLakeMetaParser(StorageS3::S3Configuration & configuration_, const String & table_path_, ContextPtr context) : base_configuration(configuration_), table_path(table_path_) { init(context); } -void JsonMetadataGetter::init(ContextPtr context) +void DeltaLakeMetaParser::init(ContextPtr context) { auto keys = getJsonLogFiles(); @@ -95,7 +95,7 @@ void JsonMetadataGetter::init(ContextPtr context) } } -std::vector JsonMetadataGetter::getJsonLogFiles() +std::vector DeltaLakeMetaParser::getJsonLogFiles() { /// DeltaLake format stores all metadata json files in _delta_log directory static constexpr auto deltalake_metadata_directory = "_delta_log"; @@ -108,7 +108,7 @@ std::vector JsonMetadataGetter::getJsonLogFiles() ".json"); } -std::shared_ptr JsonMetadataGetter::createS3ReadBuffer(const String & key, ContextPtr context) +std::shared_ptr DeltaLakeMetaParser::createS3ReadBuffer(const String & key, ContextPtr context) { S3Settings::RequestSettings request_settings; request_settings.max_single_read_retries = context->getSettingsRef().s3_max_single_read_retries; @@ -121,7 +121,7 @@ std::shared_ptr JsonMetadataGetter::createS3ReadBuffer(const String context->getReadSettings()); } -void JsonMetadataGetter::handleJSON(const JSON & json) +void DeltaLakeMetaParser::handleJSON(const JSON & json) { if (json.has("add")) { @@ -139,149 +139,28 @@ void JsonMetadataGetter::handleJSON(const JSON & json) } } -namespace -{ - -StorageS3::S3Configuration getBaseConfiguration(const StorageS3Configuration & configuration) -{ - return {configuration.url, configuration.auth_settings, configuration.request_settings, configuration.headers}; -} - // DeltaLake stores data in parts in different files // keys is vector of parts with latest version // generateQueryFromKeys constructs query from parts filenames for // underlying StorageS3 engine -String generateQueryFromKeys(const std::vector & keys) +String DeltaLakeMetaParser::generateQueryFromKeys(const std::vector & keys, const String &) { std::string new_query = fmt::format("{{{}}}", fmt::join(keys, ",")); return new_query; } - -StorageS3Configuration getAdjustedS3Configuration( - const ContextPtr & context, - StorageS3::S3Configuration & base_configuration, - const StorageS3Configuration & configuration, - const std::string & table_path, - Poco::Logger * log) -{ - JsonMetadataGetter getter{base_configuration, table_path, context}; - - auto keys = getter.getFiles(); - auto new_uri = base_configuration.uri.uri.toString() + generateQueryFromKeys(keys); - - LOG_DEBUG(log, "New uri: {}", new_uri); - LOG_DEBUG(log, "Table path: {}", table_path); - - // set new url in configuration - StorageS3Configuration new_configuration; - new_configuration.url = new_uri; - new_configuration.auth_settings.access_key_id = configuration.auth_settings.access_key_id; - new_configuration.auth_settings.secret_access_key = configuration.auth_settings.secret_access_key; - new_configuration.format = configuration.format; - - return new_configuration; -} - -} - -StorageDeltaLake::StorageDeltaLake( - const StorageS3Configuration & configuration_, - const StorageID & table_id_, - ColumnsDescription columns_, - const ConstraintsDescription & constraints_, - const String & comment, - ContextPtr context_, - std::optional format_settings_) - : IStorage(table_id_) - , base_configuration{getBaseConfiguration(configuration_)} - , log(&Poco::Logger::get("StorageDeltaLake (" + table_id_.table_name + ")")) - , table_path(base_configuration.uri.key) -{ - StorageInMemoryMetadata storage_metadata; - StorageS3::updateS3Configuration(context_, base_configuration); - - auto new_configuration = getAdjustedS3Configuration(context_, base_configuration, configuration_, table_path, log); - - if (columns_.empty()) - { - columns_ = StorageS3::getTableStructureFromData( - new_configuration, /*distributed processing*/ false, format_settings_, context_, nullptr); - storage_metadata.setColumns(columns_); - } - else - storage_metadata.setColumns(columns_); - - - storage_metadata.setConstraints(constraints_); - storage_metadata.setComment(comment); - setInMemoryMetadata(storage_metadata); - - s3engine = std::make_shared( - new_configuration, - table_id_, - columns_, - constraints_, - comment, - context_, - format_settings_, - /* distributed_processing_ */ false, - nullptr); -} - -Pipe StorageDeltaLake::read( - const Names & column_names, - const StorageSnapshotPtr & storage_snapshot, - SelectQueryInfo & query_info, - ContextPtr context, - QueryProcessingStage::Enum processed_stage, - size_t max_block_size, - size_t num_streams) -{ - StorageS3::updateS3Configuration(context, base_configuration); - - return s3engine->read(column_names, storage_snapshot, query_info, context, processed_stage, max_block_size, num_streams); -} - -ColumnsDescription StorageDeltaLake::getTableStructureFromData( - const StorageS3Configuration & configuration, const std::optional & format_settings, ContextPtr ctx) -{ - auto base_configuration = getBaseConfiguration(configuration); - StorageS3::updateS3Configuration(ctx, base_configuration); - auto new_configuration = getAdjustedS3Configuration( - ctx, base_configuration, configuration, base_configuration.uri.key, &Poco::Logger::get("StorageDeltaLake")); - return StorageS3::getTableStructureFromData( - new_configuration, /*distributed processing*/ false, format_settings, ctx, /*object_infos*/ nullptr); -} - void registerStorageDeltaLake(StorageFactory & factory) { factory.registerStorage( "DeltaLake", [](const StorageFactory::Arguments & args) { - auto & engine_args = args.engine_args; - if (engine_args.empty() || engine_args.size() < 3) - throw Exception( - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, - "Storage DeltaLake requires 3 to 4 arguments: table_url, access_key, secret_access_key, [format]"); + StorageS3Configuration configuration = StorageDeltaLake::getConfiguration(args.engine_args, args.getLocalContext()); - StorageS3Configuration configuration; - - configuration.url = checkAndGetLiteralArgument(engine_args[0], "url"); - configuration.auth_settings.access_key_id = checkAndGetLiteralArgument(engine_args[1], "access_key_id"); - configuration.auth_settings.secret_access_key = checkAndGetLiteralArgument(engine_args[2], "secret_access_key"); - - if (engine_args.size() == 4) - configuration.format = checkAndGetLiteralArgument(engine_args[3], "format"); - else - { - /// DeltaLake uses Parquet by default. - configuration.format = "Parquet"; - } + auto format_settings = getFormatSettings(args.getContext()); return std::make_shared( - configuration, args.table_id, args.columns, args.constraints, args.comment, args.getContext(), std::nullopt); + configuration, args.table_id, args.columns, args.constraints, args.comment, args.getContext(), format_settings); }, { .supports_settings = true, diff --git a/src/Storages/StorageDeltaLake.h b/src/Storages/StorageDeltaLake.h index af6485b9a40..95b89ad9221 100644 --- a/src/Storages/StorageDeltaLake.h +++ b/src/Storages/StorageDeltaLake.h @@ -4,17 +4,11 @@ #if USE_AWS_S3 -# include +# include # include -# include # include -namespace Poco -{ -class Logger; -} - namespace Aws::S3 { class S3Client; @@ -39,13 +33,15 @@ private: }; // class to get deltalake log json files and read json from them -class JsonMetadataGetter +class DeltaLakeMetaParser { public: - JsonMetadataGetter(StorageS3::S3Configuration & configuration_, const String & table_path_, ContextPtr context); + DeltaLakeMetaParser(StorageS3::S3Configuration & configuration_, const String & table_path_, ContextPtr context); std::vector getFiles() { return std::move(metadata).listCurrentFiles(); } + static String generateQueryFromKeys(const std::vector & keys, const String & format); + private: void init(ContextPtr context); @@ -60,44 +56,12 @@ private: DeltaLakeMetadata metadata; }; -class StorageDeltaLake : public IStorage +struct StorageDeltaLakeName { -public: - // 1. Parses internal file structure of table - // 2. Finds out parts with latest version - // 3. Creates url for underlying StorageS3 enigne to handle reads - StorageDeltaLake( - const StorageS3Configuration & configuration_, - const StorageID & table_id_, - ColumnsDescription columns_, - const ConstraintsDescription & constraints_, - const String & comment, - ContextPtr context_, - std::optional format_settings_); - - String getName() const override { return "DeltaLake"; } - - // Reads latest version of DeltaLake table - Pipe read( - const Names & column_names, - const StorageSnapshotPtr & storage_snapshot, - SelectQueryInfo & query_info, - ContextPtr context, - QueryProcessingStage::Enum processed_stage, - size_t max_block_size, - size_t num_streams) override; - - static ColumnsDescription getTableStructureFromData( - const StorageS3Configuration & configuration, - const std::optional & format_settings, - ContextPtr ctx); -private: - StorageS3::S3Configuration base_configuration; - std::shared_ptr s3engine; - Poco::Logger * log; - String table_path; + static constexpr auto name = "DeltaLake"; }; +using StorageDeltaLake = IStorageDataLake; } #endif diff --git a/src/Storages/StorageHudi.cpp b/src/Storages/StorageHudi.cpp index d5675ceb17c..a8daee3d7cd 100644 --- a/src/Storages/StorageHudi.cpp +++ b/src/Storages/StorageHudi.cpp @@ -28,12 +28,9 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -namespace +HudiMetaParser::HudiMetaParser(const StorageS3::S3Configuration & configuration_, const String & table_path_, ContextPtr context_) + : configuration(configuration_), table_path(table_path_), context(context_), log(&Poco::Logger::get("StorageHudi")) { - -StorageS3::S3Configuration getBaseConfiguration(const StorageS3Configuration & configuration) -{ - return {configuration.url, configuration.auth_settings, configuration.request_settings, configuration.headers}; } /// Apache Hudi store parts of data in different files. @@ -41,7 +38,7 @@ StorageS3::S3Configuration getBaseConfiguration(const StorageS3Configuration & c /// Every partition(directory) in Apache Hudi has different versions of part. /// To find needed parts we need to find out latest part file for every partition. /// Part format is usually parquet, but can differ. -String generateQueryFromKeys(const std::vector & keys, const String & format) +String HudiMetaParser::generateQueryFromKeys(const std::vector & keys, const String & format) { /// For each partition path take only latest file. struct FileInfo @@ -92,17 +89,17 @@ String generateQueryFromKeys(const std::vector & keys, const String return "{" + list_of_keys + "}"; } -std::vector getKeysFromS3(const StorageS3::S3Configuration & base_configuration, const std::string & table_path, Poco::Logger * log) +std::vector HudiMetaParser::getFiles() const { std::vector keys; - const auto & client = base_configuration.client; + const auto & client = configuration.client; Aws::S3::Model::ListObjectsV2Request request; Aws::S3::Model::ListObjectsV2Outcome outcome; bool is_finished{false}; - const auto bucket{base_configuration.uri.bucket}; + const auto bucket{configuration.uri.bucket}; request.SetBucket(bucket); request.SetPrefix(table_path); @@ -134,122 +131,13 @@ std::vector getKeysFromS3(const StorageS3::S3Configuration & base_c return keys; } - -StorageS3Configuration getAdjustedS3Configuration( - StorageS3::S3Configuration & base_configuration, - const StorageS3Configuration & configuration, - const std::string & table_path, - Poco::Logger * log) -{ - auto keys = getKeysFromS3(base_configuration, table_path, log); - auto new_uri = base_configuration.uri.uri.toString() + generateQueryFromKeys(keys, configuration.format); - - LOG_DEBUG(log, "New uri: {}", new_uri); - LOG_DEBUG(log, "Table path: {}", table_path); - - StorageS3Configuration new_configuration; - new_configuration.url = new_uri; - new_configuration.auth_settings.access_key_id = configuration.auth_settings.access_key_id; - new_configuration.auth_settings.secret_access_key = configuration.auth_settings.secret_access_key; - new_configuration.format = configuration.format; - - return new_configuration; -} - -} - -StorageHudi::StorageHudi( - const StorageS3Configuration & configuration_, - const StorageID & table_id_, - ColumnsDescription columns_, - const ConstraintsDescription & constraints_, - const String & comment, - ContextPtr context_, - std::optional format_settings_) - : IStorage(table_id_) - , base_configuration{getBaseConfiguration(configuration_)} - , log(&Poco::Logger::get("StorageHudi (" + table_id_.table_name + ")")) - , table_path(base_configuration.uri.key) -{ - StorageInMemoryMetadata storage_metadata; - StorageS3::updateS3Configuration(context_, base_configuration); - - auto new_configuration = getAdjustedS3Configuration(base_configuration, configuration_, table_path, log); - - if (columns_.empty()) - { - columns_ = StorageS3::getTableStructureFromData( - new_configuration, /*distributed processing*/ false, format_settings_, context_, nullptr); - storage_metadata.setColumns(columns_); - } - else - storage_metadata.setColumns(columns_); - - storage_metadata.setConstraints(constraints_); - storage_metadata.setComment(comment); - setInMemoryMetadata(storage_metadata); - - s3engine = std::make_shared( - new_configuration, - table_id_, - columns_, - constraints_, - comment, - context_, - format_settings_, - /* distributed_processing_ */ false, - nullptr); -} - -Pipe StorageHudi::read( - const Names & column_names, - const StorageSnapshotPtr & storage_snapshot, - SelectQueryInfo & query_info, - ContextPtr context, - QueryProcessingStage::Enum processed_stage, - size_t max_block_size, - size_t num_streams) -{ - StorageS3::updateS3Configuration(context, base_configuration); - return s3engine->read(column_names, storage_snapshot, query_info, context, processed_stage, max_block_size, num_streams); -} - -ColumnsDescription StorageHudi::getTableStructureFromData( - const StorageS3Configuration & configuration, const std::optional & format_settings, ContextPtr ctx) -{ - auto base_configuration = getBaseConfiguration(configuration); - StorageS3::updateS3Configuration(ctx, base_configuration); - auto new_configuration = getAdjustedS3Configuration( - base_configuration, configuration, base_configuration.uri.key, &Poco::Logger::get("StorageDeltaLake")); - return StorageS3::getTableStructureFromData( - new_configuration, /*distributed processing*/ false, format_settings, ctx, /*object_infos*/ nullptr); -} - void registerStorageHudi(StorageFactory & factory) { factory.registerStorage( "Hudi", [](const StorageFactory::Arguments & args) { - auto & engine_args = args.engine_args; - if (engine_args.empty() || engine_args.size() < 3) - throw Exception( - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, - "Storage Hudi requires 3 to 4 arguments: table_url, access_key, secret_access_key, [format]"); - - StorageS3Configuration configuration; - - configuration.url = checkAndGetLiteralArgument(engine_args[0], "url"); - configuration.auth_settings.access_key_id = checkAndGetLiteralArgument(engine_args[1], "access_key_id"); - configuration.auth_settings.secret_access_key = checkAndGetLiteralArgument(engine_args[2], "secret_access_key"); - - if (engine_args.size() == 4) - configuration.format = checkAndGetLiteralArgument(engine_args[3], "format"); - else - { - // Apache Hudi uses Parquet by default - configuration.format = "Parquet"; - } + StorageS3Configuration configuration = StorageHudi::getConfiguration(args.engine_args, args.getLocalContext()); auto format_settings = getFormatSettings(args.getContext()); diff --git a/src/Storages/StorageHudi.h b/src/Storages/StorageHudi.h index 00b8c01a46d..b84cb45a4b3 100644 --- a/src/Storages/StorageHudi.h +++ b/src/Storages/StorageHudi.h @@ -5,13 +5,9 @@ #if USE_AWS_S3 # include +# include # include -namespace Poco -{ -class Logger; -} - namespace Aws::S3 { class S3Client; @@ -20,45 +16,28 @@ class S3Client; namespace DB { -class StorageHudi : public IStorage +class HudiMetaParser { public: - /// 1. Parses internal file structure of table. - /// 2. Finds out parts with latest version. - /// 3. Creates url for underlying StorageS3 enigne to handle reads. - StorageHudi( - const StorageS3Configuration & configuration_, - const StorageID & table_id_, - ColumnsDescription columns_, - const ConstraintsDescription & constraints_, - const String & comment, - ContextPtr context_, - std::optional format_settings_); + HudiMetaParser(const StorageS3::S3Configuration & configuration_, const String & table_path_, ContextPtr context_); - String getName() const override { return "Hudi"; } + std::vector getFiles() const; + static String generateQueryFromKeys(const std::vector & keys, const String & format); - /// Reads latest version of Apache Hudi table - Pipe read( - const Names & column_names, - const StorageSnapshotPtr & storage_snapshot, - SelectQueryInfo & query_info, - ContextPtr context, - QueryProcessingStage::Enum processed_stage, - size_t max_block_size, - size_t num_streams) override; - - static ColumnsDescription getTableStructureFromData( - const StorageS3Configuration & configuration, - const std::optional & format_settings, - ContextPtr ctx); private: - StorageS3::S3Configuration base_configuration; - std::shared_ptr s3engine; - Poco::Logger * log; + StorageS3::S3Configuration configuration; String table_path; + ContextPtr context; + Poco::Logger * log; }; +struct StorageHudiName +{ + static constexpr auto name = "Hudi"; +}; + +using StorageHudi = IStorageDataLake; } #endif diff --git a/src/Storages/StorageIceberg.cpp b/src/Storages/StorageIceberg.cpp index 482443ef783..2f92530f272 100644 --- a/src/Storages/StorageIceberg.cpp +++ b/src/Storages/StorageIceberg.cpp @@ -1,6 +1,7 @@ #include "config.h" #if USE_AWS_S3 +# include # include # include @@ -234,120 +235,14 @@ std::shared_ptr IcebergMetaParser::createS3ReadBuffer(const String & context->getReadSettings()); } -namespace -{ - -StorageS3::S3Configuration getBaseConfiguration(const StorageS3Configuration & configuration) -{ - return {configuration.url, configuration.auth_settings, configuration.request_settings, configuration.headers}; -} - // generateQueryFromKeys constructs query from all parquet filenames // for underlying StorageS3 engine -String generateQueryFromKeys(const std::vector & keys) +String IcebergMetaParser::generateQueryFromKeys(const std::vector & keys, const String &) { std::string new_query = fmt::format("{{{}}}", fmt::join(keys, ",")); return new_query; } - -StorageS3Configuration getAdjustedS3Configuration( - const ContextPtr & context, - StorageS3::S3Configuration & base_configuration, - const StorageS3Configuration & configuration, - const std::string & table_path, - Poco::Logger * log) -{ - IcebergMetaParser parser{base_configuration, table_path, context}; - - auto keys = parser.getFiles(); - static constexpr auto iceberg_data_directory = "data"; - auto new_uri = std::filesystem::path(base_configuration.uri.uri.toString()) / iceberg_data_directory / generateQueryFromKeys(keys); - - LOG_DEBUG(log, "New uri: {}", new_uri.c_str()); - LOG_DEBUG(log, "Table path: {}", table_path); - - // set new url in configuration - StorageS3Configuration new_configuration; - new_configuration.url = new_uri; - new_configuration.auth_settings.access_key_id = configuration.auth_settings.access_key_id; - new_configuration.auth_settings.secret_access_key = configuration.auth_settings.secret_access_key; - new_configuration.format = configuration.format; - - return new_configuration; -} - -} - -StorageIceberg::StorageIceberg( - const StorageS3Configuration & configuration_, - const StorageID & table_id_, - ColumnsDescription columns_, - const ConstraintsDescription & constraints_, - const String & comment, - ContextPtr context_, - std::optional format_settings_) - : IStorage(table_id_) - , base_configuration{getBaseConfiguration(configuration_)} - , log(&Poco::Logger::get("StorageIceberg(" + table_id_.table_name + ")")) - , table_path(base_configuration.uri.key) -{ - StorageInMemoryMetadata storage_metadata; - StorageS3::updateS3Configuration(context_, base_configuration); - - auto new_configuration = getAdjustedS3Configuration(context_, base_configuration, configuration_, table_path, log); - - if (columns_.empty()) - { - columns_ = StorageS3::getTableStructureFromData( - new_configuration, /*distributed processing*/ false, format_settings_, context_, nullptr); - storage_metadata.setColumns(columns_); - } - else - storage_metadata.setColumns(columns_); - - - storage_metadata.setConstraints(constraints_); - storage_metadata.setComment(comment); - setInMemoryMetadata(storage_metadata); - - s3engine = std::make_shared( - new_configuration, - table_id_, - columns_, - constraints_, - comment, - context_, - format_settings_, - /* distributed_processing_ */ false, - nullptr); -} - -Pipe StorageIceberg::read( - const Names & column_names, - const StorageSnapshotPtr & storage_snapshot, - SelectQueryInfo & query_info, - ContextPtr context, - QueryProcessingStage::Enum processed_stage, - size_t max_block_size, - size_t num_streams) -{ - StorageS3::updateS3Configuration(context, base_configuration); - - return s3engine->read(column_names, storage_snapshot, query_info, context, processed_stage, max_block_size, num_streams); -} - -ColumnsDescription StorageIceberg::getTableStructureFromData( - const StorageS3Configuration & configuration, const std::optional & format_settings, ContextPtr ctx) -{ - auto base_configuration = getBaseConfiguration(configuration); - StorageS3::updateS3Configuration(ctx, base_configuration); - auto new_configuration = getAdjustedS3Configuration( - ctx, base_configuration, configuration, base_configuration.uri.key, &Poco::Logger::get("StorageIceberg")); - return StorageS3::getTableStructureFromData( - new_configuration, /*distributed processing*/ false, format_settings, ctx, /*object_infos*/ nullptr); -} - void registerStorageIceberg(StorageFactory & factory) { factory.registerStorage( @@ -355,27 +250,12 @@ void registerStorageIceberg(StorageFactory & factory) [](const StorageFactory::Arguments & args) { auto & engine_args = args.engine_args; - if (engine_args.empty() || engine_args.size() < 3) - throw Exception( - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, - "Storage Iceberg requires 3 to 4 arguments: table_url, access_key, secret_access_key, [format]"); + StorageS3Configuration configuration = StorageIceberg::getConfiguration(engine_args, args.getLocalContext()); - StorageS3Configuration configuration; - - configuration.url = checkAndGetLiteralArgument(engine_args[0], "url"); - configuration.auth_settings.access_key_id = checkAndGetLiteralArgument(engine_args[1], "access_key_id"); - configuration.auth_settings.secret_access_key = checkAndGetLiteralArgument(engine_args[2], "secret_access_key"); - - if (engine_args.size() == 4) - configuration.format = checkAndGetLiteralArgument(engine_args[3], "format"); - else - { - /// Iceberg uses Parquet by default. - configuration.format = "Parquet"; - } + auto format_settings = getFormatSettings(args.getContext()); return std::make_shared( - configuration, args.table_id, args.columns, args.constraints, args.comment, args.getContext(), std::nullopt); + configuration, args.table_id, args.columns, args.constraints, args.comment, args.getContext(), format_settings); }, { .supports_settings = true, diff --git a/src/Storages/StorageIceberg.h b/src/Storages/StorageIceberg.h index bd20213a5d6..5bfe71c6d95 100644 --- a/src/Storages/StorageIceberg.h +++ b/src/Storages/StorageIceberg.h @@ -4,8 +4,7 @@ #if USE_AWS_S3 -# include -# include +# include # include # include @@ -36,6 +35,8 @@ public: std::vector getFiles() const; + static String generateQueryFromKeys(const std::vector & keys, const String & format); + private: static constexpr auto metadata_directory = "metadata"; StorageS3::S3Configuration base_configuration; @@ -51,45 +52,12 @@ private: std::shared_ptr createS3ReadBuffer(const String & key) const; }; -class StorageIceberg : public IStorage +struct StorageIcebergName { -public: - // 1. Parses internal file structure of table - // 2. Finds out parts with latest version - // 3. Creates url for underlying StorageS3 enigne to handle reads - StorageIceberg( - const StorageS3Configuration & configuration_, - const StorageID & table_id_, - ColumnsDescription columns_, - const ConstraintsDescription & constraints_, - const String & comment, - ContextPtr context_, - std::optional format_settings_); - - String getName() const override { return "Iceberg"; } - - // Reads latest version of Iceberg table - Pipe read( - const Names & column_names, - const StorageSnapshotPtr & storage_snapshot, - SelectQueryInfo & query_info, - ContextPtr context, - QueryProcessingStage::Enum processed_stage, - size_t max_block_size, - size_t num_streams) override; - - static ColumnsDescription getTableStructureFromData( - const StorageS3Configuration & configuration, - const std::optional & format_settings, - ContextPtr ctx); -private: - - StorageS3::S3Configuration base_configuration; - std::shared_ptr s3engine; - Poco::Logger * log; - String table_path; + static constexpr auto name = "Iceberg"; }; +using StorageIceberg = IStorageDataLake; } #endif diff --git a/src/Storages/StorageS3.h b/src/Storages/StorageS3.h index 7b86c98bcd5..53727d0dbd9 100644 --- a/src/Storages/StorageS3.h +++ b/src/Storages/StorageS3.h @@ -11,16 +11,16 @@ #include #include -#include -#include -#include -#include -#include #include +#include #include #include -#include +#include +#include #include +#include +#include +#include namespace Aws::S3 { @@ -34,6 +34,18 @@ class PullingPipelineExecutor; class StorageS3SequentialSource; class NamedCollection; +template +class IStorageDataLake; + +struct StorageIcebergName; +class IcebergMetaParser; + +struct StorageDeltaLakeName; +class DeltaLakeMetaParser; + +struct StorageHudiName; +class HudiMetaParser; + class StorageS3Source : public ISource, WithContext { public: @@ -315,9 +327,9 @@ public: private: friend class StorageS3Cluster; friend class TableFunctionS3Cluster; - friend class StorageHudi; - friend class StorageDeltaLake; - friend class StorageIceberg; + friend class IStorageDataLake; + friend class IStorageDataLake; + friend class IStorageDataLake; S3Configuration s3_configuration; std::vector keys; From 810d3e584204dca3dbf8a00c787b196cdc7d5b93 Mon Sep 17 00:00:00 2001 From: flynn Date: Sun, 29 Jan 2023 14:58:14 +0000 Subject: [PATCH 055/717] fix comment --- src/Storages/IStorageDataLake.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/IStorageDataLake.h b/src/Storages/IStorageDataLake.h index f97f949e540..cc23b2f8f92 100644 --- a/src/Storages/IStorageDataLake.h +++ b/src/Storages/IStorageDataLake.h @@ -114,7 +114,7 @@ public: static constexpr auto name = Name::name; String getName() const override { return name; } - // Reads latest version of Iceberg table + // Reads latest version of Lake Table Pipe read( const Names & column_names, const StorageSnapshotPtr & storage_snapshot, From 2911927134cbf4eeb34528cf3e6307eff00afc58 Mon Sep 17 00:00:00 2001 From: flynn Date: Sun, 29 Jan 2023 15:00:04 +0000 Subject: [PATCH 056/717] fix style --- src/Storages/StorageDeltaLake.cpp | 1 - src/Storages/StorageHudi.cpp | 1 - src/Storages/StorageIceberg.cpp | 1 - 3 files changed, 3 deletions(-) diff --git a/src/Storages/StorageDeltaLake.cpp b/src/Storages/StorageDeltaLake.cpp index 08f262f8071..fb8817fbc82 100644 --- a/src/Storages/StorageDeltaLake.cpp +++ b/src/Storages/StorageDeltaLake.cpp @@ -31,7 +31,6 @@ namespace DB namespace ErrorCodes { extern const int S3_ERROR; - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int INCORRECT_DATA; } diff --git a/src/Storages/StorageHudi.cpp b/src/Storages/StorageHudi.cpp index a8daee3d7cd..b8356a9776c 100644 --- a/src/Storages/StorageHudi.cpp +++ b/src/Storages/StorageHudi.cpp @@ -23,7 +23,6 @@ namespace DB namespace ErrorCodes { - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int S3_ERROR; extern const int LOGICAL_ERROR; } diff --git a/src/Storages/StorageIceberg.cpp b/src/Storages/StorageIceberg.cpp index 2f92530f272..6420fe9ba51 100644 --- a/src/Storages/StorageIceberg.cpp +++ b/src/Storages/StorageIceberg.cpp @@ -42,7 +42,6 @@ namespace DB namespace ErrorCodes { extern const int S3_ERROR; - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int FILE_DOESNT_EXIST; extern const int ILLEGAL_COLUMN; } From bb95093eee1c556850f38d397824a133147b9692 Mon Sep 17 00:00:00 2001 From: flynn Date: Sun, 29 Jan 2023 15:05:03 +0000 Subject: [PATCH 057/717] remove unused file --- src/TableFunctions/TableFunctionHudi.h | 44 -------------------------- 1 file changed, 44 deletions(-) delete mode 100644 src/TableFunctions/TableFunctionHudi.h diff --git a/src/TableFunctions/TableFunctionHudi.h b/src/TableFunctions/TableFunctionHudi.h deleted file mode 100644 index a370bca8c45..00000000000 --- a/src/TableFunctions/TableFunctionHudi.h +++ /dev/null @@ -1,44 +0,0 @@ -#pragma once - -#include "config.h" - -#if USE_AWS_S3 - -#include -#include - - -namespace DB -{ - -class Context; -class TableFunctionS3Cluster; - -/* hudi(source, [access_key_id, secret_access_key,] format, structure[, compression]) - creates a temporary Hudi table on S3. - */ -class TableFunctionHudi : public ITableFunction -{ -public: - static constexpr auto name = "hudi"; - std::string getName() const override - { - return name; - } - -protected: - StoragePtr executeImpl( - const ASTPtr & ast_function, ContextPtr context, const std::string & table_name, ColumnsDescription cached_columns) const override; - - const char * getStorageTypeName() const override { return name; } - - ColumnsDescription getActualTableStructure(ContextPtr context) const override; - void parseArguments(const ASTPtr & ast_function, ContextPtr context) override; - - static void parseArgumentsImpl(const String & error_message, ASTs & args, ContextPtr context, StorageS3Configuration & configuration); - - StorageS3Configuration configuration; -}; - -} - -#endif From 1bf280cb50cf563d5a9337d36280144ba69ad2ba Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sun, 29 Jan 2023 17:25:08 +0000 Subject: [PATCH 058/717] Handle obscure case that /proc/cpuinfo is unmounted --- src/Common/getNumberOfPhysicalCPUCores.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Common/getNumberOfPhysicalCPUCores.cpp b/src/Common/getNumberOfPhysicalCPUCores.cpp index 5c6640dc509..0f08ab1eb9c 100644 --- a/src/Common/getNumberOfPhysicalCPUCores.cpp +++ b/src/Common/getNumberOfPhysicalCPUCores.cpp @@ -54,6 +54,10 @@ try /// See https://doc.callmematthi.eu/static/webArticles/Understanding%20Linux%20_proc_cpuinfo.pdf std::ifstream proc_cpuinfo("/proc/cpuinfo"); + if (proc_cpuinfo.fail()) + /// In obscure cases (chroot) /proc can be unmounted + return std::thread::hardware_concurrency(); + using CoreEntry = std::pair; /// physical id, core id using CoreEntrySet = std::set; From 33a719b9bec9ff0bacf33b4e1794dd0adf76d726 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sun, 29 Jan 2023 21:37:19 +0100 Subject: [PATCH 059/717] Update getNumberOfPhysicalCPUCores.cpp --- src/Common/getNumberOfPhysicalCPUCores.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/getNumberOfPhysicalCPUCores.cpp b/src/Common/getNumberOfPhysicalCPUCores.cpp index 0f08ab1eb9c..ed82c59140d 100644 --- a/src/Common/getNumberOfPhysicalCPUCores.cpp +++ b/src/Common/getNumberOfPhysicalCPUCores.cpp @@ -54,7 +54,7 @@ try /// See https://doc.callmematthi.eu/static/webArticles/Understanding%20Linux%20_proc_cpuinfo.pdf std::ifstream proc_cpuinfo("/proc/cpuinfo"); - if (proc_cpuinfo.fail()) + if (!proc_cpuinfo.is_open()) /// In obscure cases (chroot) /proc can be unmounted return std::thread::hardware_concurrency(); From c5581b67339e5af7736611bc0933fb140c6960a3 Mon Sep 17 00:00:00 2001 From: flynn Date: Mon, 30 Jan 2023 02:41:08 +0000 Subject: [PATCH 060/717] fix --- src/Storages/IStorageDataLake.h | 29 +++++++---------------------- src/Storages/StorageDeltaLake.h | 1 + src/Storages/StorageHudi.h | 1 + src/Storages/StorageIceberg.h | 1 + 4 files changed, 10 insertions(+), 22 deletions(-) diff --git a/src/Storages/IStorageDataLake.h b/src/Storages/IStorageDataLake.h index cc23b2f8f92..f8c27ea4cb7 100644 --- a/src/Storages/IStorageDataLake.h +++ b/src/Storages/IStorageDataLake.h @@ -155,8 +155,7 @@ public: MetaParser parser{base_configuration, table_path, context}; auto keys = parser.getFiles(); - static constexpr auto iceberg_data_directory = "data"; - auto new_uri = std::filesystem::path(base_configuration.uri.uri.toString()) / iceberg_data_directory + auto new_uri = std::filesystem::path(base_configuration.uri.uri.toString()) / Name::data_directory_prefix / MetaParser::generateQueryFromKeys(keys, configuration.format); LOG_DEBUG(log, "New uri: {}", new_uri.c_str()); @@ -203,13 +202,10 @@ public: { /// Supported signatures: /// - /// xx('url') - /// xx('url', 'format') - /// xx('url', 'format', 'compression') + /// xx('url', 'aws_access_key_id', 'aws_secret_access_key') /// xx('url', 'aws_access_key_id', 'aws_secret_access_key', 'format') - /// xx('url', 'aws_access_key_id', 'aws_secret_access_key', 'format', 'compression') - if (engine_args.empty() || engine_args.size() > 5) + if (engine_args.empty() || engine_args.size() < 4) throw Exception( ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Storage {} requires 1 to 5 arguments: " @@ -224,22 +220,11 @@ public: engine_arg = evaluateConstantExpressionOrIdentifierAsLiteral(engine_arg, local_context); configuration.url = checkAndGetLiteralArgument(engine_args[0], "url"); - if (engine_args.size() >= 4) - { - configuration.auth_settings.access_key_id = checkAndGetLiteralArgument(engine_args[1], "access_key_id"); - configuration.auth_settings.secret_access_key = checkAndGetLiteralArgument(engine_args[2], "secret_access_key"); - } + configuration.auth_settings.access_key_id = checkAndGetLiteralArgument(engine_args[1], "access_key_id"); + configuration.auth_settings.secret_access_key = checkAndGetLiteralArgument(engine_args[2], "secret_access_key"); - if (engine_args.size() == 3 || engine_args.size() == 5) - { - configuration.compression_method = checkAndGetLiteralArgument(engine_args.back(), "compression_method"); - configuration.format = checkAndGetLiteralArgument(engine_args[engine_args.size() - 2], "format"); - } - else if (engine_args.size() != 1) - { - configuration.compression_method = "auto"; - configuration.format = checkAndGetLiteralArgument(engine_args.back(), "format"); - } + if (engine_args.size() == 4) + configuration.format = checkAndGetLiteralArgument(engine_args[3], "format"); } if (configuration.format == "auto") diff --git a/src/Storages/StorageDeltaLake.h b/src/Storages/StorageDeltaLake.h index 95b89ad9221..0484e206861 100644 --- a/src/Storages/StorageDeltaLake.h +++ b/src/Storages/StorageDeltaLake.h @@ -59,6 +59,7 @@ private: struct StorageDeltaLakeName { static constexpr auto name = "DeltaLake"; + static constexpr auto data_directory_prefix = ""; }; using StorageDeltaLake = IStorageDataLake; diff --git a/src/Storages/StorageHudi.h b/src/Storages/StorageHudi.h index b84cb45a4b3..82295134a8a 100644 --- a/src/Storages/StorageHudi.h +++ b/src/Storages/StorageHudi.h @@ -35,6 +35,7 @@ private: struct StorageHudiName { static constexpr auto name = "Hudi"; + static constexpr auto data_directory_prefix = ""; }; using StorageHudi = IStorageDataLake; diff --git a/src/Storages/StorageIceberg.h b/src/Storages/StorageIceberg.h index 5bfe71c6d95..0ecc78ff9bc 100644 --- a/src/Storages/StorageIceberg.h +++ b/src/Storages/StorageIceberg.h @@ -55,6 +55,7 @@ private: struct StorageIcebergName { static constexpr auto name = "Iceberg"; + static constexpr auto data_directory_prefix = "data"; }; using StorageIceberg = IStorageDataLake; From a742c142b3308c129c988b43fa76baf407ed8b1b Mon Sep 17 00:00:00 2001 From: flynn Date: Mon, 30 Jan 2023 03:27:53 +0000 Subject: [PATCH 061/717] fix --- src/Storages/IStorageDataLake.h | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/Storages/IStorageDataLake.h b/src/Storages/IStorageDataLake.h index f8c27ea4cb7..a5080b46d8d 100644 --- a/src/Storages/IStorageDataLake.h +++ b/src/Storages/IStorageDataLake.h @@ -143,7 +143,7 @@ public: static StorageS3::S3Configuration getBaseConfiguration(const StorageS3Configuration & configuration) { return {configuration.url, configuration.auth_settings, configuration.request_settings, configuration.headers}; -} + } static StorageS3Configuration getAdjustedS3Configuration( const ContextPtr & context, @@ -205,11 +205,10 @@ public: /// xx('url', 'aws_access_key_id', 'aws_secret_access_key') /// xx('url', 'aws_access_key_id', 'aws_secret_access_key', 'format') - if (engine_args.empty() || engine_args.size() < 4) + if (engine_args.empty() || engine_args.size() < 3) throw Exception( ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, - "Storage {} requires 1 to 5 arguments: " - "url, [access_key_id, secret_access_key], name of used format and [compression_method]", + "Storage {} requires 3 or 4 arguments: url, access_key_id, secret_access_key, [format]", name); auto * header_it = StorageURL::collectHeaders(engine_args, configuration.headers, local_context); From a2c9aeb7c9f40b8f5c3d583523bc58412e885552 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Thu, 5 Jan 2023 19:22:42 +0000 Subject: [PATCH 062/717] stash --- src/Interpreters/Aggregator.h | 1 + src/Processors/QueryPlan/AggregatingStep.cpp | 28 ++- src/Processors/QueryPlan/AggregatingStep.h | 2 + .../QueryPlan/Optimizations/Optimizations.h | 9 +- .../useDataParallelAggregation.cpp | 70 +++++++ .../QueryPlan/ReadFromMergeTree.cpp | 132 +++++++++++-- src/Processors/QueryPlan/ReadFromMergeTree.h | 11 ++ .../Transforms/AggregatingTransform.cpp | 179 +++++++++++++++--- .../Transforms/AggregatingTransform.h | 7 +- ...ergingAggregatedMemoryEfficientTransform.h | 3 +- .../performance/aggregation_by_partitions.xml | 13 ++ .../02521_aggregation_by_partitions.reference | 83 ++++++++ .../02521_aggregation_by_partitions.sql | 42 ++++ 13 files changed, 525 insertions(+), 55 deletions(-) create mode 100644 src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp create mode 100644 tests/performance/aggregation_by_partitions.xml create mode 100644 tests/queries/0_stateless/02521_aggregation_by_partitions.reference create mode 100644 tests/queries/0_stateless/02521_aggregation_by_partitions.sql diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h index 7fe4dd89d8e..65d84b9111a 100644 --- a/src/Interpreters/Aggregator.h +++ b/src/Interpreters/Aggregator.h @@ -1084,6 +1084,7 @@ private: friend struct AggregatedDataVariants; friend class ConvertingAggregatedToChunksTransform; friend class ConvertingAggregatedToChunksSource; + friend class ConvertingAggregatedToChunksWithMergingSource; friend class AggregatingInOrderTransform; /// Data structure of source blocks. diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp index 4fd6e7c11dd..c0ea848fea8 100644 --- a/src/Processors/QueryPlan/AggregatingStep.cpp +++ b/src/Processors/QueryPlan/AggregatingStep.cpp @@ -166,6 +166,12 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B params.group_by_two_level_threshold_bytes = 0; } + /// In case of external aggregation we cannot completely avoid merging, + /// because each thread might use multiple hash tables during the execution and then the same key might be present in multiple hash tables. + /// But nevertheless we could save some time merging only HTs from the same thread (future task). + if (params.max_bytes_before_external_group_by) + skip_merging = false; + /** Two-level aggregation is useful in two cases: * 1. Parallel aggregation is done, and the results should be merged in parallel. * 2. An aggregation is done with store of temporary data on the disk, and they need to be merged in a memory efficient way. @@ -228,7 +234,14 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B auto many_data = std::make_shared(streams); for (size_t j = 0; j < streams; ++j) { - auto aggregation_for_set = std::make_shared(input_header, transform_params_for_set, many_data, j, merge_threads, temporary_data_merge_threads); + auto aggregation_for_set = std::make_shared( + input_header, + transform_params_for_set, + many_data, + j, + merge_threads, + temporary_data_merge_threads, + skip_merging); // For each input stream we have `grouping_sets_size` copies, so port index // for transform #j should skip ports of first (j-1) streams. connect(*ports[i + grouping_sets_size * j], aggregation_for_set->getInputs().front()); @@ -410,16 +423,19 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B if (pipeline.getNumStreams() > 1) { /// Add resize transform to uniformly distribute data between aggregating streams. - if (!storage_has_evenly_distributed_read) + /// But not if we execute aggregation over partitoned data in which case data streams shouldn't be mixed. + if (!storage_has_evenly_distributed_read && !skip_merging) pipeline.resize(pipeline.getNumStreams(), true, true); auto many_data = std::make_shared(pipeline.getNumStreams()); size_t counter = 0; - pipeline.addSimpleTransform([&](const Block & header) - { - return std::make_shared(header, transform_params, many_data, counter++, merge_threads, temporary_data_merge_threads); - }); + pipeline.addSimpleTransform( + [&](const Block & header) + { + return std::make_shared( + header, transform_params, many_data, counter++, merge_threads, temporary_data_merge_threads, skip_merging); + }); pipeline.resize(should_produce_results_in_order_of_bucket_number ? 1 : params.max_threads, true /* force */); diff --git a/src/Processors/QueryPlan/AggregatingStep.h b/src/Processors/QueryPlan/AggregatingStep.h index 0dc06649d2d..c0729809ccf 100644 --- a/src/Processors/QueryPlan/AggregatingStep.h +++ b/src/Processors/QueryPlan/AggregatingStep.h @@ -59,6 +59,7 @@ public: bool isGroupingSets() const { return !grouping_sets_params.empty(); } void applyOrder(SortDescription sort_description_for_merging_, SortDescription group_by_sort_description_); bool memoryBoundMergingWillBeUsed() const; + void skipMerging() { skip_merging = true; } private: void updateOutputStream() override; @@ -70,6 +71,7 @@ private: size_t aggregation_in_order_max_block_bytes; size_t merge_threads; size_t temporary_data_merge_threads; + bool skip_merging = false; // if we aggregate partitioned data merging is not needed bool storage_has_evenly_distributed_read; bool group_by_use_nulls; diff --git a/src/Processors/QueryPlan/Optimizations/Optimizations.h b/src/Processors/QueryPlan/Optimizations/Optimizations.h index 4b587ada2c0..91c2f2550e8 100644 --- a/src/Processors/QueryPlan/Optimizations/Optimizations.h +++ b/src/Processors/QueryPlan/Optimizations/Optimizations.h @@ -68,17 +68,22 @@ void tryRemoveRedundantSorting(QueryPlan::Node * root); /// - Something - - Expression - Something - size_t tryLiftUpUnion(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes); +size_t tryAggregateEachPartitionIndependently(QueryPlan::Node * node, QueryPlan::Nodes &); + inline const auto & getOptimizations() { - static const std::array optimizations = {{ + static const std::array optimizations = {{ {tryLiftUpArrayJoin, "liftUpArrayJoin", &QueryPlanOptimizationSettings::optimize_plan}, {tryPushDownLimit, "pushDownLimit", &QueryPlanOptimizationSettings::optimize_plan}, {trySplitFilter, "splitFilter", &QueryPlanOptimizationSettings::optimize_plan}, {tryMergeExpressions, "mergeExpressions", &QueryPlanOptimizationSettings::optimize_plan}, {tryPushDownFilter, "pushDownFilter", &QueryPlanOptimizationSettings::filter_push_down}, {tryExecuteFunctionsAfterSorting, "liftUpFunctions", &QueryPlanOptimizationSettings::optimize_plan}, - {tryReuseStorageOrderingForWindowFunctions, "reuseStorageOrderingForWindowFunctions", &QueryPlanOptimizationSettings::optimize_plan}, + {tryReuseStorageOrderingForWindowFunctions, + "reuseStorageOrderingForWindowFunctions", + &QueryPlanOptimizationSettings::optimize_plan}, {tryLiftUpUnion, "liftUpUnion", &QueryPlanOptimizationSettings::optimize_plan}, + {tryAggregateEachPartitionIndependently, "aggregationPartitionsIndepedently", &QueryPlanOptimizationSettings::optimize_plan}, }}; return optimizations; diff --git a/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp b/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp new file mode 100644 index 00000000000..2fd225bdcba --- /dev/null +++ b/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp @@ -0,0 +1,70 @@ +#include + +#include +#include +#include + +using namespace DB; + +namespace +{ + +bool isPartitionKeySuitsGroupByKey(const ReadFromMergeTree & reading, const AggregatingStep & aggregating) +{ + const auto & gb_keys = aggregating.getParams().keys; + if (aggregating.isGroupingSets() || gb_keys.size() != 1) + return false; + + const auto & pkey_nodes = reading.getStorageMetadata()->getPartitionKey().expression->getActionsDAG().getNodes(); + LOG_DEBUG(&Poco::Logger::get("debug"), "{}", reading.getStorageMetadata()->getPartitionKey().expression->getActionsDAG().dumpDAG()); + if (!pkey_nodes.empty()) + { + const auto & func_node = pkey_nodes.back(); + LOG_DEBUG(&Poco::Logger::get("debug"), "{} {} {}", func_node.type, func_node.is_deterministic, func_node.children.size()); + if (func_node.type == ActionsDAG::ActionType::FUNCTION && func_node.function->getName() == "modulo" + && func_node.children.size() == 2) + { + const auto & arg1 = func_node.children.front(); + const auto & arg2 = func_node.children.back(); + LOG_DEBUG(&Poco::Logger::get("debug"), "{} {} {}", arg1->type, arg1->result_name, arg2->type); + if (arg1->type == ActionsDAG::ActionType::INPUT && arg1->result_name == gb_keys[0] + && arg2->type == ActionsDAG::ActionType::COLUMN && typeid_cast(arg2->column.get())) + return true; + } + } + + return false; +} +} + +namespace DB::QueryPlanOptimizations +{ + +size_t tryAggregateEachPartitionIndependently(QueryPlan::Node * node, QueryPlan::Nodes &) +{ + if (!node || node->children.size() != 1) + return 0; + + auto * aggregating_step = typeid_cast(node->step.get()); + if (!aggregating_step) + return 0; + + const auto * expression_node = node->children.front(); + if (expression_node->children.size() != 1 || !typeid_cast(expression_node->step.get())) + return 0; + + auto * reading_step = expression_node->children.front()->step.get(); + auto * reading = typeid_cast(reading_step); + if (!reading) + return 0; + + if (!reading->willOutputEachPartitionThroughSeparatePort() && isPartitionKeySuitsGroupByKey(*reading, *aggregating_step)) + { + if (reading->requestOutputEachPartitionThroughSeparatePort()) + aggregating_step->skipMerging(); + } + + return 0; +} + +} diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 22245b82966..addfcb508ce 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -1,13 +1,14 @@ #include #include +#include #include #include #include #include #include +#include #include #include -#include #include #include #include @@ -30,13 +31,31 @@ #include #include #include -#include #include +#include #include -#include #include #include #include +#include + +namespace +{ +size_t countPartitions(DB::RangesInDataParts & parts_with_ranges) +{ + std::string cur_partition_id = parts_with_ranges[0].data_part->info.partition_id; + size_t unique_partitions = 1; + for (size_t i = 1; i < parts_with_ranges.size(); ++i) + { + if (parts_with_ranges[i].data_part->info.partition_id != cur_partition_id) + { + ++unique_partitions; + cur_partition_id = parts_with_ranges[i].data_part->info.partition_id; + } + } + return unique_partitions; +} +} namespace ProfileEvents { @@ -242,7 +261,6 @@ Pipe ReadFromMergeTree::readFromPool( if (i == 0 && !client_info.collaborate_with_initiator) source->addTotalRowsApprox(total_rows); - pipes.emplace_back(std::move(source)); } @@ -335,8 +353,12 @@ Pipe ReadFromMergeTree::read( size_t max_streams, size_t min_marks_for_concurrent_read, bool use_uncompressed_cache) { if (read_type == ReadType::Default && max_streams > 1) - return readFromPool(parts_with_range, required_columns, max_streams, - min_marks_for_concurrent_read, use_uncompressed_cache); + { + Pipe pipe = readFromPool(parts_with_range, required_columns, max_streams, min_marks_for_concurrent_read, use_uncompressed_cache); + if (output_each_partition_through_separate_port) + pipe.resize(1); + return pipe; + } auto pipe = readInOrder(parts_with_range, required_columns, read_type, use_uncompressed_cache, 0); @@ -418,9 +440,8 @@ struct PartRangesReadInfo } -Pipe ReadFromMergeTree::spreadMarkRangesAmongStreams( - RangesInDataParts && parts_with_ranges, - const Names & column_names) +Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsImpl( + RangesInDataParts && parts_with_ranges, const Names & column_names, size_t num_streams) { const auto & settings = context->getSettingsRef(); const auto data_settings = data.getSettings(); @@ -430,16 +451,67 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreams( if (0 == info.sum_marks) return {}; - size_t num_streams = requested_num_streams; if (num_streams > 1) { /// Reduce the number of num_streams if the data is small. if (info.sum_marks < num_streams * info.min_marks_for_concurrent_read && parts_with_ranges.size() < num_streams) - num_streams = std::max((info.sum_marks + info.min_marks_for_concurrent_read - 1) / info.min_marks_for_concurrent_read, parts_with_ranges.size()); + num_streams = std::max( + (info.sum_marks + info.min_marks_for_concurrent_read - 1) / info.min_marks_for_concurrent_read, parts_with_ranges.size()); } - return read(std::move(parts_with_ranges), column_names, ReadType::Default, - num_streams, info.min_marks_for_concurrent_read, info.use_uncompressed_cache); + return read( + std::move(parts_with_ranges), + column_names, + ReadType::Default, + num_streams, + info.min_marks_for_concurrent_read, + info.use_uncompressed_cache); +} + +Pipe ReadFromMergeTree::spreadMarkRangesAmongStreams( + RangesInDataParts && parts_with_ranges, + const Names & column_names) +{ + if (parts_with_ranges.empty()) + return {}; + + size_t num_streams = requested_num_streams; + + if (!output_each_partition_through_separate_port) + { + return spreadMarkRangesAmongStreamsImpl(std::move(parts_with_ranges), column_names, num_streams); + } + else + { + num_streams = std::max(1, num_streams / countPartitions(parts_with_ranges)); + + LOG_DEBUG( + &Poco::Logger::get("debug"), + "spreadMarkRangesAmongStreams {} {} {}", + parts_with_ranges.size(), + requested_num_streams, + countPartitions(parts_with_ranges)); + + Pipes pipes; + for (auto begin = parts_with_ranges.begin(); begin != parts_with_ranges.end();) + { + const auto end = std::find_if( + begin, + parts_with_ranges.end(), + [&begin](auto & part) { return begin->data_part->info.partition_id != part.data_part->info.partition_id; }); + + LOG_DEBUG(&Poco::Logger::get("debug"), "spreadMarkRangesAmongStreams {} {}", begin->data_part->info.partition_id, end - begin); + + RangesInDataParts partition_parts; + partition_parts.insert(partition_parts.end(), std::make_move_iterator(begin), std::make_move_iterator(end)); + + pipes.emplace_back(spreadMarkRangesAmongStreamsImpl(std::move(partition_parts), column_names, num_streams)); + + begin = end; + } + + return Pipe::unitePipes(std::move(pipes)); + } } static ActionsDAGPtr createProjection(const Block & header) @@ -1197,6 +1269,18 @@ void ReadFromMergeTree::requestReadingInOrder(size_t prefix_size, int direction, output_stream->sort_description = std::move(sort_description); output_stream->sort_scope = DataStream::SortScope::Stream; } + + /// Not supported currently. Disable optimisation. + output_each_partition_through_separate_port = false; +} + +bool ReadFromMergeTree::requestOutputEachPartitionThroughSeparatePort() +{ + if (isQueryWithFinal() || query_info.getInputOrderInfo()) + return false; + + output_each_partition_through_separate_port = true; + return true; } ReadFromMergeTree::AnalysisResult ReadFromMergeTree::getAnalysisResult() const @@ -1208,6 +1292,15 @@ ReadFromMergeTree::AnalysisResult ReadFromMergeTree::getAnalysisResult() const return std::get(result_ptr->result); } +bool ReadFromMergeTree::isQueryWithFinal() const +{ + const auto & select = query_info.query->as(); + if (query_info.table_expression_modifiers) + return query_info.table_expression_modifiers->hasFinal(); + else + return select.final(); +} + void ReadFromMergeTree::initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) { auto result = getAnalysisResult(); @@ -1243,12 +1336,7 @@ void ReadFromMergeTree::initializePipeline(QueryPipelineBuilder & pipeline, cons ActionsDAGPtr result_projection; Names column_names_to_read = std::move(result.column_names_to_read); - const auto & select = query_info.query->as(); - bool final = false; - if (query_info.table_expression_modifiers) - final = query_info.table_expression_modifiers->hasFinal(); - else - final = select.final(); + bool final = isQueryWithFinal(); if (!final && result.sampling.use_sampling) { @@ -1267,6 +1355,9 @@ void ReadFromMergeTree::initializePipeline(QueryPipelineBuilder & pipeline, cons if (final) { + if (output_each_partition_through_separate_port) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Optimisation isn't supposed to be used for queries with final"); + /// Add columns needed to calculate the sorting expression and the sign. std::vector add_columns = metadata_for_reading->getColumnsRequiredForSortingKey(); column_names_to_read.insert(column_names_to_read.end(), add_columns.begin(), add_columns.end()); @@ -1286,6 +1377,9 @@ void ReadFromMergeTree::initializePipeline(QueryPipelineBuilder & pipeline, cons } else if (input_order_info) { + if (output_each_partition_through_separate_port) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Optimisation isn't supposed to be used when reading in order is used"); + pipe = spreadMarkRangesAmongStreamsWithOrder( std::move(result.parts_with_ranges), column_names_to_read, diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.h b/src/Processors/QueryPlan/ReadFromMergeTree.h index a3cea2a8afe..1599e36fc41 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.h +++ b/src/Processors/QueryPlan/ReadFromMergeTree.h @@ -154,6 +154,10 @@ public: void requestReadingInOrder(size_t prefix_size, int direction, size_t limit); + /// Returns true if the optimisation is applicable (and applies it then). + bool requestOutputEachPartitionThroughSeparatePort(); + bool willOutputEachPartitionThroughSeparatePort() const { return output_each_partition_through_separate_port; } + private: static MergeTreeDataSelectAnalysisResultPtr selectRangesToReadImpl( MergeTreeData::DataPartsVector parts, @@ -168,6 +172,8 @@ private: bool sample_factor_column_queried, Poco::Logger * log); + bool isQueryWithFinal() const; + int getSortDirection() const { const InputOrderInfoPtr & order_info = query_info.getInputOrderInfo(); @@ -203,6 +209,9 @@ private: const size_t preferred_max_column_in_block_size_bytes; const bool sample_factor_column_queried; + /// Used for aggregation optimisation (see DB::QueryPlanOptimizations::tryAggregateEachPartitionIndependently). + bool output_each_partition_through_separate_port = false; + std::shared_ptr max_block_numbers_to_read; Poco::Logger * log; @@ -221,6 +230,8 @@ private: RangesInDataParts && parts_with_ranges, const Names & column_names); + Pipe spreadMarkRangesAmongStreamsImpl(RangesInDataParts && parts_with_ranges, const Names & column_names, size_t num_streams); + Pipe spreadMarkRangesAmongStreamsWithOrder( RangesInDataParts && parts_with_ranges, const Names & column_names, diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index 836458ef792..f57ac4774d6 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -81,7 +81,7 @@ namespace /// Worker which merges buckets for two-level aggregation. /// Atomically increments bucket counter and returns merged result. -class ConvertingAggregatedToChunksSource : public ISource +class ConvertingAggregatedToChunksWithMergingSource : public ISource { public: static constexpr UInt32 NUM_BUCKETS = 256; @@ -101,19 +101,17 @@ public: using SharedDataPtr = std::shared_ptr; - ConvertingAggregatedToChunksSource( - AggregatingTransformParamsPtr params_, - ManyAggregatedDataVariantsPtr data_, - SharedDataPtr shared_data_, - Arena * arena_) + ConvertingAggregatedToChunksWithMergingSource( + AggregatingTransformParamsPtr params_, ManyAggregatedDataVariantsPtr data_, SharedDataPtr shared_data_, Arena * arena_) : ISource(params_->getHeader(), false) , params(std::move(params_)) , data(std::move(data_)) , shared_data(std::move(shared_data_)) , arena(arena_) - {} + { + } - String getName() const override { return "ConvertingAggregatedToChunksSource"; } + String getName() const override { return "ConvertingAggregatedToChunksWithMergingSource"; } protected: Chunk generate() override @@ -138,21 +136,131 @@ private: Arena * arena; }; +class ConvertingAggregatedToChunksSource : public ISource +{ +public: + ConvertingAggregatedToChunksSource(AggregatingTransformParamsPtr params_, AggregatedDataVariantsPtr variant_) + : ISource(params_->getHeader(), false), params(params_), variant(variant_) + { + } + + String getName() const override { return "ConvertingAggregatedToChunksSource"; } + +protected: + Chunk generate() override + { + if (!converted) + { + blocks = params->aggregator.convertToBlocks(*variant, params->final, 1 /* max_threads */); + converted = true; + } + + if (blocks.empty()) + return {}; + + auto res = convertToChunk(blocks.front()); + blocks.pop_front(); + return res; + } + +private: + AggregatingTransformParamsPtr params; + AggregatedDataVariantsPtr variant; + + bool converted = false; + BlocksList blocks; +}; + +/// Reads chunks from GroupingAggregatedTransform and outputs them. +class FlattenChunksToMergeTransform : public IProcessor +{ +public: + explicit FlattenChunksToMergeTransform(const Block & input_header, const Block & output_header) + : IProcessor({input_header}, {output_header}) + { + } + + String getName() const override { return "FlattenChunksToMergeTransform"; } + +private: + void work() override + { + } + + void process(Chunk chunk) + { + if (chunk.hasChunkInfo()) + { + const auto & info = chunk.getChunkInfo(); + if (const auto * chunks_to_merge = typeid_cast(info.get()); chunks_to_merge && chunks_to_merge->chunks) + for (auto & cur_chunk : *chunks_to_merge->chunks) + chunks.emplace_back(std::move(cur_chunk)); + } + } + + Status prepare() override + { + auto & input = inputs.front(); + auto & output = outputs.front(); + + if (output.isFinished()) + { + input.close(); + return Status::Finished; + } + + if (!output.canPush()) + { + input.setNotNeeded(); + return Status::PortFull; + } + + if (!chunks.empty()) + { + output.push(std::move(chunks.front())); + chunks.pop_front(); + } + + if (input.isFinished() && chunks.empty()) + { + output.finish(); + return Status::Finished; + } + + if (input.isFinished()) + return Status::Ready; + + input.setNeeded(); + + if (!input.hasData()) + return Status::NeedData; + + Chunk chunk = input.pull(false /* set_not_needed */); + process(std::move(chunk)); + + /// Now transform. + return Status::Ready; + } + + std::list chunks; +}; + /// Generates chunks with aggregated data. /// In single level case, aggregates data itself. -/// In two-level case, creates `ConvertingAggregatedToChunksSource` workers: +/// In two-level case, creates `ConvertingAggregatedToChunksWithMergingSource` workers: /// -/// ConvertingAggregatedToChunksSource -> -/// ConvertingAggregatedToChunksSource -> ConvertingAggregatedToChunksTransform -> AggregatingTransform -/// ConvertingAggregatedToChunksSource -> +/// ConvertingAggregatedToChunksWithMergingSource -> +/// ConvertingAggregatedToChunksWithMergingSource -> ConvertingAggregatedToChunksTransform -> AggregatingTransform +/// ConvertingAggregatedToChunksWithMergingSource -> /// /// Result chunks guaranteed to be sorted by bucket number. class ConvertingAggregatedToChunksTransform : public IProcessor { public: ConvertingAggregatedToChunksTransform(AggregatingTransformParamsPtr params_, ManyAggregatedDataVariantsPtr data_, size_t num_threads_) - : IProcessor({}, {params_->getHeader()}) - , params(std::move(params_)), data(std::move(data_)), num_threads(num_threads_) {} + : IProcessor({}, {params_->getHeader()}), params(std::move(params_)), data(std::move(data_)), num_threads(num_threads_) + { + } String getName() const override { return "ConvertingAggregatedToChunksTransform"; } @@ -298,7 +406,7 @@ private: AggregatingTransformParamsPtr params; ManyAggregatedDataVariantsPtr data; - ConvertingAggregatedToChunksSource::SharedDataPtr shared_data; + ConvertingAggregatedToChunksWithMergingSource::SharedDataPtr shared_data; size_t num_threads; @@ -368,13 +476,13 @@ private: void createSources() { AggregatedDataVariantsPtr & first = data->at(0); - shared_data = std::make_shared(); + shared_data = std::make_shared(); for (size_t thread = 0; thread < num_threads; ++thread) { /// Select Arena to avoid race conditions Arena * arena = first->aggregates_pools.at(thread).get(); - auto source = std::make_shared(params, data, shared_data, arena); + auto source = std::make_shared(params, data, shared_data, arena); processors.emplace_back(std::move(source)); } @@ -382,8 +490,7 @@ private: }; AggregatingTransform::AggregatingTransform(Block header, AggregatingTransformParamsPtr params_) - : AggregatingTransform(std::move(header), std::move(params_) - , std::make_unique(1), 0, 1, 1) + : AggregatingTransform(std::move(header), std::move(params_), std::make_unique(1), 0, 1, 1, false) { } @@ -391,18 +498,22 @@ AggregatingTransform::AggregatingTransform( Block header, AggregatingTransformParamsPtr params_, ManyAggregatedDataPtr many_data_, - size_t current_variant, + size_t current_variant_, size_t max_threads_, - size_t temporary_data_merge_threads_) + size_t temporary_data_merge_threads_, + bool skip_merging_) : IProcessor({std::move(header)}, {params_->getHeader()}) , params(std::move(params_)) , key_columns(params->params.keys_size) , aggregate_columns(params->params.aggregates_size) , many_data(std::move(many_data_)) - , variants(*many_data->variants[current_variant]) + , variants(*many_data->variants[current_variant_]) , max_threads(std::min(many_data->variants.size(), max_threads_)) , temporary_data_merge_threads(temporary_data_merge_threads_) + , current_variant(current_variant_) + , skip_merging(skip_merging_) { + (void)current_variant; } AggregatingTransform::~AggregatingTransform() = default; @@ -575,12 +686,30 @@ void AggregatingTransform::initGenerate() if (!params->aggregator.hasTemporaryData()) { - auto prepared_data = params->aggregator.prepareVariantsToMerge(many_data->variants); - auto prepared_data_ptr = std::make_shared(std::move(prepared_data)); - processors.emplace_back(std::make_shared(params, std::move(prepared_data_ptr), max_threads)); + if (!skip_merging) + { + auto prepared_data = params->aggregator.prepareVariantsToMerge(many_data->variants); + auto prepared_data_ptr = std::make_shared(std::move(prepared_data)); + processors.emplace_back( + std::make_shared(params, std::move(prepared_data_ptr), max_threads)); + } + else + { + auto prepared_data = params->aggregator.prepareVariantsToMerge(many_data->variants); + Pipes pipes; + for (auto & variant : prepared_data) + pipes.emplace_back(std::make_shared(params, variant)); + Pipe pipe = Pipe::unitePipes(std::move(pipes)); + pipe.addTransform(std::make_shared(pipe.getHeader(), pipe.numOutputPorts(), params)); + pipe.addTransform(std::make_shared(pipe.getHeader(), params->getHeader())); + processors = Pipe::detachProcessors(std::move(pipe)); + } } else { + if (skip_merging) + throw Exception(ErrorCodes::LOGICAL_ERROR, "not expected for external aggregation"); + /// If there are temporary files with partially-aggregated data on the disk, /// then read and merge them, spending the minimum amount of memory. diff --git a/src/Processors/Transforms/AggregatingTransform.h b/src/Processors/Transforms/AggregatingTransform.h index 0771761fa5c..24d9968bdb4 100644 --- a/src/Processors/Transforms/AggregatingTransform.h +++ b/src/Processors/Transforms/AggregatingTransform.h @@ -147,9 +147,10 @@ public: Block header, AggregatingTransformParamsPtr params_, ManyAggregatedDataPtr many_data, - size_t current_variant, + size_t current_variant_, size_t max_threads, - size_t temporary_data_merge_threads); + size_t temporary_data_merge_threads, + bool skip_merging_ = false); ~AggregatingTransform() override; String getName() const override { return "AggregatingTransform"; } @@ -181,6 +182,8 @@ private: AggregatedDataVariants & variants; size_t max_threads = 1; size_t temporary_data_merge_threads = 1; + size_t current_variant; + bool skip_merging = false; /// TODO: calculate time only for aggregation. Stopwatch watch; diff --git a/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.h b/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.h index 7c59ad1719f..8ff31782879 100644 --- a/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.h +++ b/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.h @@ -86,7 +86,8 @@ private: bool read_from_all_inputs = false; std::vector read_from_input; - bool expect_several_chunks_for_single_bucket_per_source = false; + /// If we aggregate partitioned data several chunks might be produced for the same bucket: one for each partition. + bool expect_several_chunks_for_single_bucket_per_source = true; /// Add chunk read from input to chunks_map, overflow_chunks or single_level_chunks according to it's chunk info. void addChunk(Chunk chunk, size_t input); diff --git a/tests/performance/aggregation_by_partitions.xml b/tests/performance/aggregation_by_partitions.xml new file mode 100644 index 00000000000..db6bbb8f215 --- /dev/null +++ b/tests/performance/aggregation_by_partitions.xml @@ -0,0 +1,13 @@ + + + + + + create table t(a UInt32) engine=MergeTree order by tuple() partition by a % 16 + + insert into t select * from numbers_mt(5e7) + + select a from t group by a format Null + + drop table t + diff --git a/tests/queries/0_stateless/02521_aggregation_by_partitions.reference b/tests/queries/0_stateless/02521_aggregation_by_partitions.reference new file mode 100644 index 00000000000..2670ad4212b --- /dev/null +++ b/tests/queries/0_stateless/02521_aggregation_by_partitions.reference @@ -0,0 +1,83 @@ +(Expression) +ExpressionTransform × 16 + (Aggregating) + Resize 4 → 16 + AggregatingTransform × 4 + (Expression) + ExpressionTransform × 4 + (ReadFromMergeTree) + Resize 3 → 1 + MergeTreeThread × 3 0 → 1 + Resize 3 → 1 + MergeTreeThread × 3 0 → 1 + Resize 3 → 1 + MergeTreeThread × 3 0 → 1 + Resize 3 → 1 + MergeTreeThread × 3 0 → 1 +1000000 +(Expression) +ExpressionTransform × 16 + (Aggregating) + Resize 8 → 16 + AggregatingTransform × 8 + (Expression) + ExpressionTransform × 8 + (ReadFromMergeTree) + Resize 2 → 1 + MergeTreeThread × 2 0 → 1 + Resize 2 → 1 + MergeTreeThread × 2 0 → 1 + Resize 2 → 1 + MergeTreeThread × 2 0 → 1 + Resize 2 → 1 + MergeTreeThread × 2 0 → 1 + Resize 2 → 1 + MergeTreeThread × 2 0 → 1 + Resize 2 → 1 + MergeTreeThread × 2 0 → 1 + Resize 2 → 1 + MergeTreeThread × 2 0 → 1 + Resize 2 → 1 + MergeTreeThread × 2 0 → 1 +1000000 +(Expression) +ExpressionTransform × 16 + (Aggregating) + Resize 16 → 16 + AggregatingTransform × 16 + (Expression) + ExpressionTransform × 16 + (ReadFromMergeTree) + Concat 2 → 1 + MergeTreeInOrder × 2 0 → 1 + Concat 2 → 1 + MergeTreeInOrder × 2 0 → 1 + Concat 2 → 1 + MergeTreeInOrder × 2 0 → 1 + Concat 2 → 1 + MergeTreeInOrder × 2 0 → 1 + Concat 2 → 1 + MergeTreeInOrder × 2 0 → 1 + Concat 2 → 1 + MergeTreeInOrder × 2 0 → 1 + Concat 2 → 1 + MergeTreeInOrder × 2 0 → 1 + Concat 2 → 1 + MergeTreeInOrder × 2 0 → 1 + Concat 2 → 1 + MergeTreeInOrder × 2 0 → 1 + Concat 2 → 1 + MergeTreeInOrder × 2 0 → 1 + Concat 2 → 1 + MergeTreeInOrder × 2 0 → 1 + Concat 2 → 1 + MergeTreeInOrder × 2 0 → 1 + Concat 2 → 1 + MergeTreeInOrder × 2 0 → 1 + Concat 2 → 1 + MergeTreeInOrder × 2 0 → 1 + Concat 2 → 1 + MergeTreeInOrder × 2 0 → 1 + Concat 2 → 1 + MergeTreeInOrder × 2 0 → 1 +1000000 diff --git a/tests/queries/0_stateless/02521_aggregation_by_partitions.sql b/tests/queries/0_stateless/02521_aggregation_by_partitions.sql new file mode 100644 index 00000000000..ed880ef8631 --- /dev/null +++ b/tests/queries/0_stateless/02521_aggregation_by_partitions.sql @@ -0,0 +1,42 @@ +set max_threads = 16; + +create table t1(a UInt32) engine=MergeTree order by tuple() partition by a % 4; + +system stop merges t1; + +insert into t1 select number from numbers_mt(1e6); +insert into t1 select number from numbers_mt(1e6); + +explain pipeline select a from t1 group by a; + +select count() from (select throwIf(count() != 2) from t1 group by a); + +drop table t1; + +create table t2(a UInt32) engine=MergeTree order by tuple() partition by a % 8; + +system stop merges t2; + +insert into t2 select number from numbers_mt(1e6); +insert into t2 select number from numbers_mt(1e6); + +explain pipeline select a from t2 group by a; + +select count() from (select throwIf(count() != 2) from t2 group by a); + +drop table t2; + +create table t3(a UInt32) engine=MergeTree order by tuple() partition by a % 16; + +system stop merges t3; + +insert into t3 select number from numbers_mt(1e6); +insert into t3 select number from numbers_mt(1e6); + +explain pipeline select a from t3 group by a; + +select count() from (select throwIf(count() != 2) from t3 group by a); + +select throwIf(count() != 4) from remote('127.0.0.{1,2}', currentDatabase(), t3) group by a format Null; + +drop table t3; From 1d45cce03c0e7e75b69254855b38b7d369cc52ef Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Wed, 11 Jan 2023 21:10:51 +0000 Subject: [PATCH 063/717] support for aggr in order --- src/Processors/QueryPlan/AggregatingStep.cpp | 9 ++ .../QueryPlan/ReadFromMergeTree.cpp | 152 +++++++++++------- src/Processors/QueryPlan/ReadFromMergeTree.h | 7 + .../performance/aggregation_by_partitions.xml | 4 +- .../02521_aggregation_by_partitions.reference | 113 +++++++++++++ .../02521_aggregation_by_partitions.sql | 43 +++++ 6 files changed, 267 insertions(+), 61 deletions(-) diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp index c0ea848fea8..468463a0504 100644 --- a/src/Processors/QueryPlan/AggregatingStep.cpp +++ b/src/Processors/QueryPlan/AggregatingStep.cpp @@ -369,6 +369,15 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B many_data, counter++); }); + if (skip_merging) + { + pipeline.addSimpleTransform([&](const Block & header) + { return std::make_shared(header, transform_params); }); + pipeline.resize(merge_threads); + aggregating_in_order = collector.detachProcessors(0); + return; + } + aggregating_in_order = collector.detachProcessors(0); auto transform = std::make_shared( diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index addfcb508ce..55aae54c427 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -353,12 +353,7 @@ Pipe ReadFromMergeTree::read( size_t max_streams, size_t min_marks_for_concurrent_read, bool use_uncompressed_cache) { if (read_type == ReadType::Default && max_streams > 1) - { - Pipe pipe = readFromPool(parts_with_range, required_columns, max_streams, min_marks_for_concurrent_read, use_uncompressed_cache); - if (output_each_partition_through_separate_port) - pipe.resize(1); - return pipe; - } + return readFromPool(parts_with_range, required_columns, max_streams, min_marks_for_concurrent_read, use_uncompressed_cache); auto pipe = readInOrder(parts_with_range, required_columns, read_type, use_uncompressed_cache, 0); @@ -468,9 +463,9 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsImpl( info.use_uncompressed_cache); } -Pipe ReadFromMergeTree::spreadMarkRangesAmongStreams( - RangesInDataParts && parts_with_ranges, - const Names & column_names) +template +static Pipe runForEachPartition( + RangesInDataParts && parts_with_ranges, size_t requested_num_streams, bool output_each_partition_through_separate_port, ReadFunc read) { if (parts_with_ranges.empty()) return {}; @@ -479,7 +474,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreams( if (!output_each_partition_through_separate_port) { - return spreadMarkRangesAmongStreamsImpl(std::move(parts_with_ranges), column_names, num_streams); + return read(std::move(parts_with_ranges), num_streams); } else { @@ -487,7 +482,8 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreams( LOG_DEBUG( &Poco::Logger::get("debug"), - "spreadMarkRangesAmongStreams {} {} {}", + "{} {} {} {}", + demangle(typeid(ReadFunc).name()).substr(0, 60), parts_with_ranges.size(), requested_num_streams, countPartitions(parts_with_ranges)); @@ -500,12 +496,19 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreams( parts_with_ranges.end(), [&begin](auto & part) { return begin->data_part->info.partition_id != part.data_part->info.partition_id; }); - LOG_DEBUG(&Poco::Logger::get("debug"), "spreadMarkRangesAmongStreams {} {}", begin->data_part->info.partition_id, end - begin); + LOG_DEBUG( + &Poco::Logger::get("debug"), + "{} {} {}", + demangle(typeid(ReadFunc).name()).substr(0, 60), + begin->data_part->info.partition_id, + end - begin); RangesInDataParts partition_parts; partition_parts.insert(partition_parts.end(), std::make_move_iterator(begin), std::make_move_iterator(end)); - pipes.emplace_back(spreadMarkRangesAmongStreamsImpl(std::move(partition_parts), column_names, num_streams)); + pipes.emplace_back(read(std::move(partition_parts), num_streams)); + if (!pipes.back().empty()) + pipes.back().resize(1); begin = end; } @@ -514,6 +517,15 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreams( } } +Pipe ReadFromMergeTree::spreadMarkRangesAmongStreams(RangesInDataParts && parts_with_ranges, const Names & column_names) +{ + auto read = [this, &column_names](RangesInDataParts && parts_with_ranges_, size_t num_streams_) + { + return spreadMarkRangesAmongStreamsImpl(std::move(parts_with_ranges_), column_names, num_streams_); + }; + return runForEachPartition(std::move(parts_with_ranges), requested_num_streams, output_each_partition_through_separate_port, read); +} + static ActionsDAGPtr createProjection(const Block & header) { auto projection = std::make_shared(header.getNamesAndTypesList()); @@ -522,41 +534,21 @@ static ActionsDAGPtr createProjection(const Block & header) return projection; } -Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder( +Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrderImpl( RangesInDataParts && parts_with_ranges, const Names & column_names, - ActionsDAGPtr & out_projection, - const InputOrderInfoPtr & input_order_info) + const InputOrderInfoPtr & input_order_info, + size_t num_streams, + bool need_preliminary_merge) { const auto & settings = context->getSettingsRef(); const auto data_settings = data.getSettings(); PartRangesReadInfo info(parts_with_ranges, settings, *data_settings); - Pipes res; - if (info.sum_marks == 0) return {}; - /// PREWHERE actions can remove some input columns (which are needed only for prewhere condition). - /// In case of read-in-order, PREWHERE is executed before sorting. But removed columns could be needed for sorting key. - /// To fix this, we prohibit removing any input in prewhere actions. Instead, projection actions will be added after sorting. - /// See 02354_read_in_order_prewhere.sql as an example. - bool have_input_columns_removed_after_prewhere = false; - if (prewhere_info && prewhere_info->prewhere_actions) - { - auto & outputs = prewhere_info->prewhere_actions->getOutputs(); - std::unordered_set outputs_set(outputs.begin(), outputs.end()); - for (const auto * input : prewhere_info->prewhere_actions->getInputs()) - { - if (!outputs_set.contains(input)) - { - outputs.push_back(input); - have_input_columns_removed_after_prewhere = true; - } - } - } - /// Let's split ranges to avoid reading much data. auto split_ranges = [rows_granularity = data_settings->index_granularity, max_block_size = max_block_size] (const auto & ranges, int direction) @@ -603,8 +595,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder( return new_ranges; }; - const size_t min_marks_per_stream = (info.sum_marks - 1) / requested_num_streams + 1; - bool need_preliminary_merge = (parts_with_ranges.size() > settings.read_in_order_two_level_merge_threshold); + const size_t min_marks_per_stream = (info.sum_marks - 1) / num_streams + 1; Pipes pipes; @@ -686,7 +677,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder( if (!pipes.empty()) pipe_header = pipes.front().getHeader(); - if (need_preliminary_merge) + if (need_preliminary_merge || output_each_partition_through_separate_port) { size_t prefix_size = input_order_info->used_prefix_of_sorting_key_size; auto order_key_prefix_ast = metadata_for_reading->getSortingKey().expression_list_ast->clone(); @@ -705,32 +696,79 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder( auto sorting_key_expr = std::make_shared(sorting_key_prefix_expr); - for (auto & pipe : pipes) + auto add_merge = [&](Pipe & pipe) { pipe.addSimpleTransform([sorting_key_expr](const Block & header) - { - return std::make_shared(header, sorting_key_expr); - }); + { return std::make_shared(header, sorting_key_expr); }); if (pipe.numOutputPorts() > 1) { auto transform = std::make_shared( - pipe.getHeader(), - pipe.numOutputPorts(), - sort_description, - max_block_size, - SortingQueueStrategy::Batch); + pipe.getHeader(), pipe.numOutputPorts(), sort_description, max_block_size, SortingQueueStrategy::Batch); pipe.addTransform(std::move(transform)); } + }; + + if (need_preliminary_merge) + for (auto & pipe : pipes) + add_merge(pipe); + + Pipe pipe = Pipe::unitePipes(std::move(pipes)); + + if (output_each_partition_through_separate_port) + add_merge(pipe); + + return pipe; + } + + return Pipe::unitePipes(std::move(pipes)); +} + +Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder( + RangesInDataParts && parts_with_ranges, + const Names & column_names, + ActionsDAGPtr & out_projection, + const InputOrderInfoPtr & input_order_info) +{ + if (parts_with_ranges.empty()) + return {}; + + /// PREWHERE actions can remove some input columns (which are needed only for prewhere condition). + /// In case of read-in-order, PREWHERE is executed before sorting. But removed columns could be needed for sorting key. + /// To fix this, we prohibit removing any input in prewhere actions. Instead, projection actions will be added after sorting. + /// See 02354_read_in_order_prewhere.sql as an example. + bool have_input_columns_removed_after_prewhere = false; + if (prewhere_info && prewhere_info->prewhere_actions) + { + auto & outputs = prewhere_info->prewhere_actions->getOutputs(); + std::unordered_set outputs_set(outputs.begin(), outputs.end()); + for (const auto * input : prewhere_info->prewhere_actions->getInputs()) + { + if (!outputs_set.contains(input)) + { + outputs.push_back(input); + have_input_columns_removed_after_prewhere = true; + } } } - if (!pipes.empty() && (need_preliminary_merge || have_input_columns_removed_after_prewhere)) - /// Drop temporary columns, added by 'sorting_key_prefix_expr' - out_projection = createProjection(pipe_header); + const auto & settings = context->getSettingsRef(); + bool need_preliminary_merge = (parts_with_ranges.size() > settings.read_in_order_two_level_merge_threshold); - return Pipe::unitePipes(std::move(pipes)); + auto read + = [this, &column_names, &input_order_info, need_preliminary_merge](RangesInDataParts && parts_with_ranges_, size_t num_streams_) + { + return spreadMarkRangesAmongStreamsWithOrderImpl( + std::move(parts_with_ranges_), column_names, input_order_info, num_streams_, need_preliminary_merge); + }; + Pipe pipe = runForEachPartition(std::move(parts_with_ranges), requested_num_streams, output_each_partition_through_separate_port, read); + + if (!pipe.empty() && (need_preliminary_merge || have_input_columns_removed_after_prewhere)) + /// Drop temporary columns, added by 'sorting_key_prefix_expr' + out_projection = createProjection(pipe.getHeader()); + + return pipe; } static void addMergingFinal( @@ -1269,14 +1307,11 @@ void ReadFromMergeTree::requestReadingInOrder(size_t prefix_size, int direction, output_stream->sort_description = std::move(sort_description); output_stream->sort_scope = DataStream::SortScope::Stream; } - - /// Not supported currently. Disable optimisation. - output_each_partition_through_separate_port = false; } bool ReadFromMergeTree::requestOutputEachPartitionThroughSeparatePort() { - if (isQueryWithFinal() || query_info.getInputOrderInfo()) + if (isQueryWithFinal()) return false; output_each_partition_through_separate_port = true; @@ -1377,9 +1412,6 @@ void ReadFromMergeTree::initializePipeline(QueryPipelineBuilder & pipeline, cons } else if (input_order_info) { - if (output_each_partition_through_separate_port) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Optimisation isn't supposed to be used when reading in order is used"); - pipe = spreadMarkRangesAmongStreamsWithOrder( std::move(result.parts_with_ranges), column_names_to_read, diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.h b/src/Processors/QueryPlan/ReadFromMergeTree.h index 1599e36fc41..1e7a1696a30 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.h +++ b/src/Processors/QueryPlan/ReadFromMergeTree.h @@ -238,6 +238,13 @@ private: ActionsDAGPtr & out_projection, const InputOrderInfoPtr & input_order_info); + Pipe spreadMarkRangesAmongStreamsWithOrderImpl( + RangesInDataParts && parts_with_ranges, + const Names & column_names, + const InputOrderInfoPtr & input_order_info, + size_t num_streams, + bool need_preliminary_merge); + Pipe spreadMarkRangesAmongStreamsFinal( RangesInDataParts && parts, const Names & column_names, diff --git a/tests/performance/aggregation_by_partitions.xml b/tests/performance/aggregation_by_partitions.xml index db6bbb8f215..263c6d211bd 100644 --- a/tests/performance/aggregation_by_partitions.xml +++ b/tests/performance/aggregation_by_partitions.xml @@ -3,11 +3,13 @@ - create table t(a UInt32) engine=MergeTree order by tuple() partition by a % 16 + create table t(a UInt32) engine=MergeTree order by a partition by a % 16 insert into t select * from numbers_mt(5e7) select a from t group by a format Null + select a from t group by a format Null settings optimize_aggregation_in_order = 1 + drop table t diff --git a/tests/queries/0_stateless/02521_aggregation_by_partitions.reference b/tests/queries/0_stateless/02521_aggregation_by_partitions.reference index 2670ad4212b..475e1867d03 100644 --- a/tests/queries/0_stateless/02521_aggregation_by_partitions.reference +++ b/tests/queries/0_stateless/02521_aggregation_by_partitions.reference @@ -81,3 +81,116 @@ ExpressionTransform × 16 Concat 2 → 1 MergeTreeInOrder × 2 0 → 1 1000000 +(Expression) +ExpressionTransform × 16 + (Aggregating) + Resize 4 → 16 + FinalizeAggregatedTransform × 4 + AggregatingInOrderTransform × 4 + (Expression) + ExpressionTransform × 4 + (ReadFromMergeTree) + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 +1000000 +(Expression) +ExpressionTransform × 16 + (Aggregating) + Resize 8 → 16 + FinalizeAggregatedTransform × 8 + AggregatingInOrderTransform × 8 + (Expression) + ExpressionTransform × 8 + (ReadFromMergeTree) + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 +1000000 +(Expression) +ExpressionTransform × 16 + (Aggregating) + FinalizeAggregatedTransform × 16 + AggregatingInOrderTransform × 16 + (Expression) + ExpressionTransform × 16 + (ReadFromMergeTree) + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 +1000000 diff --git a/tests/queries/0_stateless/02521_aggregation_by_partitions.sql b/tests/queries/0_stateless/02521_aggregation_by_partitions.sql index ed880ef8631..33127a0233a 100644 --- a/tests/queries/0_stateless/02521_aggregation_by_partitions.sql +++ b/tests/queries/0_stateless/02521_aggregation_by_partitions.sql @@ -40,3 +40,46 @@ select count() from (select throwIf(count() != 2) from t3 group by a); select throwIf(count() != 4) from remote('127.0.0.{1,2}', currentDatabase(), t3) group by a format Null; drop table t3; + +-- aggregation in order -- + +set optimize_aggregation_in_order = 1; + +create table t4(a UInt32) engine=MergeTree order by a partition by a % 4; + +system stop merges t4; + +insert into t4 select number from numbers_mt(1e6); +insert into t4 select number from numbers_mt(1e6); + +explain pipeline select a from t4 group by a settings read_in_order_two_level_merge_threshold = 1e12; + +select count() from (select throwIf(count() != 2) from t4 group by a); + +drop table t4; + +create table t5(a UInt32) engine=MergeTree order by a partition by a % 8; + +system stop merges t5; + +insert into t5 select number from numbers_mt(1e6); +insert into t5 select number from numbers_mt(1e6); + +explain pipeline select a from t5 group by a settings read_in_order_two_level_merge_threshold = 1e12; + +select count() from (select throwIf(count() != 2) from t5 group by a); + +drop table t5; + +create table t6(a UInt32) engine=MergeTree order by a partition by a % 16; + +system stop merges t6; + +insert into t6 select number from numbers_mt(1e6); +insert into t6 select number from numbers_mt(1e6); + +explain pipeline select a from t6 group by a settings read_in_order_two_level_merge_threshold = 1e12; + +select count() from (select throwIf(count() != 2) from t6 group by a); + +drop table t6; From 2057db68a265ab8f05e377e7be1066c5b8073280 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Wed, 11 Jan 2023 22:32:54 +0000 Subject: [PATCH 064/717] cosmetics --- src/Processors/QueryPlan/AggregatingStep.cpp | 8 +- .../QueryPlan/Optimizations/Optimizations.h | 4 +- .../useDataParallelAggregation.cpp | 2 +- .../QueryPlan/ReadFromMergeTree.cpp | 139 ++++++++++-------- .../Transforms/AggregatingTransform.cpp | 44 +++--- .../Transforms/AggregatingTransform.h | 5 +- .../performance/aggregation_by_partitions.xml | 51 ++++++- .../02521_aggregation_by_partitions.reference | 1 + .../02521_aggregation_by_partitions.sql | 3 + 9 files changed, 153 insertions(+), 104 deletions(-) diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp index 468463a0504..54cb2be4efb 100644 --- a/src/Processors/QueryPlan/AggregatingStep.cpp +++ b/src/Processors/QueryPlan/AggregatingStep.cpp @@ -166,12 +166,6 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B params.group_by_two_level_threshold_bytes = 0; } - /// In case of external aggregation we cannot completely avoid merging, - /// because each thread might use multiple hash tables during the execution and then the same key might be present in multiple hash tables. - /// But nevertheless we could save some time merging only HTs from the same thread (future task). - if (params.max_bytes_before_external_group_by) - skip_merging = false; - /** Two-level aggregation is useful in two cases: * 1. Parallel aggregation is done, and the results should be merged in parallel. * 2. An aggregation is done with store of temporary data on the disk, and they need to be merged in a memory efficient way. @@ -373,7 +367,7 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B { pipeline.addSimpleTransform([&](const Block & header) { return std::make_shared(header, transform_params); }); - pipeline.resize(merge_threads); + pipeline.resize(params.max_threads); aggregating_in_order = collector.detachProcessors(0); return; } diff --git a/src/Processors/QueryPlan/Optimizations/Optimizations.h b/src/Processors/QueryPlan/Optimizations/Optimizations.h index 91c2f2550e8..e171e94a670 100644 --- a/src/Processors/QueryPlan/Optimizations/Optimizations.h +++ b/src/Processors/QueryPlan/Optimizations/Optimizations.h @@ -68,7 +68,7 @@ void tryRemoveRedundantSorting(QueryPlan::Node * root); /// - Something - - Expression - Something - size_t tryLiftUpUnion(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes); -size_t tryAggregateEachPartitionIndependently(QueryPlan::Node * node, QueryPlan::Nodes &); +size_t tryAggregatePartitionsIndependently(QueryPlan::Node * node, QueryPlan::Nodes &); inline const auto & getOptimizations() { @@ -83,7 +83,7 @@ inline const auto & getOptimizations() "reuseStorageOrderingForWindowFunctions", &QueryPlanOptimizationSettings::optimize_plan}, {tryLiftUpUnion, "liftUpUnion", &QueryPlanOptimizationSettings::optimize_plan}, - {tryAggregateEachPartitionIndependently, "aggregationPartitionsIndepedently", &QueryPlanOptimizationSettings::optimize_plan}, + {tryAggregatePartitionsIndependently, "aggregatePartitionsIndependently", &QueryPlanOptimizationSettings::optimize_plan}, }}; return optimizations; diff --git a/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp b/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp index 2fd225bdcba..445e9e4806f 100644 --- a/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp +++ b/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp @@ -40,7 +40,7 @@ bool isPartitionKeySuitsGroupByKey(const ReadFromMergeTree & reading, const Aggr namespace DB::QueryPlanOptimizations { -size_t tryAggregateEachPartitionIndependently(QueryPlan::Node * node, QueryPlan::Nodes &) +size_t tryAggregatePartitionsIndependently(QueryPlan::Node * node, QueryPlan::Nodes &) { if (!node || node->children.size() != 1) return 0; diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 55aae54c427..cd66b8bc0e4 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -39,22 +39,74 @@ #include #include +using namespace DB; + namespace { -size_t countPartitions(DB::RangesInDataParts & parts_with_ranges) +template +size_t countPartitions(const Container & parts, Getter get_partition_id) { - std::string cur_partition_id = parts_with_ranges[0].data_part->info.partition_id; + assert(!parts_with_ranges.empty()); + String cur_partition_id = get_partition_id(parts[0]); size_t unique_partitions = 1; - for (size_t i = 1; i < parts_with_ranges.size(); ++i) + for (size_t i = 1; i < parts.size(); ++i) { - if (parts_with_ranges[i].data_part->info.partition_id != cur_partition_id) + if (get_partition_id(parts[i]) != cur_partition_id) { ++unique_partitions; - cur_partition_id = parts_with_ranges[i].data_part->info.partition_id; + cur_partition_id = get_partition_id(parts[i]); } } return unique_partitions; } + +size_t countPartitions(const RangesInDataParts & parts_with_ranges) +{ + auto get_partition_id = [](const RangesInDataPart & rng) { return rng.data_part->info.partition_id; }; + return countPartitions(parts_with_ranges, get_partition_id); +} + +size_t countPartitions(const MergeTreeData::DataPartsVector & prepared_parts) +{ + auto get_partition_id = [](const MergeTreeData::DataPartPtr data_part) { return data_part->info.partition_id; }; + return countPartitions(prepared_parts, get_partition_id); +} + +template +Pipe outputPerPartitionIfRequested( + RangesInDataParts && parts_with_ranges, size_t num_streams, bool output_each_partition_through_separate_port, ReadFunc read) +{ + if (parts_with_ranges.empty()) + return {}; + + if (!output_each_partition_through_separate_port) + { + return read(std::move(parts_with_ranges), num_streams); + } + else + { + num_streams = std::max(1, num_streams / countPartitions(parts_with_ranges)); + + Pipes pipes; + for (auto begin = parts_with_ranges.begin(); begin != parts_with_ranges.end();) + { + const auto end = std::find_if( + begin, + parts_with_ranges.end(), + [&begin](const auto & part) { return begin->data_part->info.partition_id != part.data_part->info.partition_id; }); + + RangesInDataParts partition_parts{std::make_move_iterator(begin), std::make_move_iterator(end)}; + + pipes.emplace_back(read(std::move(partition_parts), num_streams)); + if (!pipes.back().empty()) + pipes.back().resize(1); + + begin = end; + } + + return Pipe::unitePipes(std::move(pipes)); + } +} } namespace ProfileEvents @@ -463,67 +515,14 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsImpl( info.use_uncompressed_cache); } -template -static Pipe runForEachPartition( - RangesInDataParts && parts_with_ranges, size_t requested_num_streams, bool output_each_partition_through_separate_port, ReadFunc read) -{ - if (parts_with_ranges.empty()) - return {}; - - size_t num_streams = requested_num_streams; - - if (!output_each_partition_through_separate_port) - { - return read(std::move(parts_with_ranges), num_streams); - } - else - { - num_streams = std::max(1, num_streams / countPartitions(parts_with_ranges)); - - LOG_DEBUG( - &Poco::Logger::get("debug"), - "{} {} {} {}", - demangle(typeid(ReadFunc).name()).substr(0, 60), - parts_with_ranges.size(), - requested_num_streams, - countPartitions(parts_with_ranges)); - - Pipes pipes; - for (auto begin = parts_with_ranges.begin(); begin != parts_with_ranges.end();) - { - const auto end = std::find_if( - begin, - parts_with_ranges.end(), - [&begin](auto & part) { return begin->data_part->info.partition_id != part.data_part->info.partition_id; }); - - LOG_DEBUG( - &Poco::Logger::get("debug"), - "{} {} {}", - demangle(typeid(ReadFunc).name()).substr(0, 60), - begin->data_part->info.partition_id, - end - begin); - - RangesInDataParts partition_parts; - partition_parts.insert(partition_parts.end(), std::make_move_iterator(begin), std::make_move_iterator(end)); - - pipes.emplace_back(read(std::move(partition_parts), num_streams)); - if (!pipes.back().empty()) - pipes.back().resize(1); - - begin = end; - } - - return Pipe::unitePipes(std::move(pipes)); - } -} - Pipe ReadFromMergeTree::spreadMarkRangesAmongStreams(RangesInDataParts && parts_with_ranges, const Names & column_names) { auto read = [this, &column_names](RangesInDataParts && parts_with_ranges_, size_t num_streams_) { return spreadMarkRangesAmongStreamsImpl(std::move(parts_with_ranges_), column_names, num_streams_); }; - return runForEachPartition(std::move(parts_with_ranges), requested_num_streams, output_each_partition_through_separate_port, read); + return outputPerPartitionIfRequested( + std::move(parts_with_ranges), requested_num_streams, output_each_partition_through_separate_port, read); } static ActionsDAGPtr createProjection(const Block & header) @@ -710,7 +709,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrderImpl( } }; - if (need_preliminary_merge) + if (!output_each_partition_through_separate_port) for (auto & pipe : pipes) add_merge(pipe); @@ -762,7 +761,8 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder( return spreadMarkRangesAmongStreamsWithOrderImpl( std::move(parts_with_ranges_), column_names, input_order_info, num_streams_, need_preliminary_merge); }; - Pipe pipe = runForEachPartition(std::move(parts_with_ranges), requested_num_streams, output_each_partition_through_separate_port, read); + Pipe pipe = outputPerPartitionIfRequested( + std::move(parts_with_ranges), requested_num_streams, output_each_partition_through_separate_port, read); if (!pipe.empty() && (need_preliminary_merge || have_input_columns_removed_after_prewhere)) /// Drop temporary columns, added by 'sorting_key_prefix_expr' @@ -1314,8 +1314,19 @@ bool ReadFromMergeTree::requestOutputEachPartitionThroughSeparatePort() if (isQueryWithFinal()) return false; - output_each_partition_through_separate_port = true; - return true; + const auto & settings = context->getSettingsRef(); + + const auto partitions_cnt = countPartitions(prepared_parts); + if (partitions_cnt < settings.max_threads / 20000) + { + LOG_TRACE(log, "no cartoons"); + return false; + } + + /* size_t min_marks_in_partition = -1; */ + /* size_t max_marks_in_partition = 0; */ + + return output_each_partition_through_separate_port = true; } ReadFromMergeTree::AnalysisResult ReadFromMergeTree::getAnalysisResult() const diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index f57ac4774d6..27c69adaf14 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -149,10 +149,10 @@ public: protected: Chunk generate() override { - if (!converted) + if (!convertion_is_done) { blocks = params->aggregator.convertToBlocks(*variant, params->final, 1 /* max_threads */); - converted = true; + convertion_is_done = true; } if (blocks.empty()) @@ -167,7 +167,7 @@ private: AggregatingTransformParamsPtr params; AggregatedDataVariantsPtr variant; - bool converted = false; + bool convertion_is_done = false; BlocksList blocks; }; @@ -189,13 +189,17 @@ private: void process(Chunk chunk) { - if (chunk.hasChunkInfo()) - { - const auto & info = chunk.getChunkInfo(); - if (const auto * chunks_to_merge = typeid_cast(info.get()); chunks_to_merge && chunks_to_merge->chunks) - for (auto & cur_chunk : *chunks_to_merge->chunks) - chunks.emplace_back(std::move(cur_chunk)); - } + if (!chunk.hasChunkInfo()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected chunk with chunk info in {}", getName()); + + const auto & info = chunk.getChunkInfo(); + const auto * chunks_to_merge = typeid_cast(info.get()); + if (!chunks_to_merge) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected chunk with ChunksToMerge info in {}", getName()); + + if (chunks_to_merge->chunks) + for (auto & cur_chunk : *chunks_to_merge->chunks) + chunks.emplace_back(std::move(cur_chunk)); } Status prepare() override @@ -219,6 +223,9 @@ private: { output.push(std::move(chunks.front())); chunks.pop_front(); + + if (!chunks.empty()) + return Status::Ready; } if (input.isFinished() && chunks.empty()) @@ -235,7 +242,7 @@ private: if (!input.hasData()) return Status::NeedData; - Chunk chunk = input.pull(false /* set_not_needed */); + Chunk chunk = input.pull(true /* set_not_needed */); process(std::move(chunk)); /// Now transform. @@ -490,7 +497,8 @@ private: }; AggregatingTransform::AggregatingTransform(Block header, AggregatingTransformParamsPtr params_) - : AggregatingTransform(std::move(header), std::move(params_), std::make_unique(1), 0, 1, 1, false) + : AggregatingTransform( + std::move(header), std::move(params_), std::make_unique(1), 0, 1, 1, false /* skip_merging */) { } @@ -498,7 +506,7 @@ AggregatingTransform::AggregatingTransform( Block header, AggregatingTransformParamsPtr params_, ManyAggregatedDataPtr many_data_, - size_t current_variant_, + size_t current_variant, size_t max_threads_, size_t temporary_data_merge_threads_, bool skip_merging_) @@ -507,13 +515,11 @@ AggregatingTransform::AggregatingTransform( , key_columns(params->params.keys_size) , aggregate_columns(params->params.aggregates_size) , many_data(std::move(many_data_)) - , variants(*many_data->variants[current_variant_]) + , variants(*many_data->variants[current_variant]) , max_threads(std::min(many_data->variants.size(), max_threads_)) , temporary_data_merge_threads(temporary_data_merge_threads_) - , current_variant(current_variant_) , skip_merging(skip_merging_) { - (void)current_variant; } AggregatingTransform::~AggregatingTransform() = default; @@ -696,20 +702,20 @@ void AggregatingTransform::initGenerate() else { auto prepared_data = params->aggregator.prepareVariantsToMerge(many_data->variants); + /// Converts hash tables to blocks with data (finalized or not). Pipes pipes; for (auto & variant : prepared_data) pipes.emplace_back(std::make_shared(params, variant)); Pipe pipe = Pipe::unitePipes(std::move(pipes)); + /// Groups chunks with the same bucket_id and outputs them (as a vector of chunks) in order of bucket_id. pipe.addTransform(std::make_shared(pipe.getHeader(), pipe.numOutputPorts(), params)); + /// Outputs one chunk from group at a time in order of bucket_id. pipe.addTransform(std::make_shared(pipe.getHeader(), params->getHeader())); processors = Pipe::detachProcessors(std::move(pipe)); } } else { - if (skip_merging) - throw Exception(ErrorCodes::LOGICAL_ERROR, "not expected for external aggregation"); - /// If there are temporary files with partially-aggregated data on the disk, /// then read and merge them, spending the minimum amount of memory. diff --git a/src/Processors/Transforms/AggregatingTransform.h b/src/Processors/Transforms/AggregatingTransform.h index 24d9968bdb4..2cc915d1ac7 100644 --- a/src/Processors/Transforms/AggregatingTransform.h +++ b/src/Processors/Transforms/AggregatingTransform.h @@ -147,7 +147,7 @@ public: Block header, AggregatingTransformParamsPtr params_, ManyAggregatedDataPtr many_data, - size_t current_variant_, + size_t current_variant, size_t max_threads, size_t temporary_data_merge_threads, bool skip_merging_ = false); @@ -182,8 +182,7 @@ private: AggregatedDataVariants & variants; size_t max_threads = 1; size_t temporary_data_merge_threads = 1; - size_t current_variant; - bool skip_merging = false; + bool skip_merging = false; /// if we aggregate partitioned data merging is not needed /// TODO: calculate time only for aggregation. Stopwatch watch; diff --git a/tests/performance/aggregation_by_partitions.xml b/tests/performance/aggregation_by_partitions.xml index 263c6d211bd..e24f589508e 100644 --- a/tests/performance/aggregation_by_partitions.xml +++ b/tests/performance/aggregation_by_partitions.xml @@ -1,15 +1,50 @@ - - - + + + 0 + 4096 + - create table t(a UInt32) engine=MergeTree order by a partition by a % 16 + + + size + + 10000 + 100000 + 1000000 + 10000000 + 100000000 + + - insert into t select * from numbers_mt(5e7) + + partitions + + 2 + 4 + 8 + 16 + 32 + 64 + 128 + 256 + + + + + + + - select a from t group by a format Null + create table t_{size}_{partitions}(a UInt64) engine=MergeTree order by a partition by a % {partitions} - select a from t group by a format Null settings optimize_aggregation_in_order = 1 + insert into t_{size}_{partitions} select * from numbers_mt({size}) - drop table t + + + select a from t_{size}_{partitions} group by a format Null + + select a from t_{size}_{partitions} group by a format Null settings optimize_aggregation_in_order = 1 + + drop table t_{size}_{partitions} diff --git a/tests/queries/0_stateless/02521_aggregation_by_partitions.reference b/tests/queries/0_stateless/02521_aggregation_by_partitions.reference index 475e1867d03..3e396bafa38 100644 --- a/tests/queries/0_stateless/02521_aggregation_by_partitions.reference +++ b/tests/queries/0_stateless/02521_aggregation_by_partitions.reference @@ -81,6 +81,7 @@ ExpressionTransform × 16 Concat 2 → 1 MergeTreeInOrder × 2 0 → 1 1000000 +1000000 (Expression) ExpressionTransform × 16 (Aggregating) diff --git a/tests/queries/0_stateless/02521_aggregation_by_partitions.sql b/tests/queries/0_stateless/02521_aggregation_by_partitions.sql index 33127a0233a..870116d4a23 100644 --- a/tests/queries/0_stateless/02521_aggregation_by_partitions.sql +++ b/tests/queries/0_stateless/02521_aggregation_by_partitions.sql @@ -39,6 +39,9 @@ select count() from (select throwIf(count() != 2) from t3 group by a); select throwIf(count() != 4) from remote('127.0.0.{1,2}', currentDatabase(), t3) group by a format Null; +-- if we happened to switch to external aggregation at some point, merging will happen as usual +select count() from (select throwIf(count() != 2) from t3 group by a) settings max_bytes_before_external_group_by = '1Ki'; + drop table t3; -- aggregation in order -- From d7e1e6314c81926bbaae8b0a0cf83f1e970b698e Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Mon, 16 Jan 2023 18:54:05 +0000 Subject: [PATCH 065/717] move matchTrees to separate file --- .../Optimizations/actionsDAGUtils.cpp | 216 ++++++++++++++ .../QueryPlan/Optimizations/actionsDAGUtils.h | 43 +++ .../Optimizations/optimizeReadInOrder.cpp | 280 ++---------------- 3 files changed, 280 insertions(+), 259 deletions(-) create mode 100644 src/Processors/QueryPlan/Optimizations/actionsDAGUtils.cpp create mode 100644 src/Processors/QueryPlan/Optimizations/actionsDAGUtils.h diff --git a/src/Processors/QueryPlan/Optimizations/actionsDAGUtils.cpp b/src/Processors/QueryPlan/Optimizations/actionsDAGUtils.cpp new file mode 100644 index 00000000000..e88d165dbae --- /dev/null +++ b/src/Processors/QueryPlan/Optimizations/actionsDAGUtils.cpp @@ -0,0 +1,216 @@ +#include + +#include +#include + +#include + +namespace DB +{ +MatchedTrees::Matches matchTrees(const ActionsDAG & inner_dag, const ActionsDAG & outer_dag) +{ + using Parents = std::set; + std::unordered_map inner_parents; + std::unordered_map inner_inputs; + + { + std::stack stack; + for (const auto * out : inner_dag.getOutputs()) + { + if (inner_parents.contains(out)) + continue; + + stack.push(out); + inner_parents.emplace(out, Parents()); + while (!stack.empty()) + { + const auto * node = stack.top(); + stack.pop(); + + if (node->type == ActionsDAG::ActionType::INPUT) + inner_inputs.emplace(node->result_name, node); + + for (const auto * child : node->children) + { + auto [it, inserted] = inner_parents.emplace(child, Parents()); + it->second.emplace(node); + + if (inserted) + stack.push(child); + } + } + } + } + + struct Frame + { + const ActionsDAG::Node * node; + ActionsDAG::NodeRawConstPtrs mapped_children; + }; + + MatchedTrees::Matches matches; + std::stack stack; + + for (const auto & node : outer_dag.getNodes()) + { + if (matches.contains(&node)) + continue; + + stack.push(Frame{&node, {}}); + while (!stack.empty()) + { + auto & frame = stack.top(); + frame.mapped_children.reserve(frame.node->children.size()); + + while (frame.mapped_children.size() < frame.node->children.size()) + { + const auto * child = frame.node->children[frame.mapped_children.size()]; + auto it = matches.find(child); + if (it == matches.end()) + { + /// If match map does not contain a child, it was not visited. + stack.push(Frame{child, {}}); + break; + } + /// A node from found match may be nullptr. + /// It means that node is visited, but no match was found. + frame.mapped_children.push_back(it->second.node); + } + + if (frame.mapped_children.size() < frame.node->children.size()) + continue; + + /// Create an empty match for current node. + /// natch.node will be set if match is found. + auto & match = matches[frame.node]; + + if (frame.node->type == ActionsDAG::ActionType::INPUT) + { + const ActionsDAG::Node * mapped = nullptr; + if (auto it = inner_inputs.find(frame.node->result_name); it != inner_inputs.end()) + mapped = it->second; + + match.node = mapped; + } + else if (frame.node->type == ActionsDAG::ActionType::ALIAS) + { + match = matches[frame.node->children.at(0)]; + } + else if (frame.node->type == ActionsDAG::ActionType::FUNCTION) + { + //std::cerr << "... Processing " << frame.node->function_base->getName() << std::endl; + + bool found_all_children = true; + for (const auto * child : frame.mapped_children) + if (!child) + found_all_children = false; + + if (found_all_children && !frame.mapped_children.empty()) + { + Parents container; + Parents * intersection = &inner_parents[frame.mapped_children[0]]; + + if (frame.mapped_children.size() > 1) + { + std::vector other_parents; + size_t mapped_children_size = frame.mapped_children.size(); + other_parents.reserve(mapped_children_size); + for (size_t i = 1; i < mapped_children_size; ++i) + other_parents.push_back(&inner_parents[frame.mapped_children[i]]); + + for (const auto * parent : *intersection) + { + bool is_common = true; + for (const auto * set : other_parents) + { + if (!set->contains(parent)) + { + is_common = false; + break; + } + } + + if (is_common) + container.insert(parent); + } + + intersection = &container; + } + + //std::cerr << ".. Candidate parents " << intersection->size() << std::endl; + + if (!intersection->empty()) + { + auto func_name = frame.node->function_base->getName(); + for (const auto * parent : *intersection) + { + //std::cerr << ".. candidate " << parent->result_name << std::endl; + if (parent->type == ActionsDAG::ActionType::FUNCTION && func_name == parent->function_base->getName()) + { + const auto & children = parent->children; + size_t num_children = children.size(); + if (frame.mapped_children.size() == num_children) + { + bool all_children_matched = true; + for (size_t i = 0; all_children_matched && i < num_children; ++i) + all_children_matched = frame.mapped_children[i] == children[i]; + + if (all_children_matched) + { + match.node = parent; + break; + } + } + } + } + } + } + + if (!match.node && frame.node->function_base->hasInformationAboutMonotonicity()) + { + size_t num_const_args = 0; + const ActionsDAG::Node * monotonic_child = nullptr; + for (const auto * child : frame.node->children) + { + if (child->column) + ++num_const_args; + else + monotonic_child = child; + } + + if (monotonic_child && num_const_args + 1 == frame.node->children.size()) + { + const auto & child_match = matches[monotonic_child]; + if (child_match.node) + { + auto info = frame.node->function_base->getMonotonicityForRange(*monotonic_child->result_type, {}, {}); + if (info.is_monotonic) + { + MatchedTrees::Monotonicity monotonicity; + monotonicity.direction *= info.is_positive ? 1 : -1; + monotonicity.strict = info.is_strict; + + if (child_match.monotonicity) + { + monotonicity.direction *= child_match.monotonicity->direction; + if (!child_match.monotonicity->strict) + monotonicity.strict = false; + } + + match.node = child_match.node; + match.monotonicity = monotonicity; + } + } + } + } + } + + stack.pop(); + } + } + + return matches; +} + + +} diff --git a/src/Processors/QueryPlan/Optimizations/actionsDAGUtils.h b/src/Processors/QueryPlan/Optimizations/actionsDAGUtils.h new file mode 100644 index 00000000000..dd689cba46b --- /dev/null +++ b/src/Processors/QueryPlan/Optimizations/actionsDAGUtils.h @@ -0,0 +1,43 @@ +#pragma once + +#include + +namespace DB +{ + +/// This structure stores a node mapping from one DAG to another. +/// The rule is following: +/// * Input nodes are mapped by name. +/// * Function is mapped to function if all children are mapped and function names are same. +/// * Alias is mapped to it's children mapping. +/// * Monotonic function can be mapped to it's children mapping if direct mapping does not exist. +/// In this case, information about monotonicity is filled. +/// * Mapped node is nullptr if there is no mapping found. +/// +/// Overall, directly mapped nodes represent equal calculations. +/// Notes: +/// * Mapped DAG can contain many nodes which represent the same calculation. +/// In this case mapping is ambiguous and only one node is mapped. +/// * Aliases for mapped DAG are not supported. +/// DAG for PK does not contain aliases and ambiguous nodes. +struct MatchedTrees +{ + /// Monotonicity is calculated for monotonic functions chain. + /// Chain is not strict if there is any non-strict monotonic function. + struct Monotonicity + { + int direction = 1; + bool strict = true; + }; + + struct Match + { + const ActionsDAG::Node * node = nullptr; + std::optional monotonicity; + }; + + using Matches = std::unordered_map; +}; + +MatchedTrees::Matches matchTrees(const ActionsDAG & inner_dag, const ActionsDAG & outer_dag); +} diff --git a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp index bdf8f24f9d6..9600b8cc416 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp @@ -1,27 +1,29 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include #include #include #include #include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include -#include -#include -#include +#include + #include @@ -289,246 +291,6 @@ void enreachFixedColumns(const ActionsDAG & dag, FixedColumns & fixed_columns) } } -/// This structure stores a node mapping from one DAG to another. -/// The rule is following: -/// * Input nodes are mapped by name. -/// * Function is mapped to function if all children are mapped and function names are same. -/// * Alias is mapped to it's children mapping. -/// * Monotonic function can be mapped to it's children mapping if direct mapping does not exist. -/// In this case, information about monotonicity is filled. -/// * Mapped node is nullptr if there is no mapping found. -/// -/// Overall, directly mapped nodes represent equal calculations. -/// Notes: -/// * Mapped DAG can contain many nodes which represent the same calculation. -/// In this case mapping is ambiguous and only one node is mapped. -/// * Aliases for mapped DAG are not supported. -/// DAG for PK does not contain aliases and ambiguous nodes. -struct MatchedTrees -{ - /// Monotonicity is calculated for monotonic functions chain. - /// Chain is not strict if there is any non-strict monotonic function. - struct Monotonicity - { - int direction = 1; - bool strict = true; - }; - - struct Match - { - const ActionsDAG::Node * node = nullptr; - std::optional monotonicity; - }; - - using Matches = std::unordered_map; -}; - -MatchedTrees::Matches matchTrees(const ActionsDAG & inner_dag, const ActionsDAG & outer_dag) -{ - using Parents = std::set; - std::unordered_map inner_parents; - std::unordered_map inner_inputs; - - { - std::stack stack; - for (const auto * out : inner_dag.getOutputs()) - { - if (inner_parents.contains(out)) - continue; - - stack.push(out); - inner_parents.emplace(out, Parents()); - while (!stack.empty()) - { - const auto * node = stack.top(); - stack.pop(); - - if (node->type == ActionsDAG::ActionType::INPUT) - inner_inputs.emplace(node->result_name, node); - - for (const auto * child : node->children) - { - auto [it, inserted] = inner_parents.emplace(child, Parents()); - it->second.emplace(node); - - if (inserted) - stack.push(child); - } - } - } - } - - struct Frame - { - const ActionsDAG::Node * node; - ActionsDAG::NodeRawConstPtrs mapped_children; - }; - - MatchedTrees::Matches matches; - std::stack stack; - - for (const auto & node : outer_dag.getNodes()) - { - if (matches.contains(&node)) - continue; - - stack.push(Frame{&node, {}}); - while (!stack.empty()) - { - auto & frame = stack.top(); - frame.mapped_children.reserve(frame.node->children.size()); - - while (frame.mapped_children.size() < frame.node->children.size()) - { - const auto * child = frame.node->children[frame.mapped_children.size()]; - auto it = matches.find(child); - if (it == matches.end()) - { - /// If match map does not contain a child, it was not visited. - stack.push(Frame{child, {}}); - break; - } - /// A node from found match may be nullptr. - /// It means that node is visited, but no match was found. - frame.mapped_children.push_back(it->second.node); - } - - if (frame.mapped_children.size() < frame.node->children.size()) - continue; - - /// Create an empty match for current node. - /// natch.node will be set if match is found. - auto & match = matches[frame.node]; - - if (frame.node->type == ActionsDAG::ActionType::INPUT) - { - const ActionsDAG::Node * mapped = nullptr; - if (auto it = inner_inputs.find(frame.node->result_name); it != inner_inputs.end()) - mapped = it->second; - - match.node = mapped; - } - else if (frame.node->type == ActionsDAG::ActionType::ALIAS) - { - match = matches[frame.node->children.at(0)]; - } - else if (frame.node->type == ActionsDAG::ActionType::FUNCTION) - { - - //std::cerr << "... Processing " << frame.node->function_base->getName() << std::endl; - - bool found_all_children = true; - for (const auto * child : frame.mapped_children) - if (!child) - found_all_children = false; - - if (found_all_children && !frame.mapped_children.empty()) - { - Parents container; - Parents * intersection = &inner_parents[frame.mapped_children[0]]; - - if (frame.mapped_children.size() > 1) - { - std::vector other_parents; - size_t mapped_children_size = frame.mapped_children.size(); - other_parents.reserve(mapped_children_size); - for (size_t i = 1; i < mapped_children_size; ++i) - other_parents.push_back(&inner_parents[frame.mapped_children[i]]); - - for (const auto * parent : *intersection) - { - bool is_common = true; - for (const auto * set : other_parents) - { - if (!set->contains(parent)) - { - is_common = false; - break; - } - } - - if (is_common) - container.insert(parent); - } - - intersection = &container; - } - - //std::cerr << ".. Candidate parents " << intersection->size() << std::endl; - - if (!intersection->empty()) - { - auto func_name = frame.node->function_base->getName(); - for (const auto * parent : *intersection) - { - //std::cerr << ".. candidate " << parent->result_name << std::endl; - if (parent->type == ActionsDAG::ActionType::FUNCTION && func_name == parent->function_base->getName()) - { - const auto & children = parent->children; - size_t num_children = children.size(); - if (frame.mapped_children.size() == num_children) - { - bool all_children_matched = true; - for (size_t i = 0; all_children_matched && i < num_children; ++i) - all_children_matched = frame.mapped_children[i] == children[i]; - - if (all_children_matched) - { - match.node = parent; - break; - } - } - } - } - } - } - - if (!match.node && frame.node->function_base->hasInformationAboutMonotonicity()) - { - size_t num_const_args = 0; - const ActionsDAG::Node * monotonic_child = nullptr; - for (const auto * child : frame.node->children) - { - if (child->column) - ++num_const_args; - else - monotonic_child = child; - } - - if (monotonic_child && num_const_args + 1 == frame.node->children.size()) - { - const auto & child_match = matches[monotonic_child]; - if (child_match.node) - { - auto info = frame.node->function_base->getMonotonicityForRange(*monotonic_child->result_type, {}, {}); - if (info.is_monotonic) - { - MatchedTrees::Monotonicity monotonicity; - monotonicity.direction *= info.is_positive ? 1 : -1; - monotonicity.strict = info.is_strict; - - if (child_match.monotonicity) - { - monotonicity.direction *= child_match.monotonicity->direction; - if (!child_match.monotonicity->strict) - monotonicity.strict = false; - } - - match.node = child_match.node; - match.monotonicity = monotonicity; - } - } - } - } - } - - stack.pop(); - } - } - - return matches; -} - InputOrderInfoPtr buildInputOrderInfo( const FixedColumns & fixed_columns, const ActionsDAGPtr & dag, From abe9ab2dff8461b1467f6aebc8bdd72d260dc315 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Mon, 16 Jan 2023 20:10:50 +0000 Subject: [PATCH 066/717] small fix for matchTrees --- .../QueryPlan/Optimizations/actionsDAGUtils.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Processors/QueryPlan/Optimizations/actionsDAGUtils.cpp b/src/Processors/QueryPlan/Optimizations/actionsDAGUtils.cpp index e88d165dbae..5d02436444f 100644 --- a/src/Processors/QueryPlan/Optimizations/actionsDAGUtils.cpp +++ b/src/Processors/QueryPlan/Optimizations/actionsDAGUtils.cpp @@ -11,7 +11,7 @@ MatchedTrees::Matches matchTrees(const ActionsDAG & inner_dag, const ActionsDAG { using Parents = std::set; std::unordered_map inner_parents; - std::unordered_map inner_inputs; + std::unordered_map inner_inputs_and_constants; { std::stack stack; @@ -27,8 +27,8 @@ MatchedTrees::Matches matchTrees(const ActionsDAG & inner_dag, const ActionsDAG const auto * node = stack.top(); stack.pop(); - if (node->type == ActionsDAG::ActionType::INPUT) - inner_inputs.emplace(node->result_name, node); + if (node->type == ActionsDAG::ActionType::INPUT || node->type == ActionsDAG::ActionType::COLUMN) + inner_inputs_and_constants.emplace(node->result_name, node); for (const auto * child : node->children) { @@ -84,10 +84,10 @@ MatchedTrees::Matches matchTrees(const ActionsDAG & inner_dag, const ActionsDAG /// natch.node will be set if match is found. auto & match = matches[frame.node]; - if (frame.node->type == ActionsDAG::ActionType::INPUT) + if (frame.node->type == ActionsDAG::ActionType::INPUT || frame.node->type == ActionsDAG::ActionType::COLUMN) { const ActionsDAG::Node * mapped = nullptr; - if (auto it = inner_inputs.find(frame.node->result_name); it != inner_inputs.end()) + if (auto it = inner_inputs_and_constants.find(frame.node->result_name); it != inner_inputs_and_constants.end()) mapped = it->second; match.node = mapped; @@ -153,7 +153,7 @@ MatchedTrees::Matches matchTrees(const ActionsDAG & inner_dag, const ActionsDAG { bool all_children_matched = true; for (size_t i = 0; all_children_matched && i < num_children; ++i) - all_children_matched = frame.mapped_children[i] == children[i]; + all_children_matched &= frame.mapped_children[i] == children[i]; if (all_children_matched) { From b59ba628a94b55fb4508904d7fcd7b1f3726eb42 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Mon, 16 Jan 2023 22:13:40 +0000 Subject: [PATCH 067/717] do not group buckets w/o need --- src/Processors/QueryPlan/AggregatingStep.cpp | 10 ++++++- .../Transforms/AggregatingTransform.cpp | 28 +++++++++++++++---- .../Transforms/AggregatingTransform.h | 4 ++- 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp index 54cb2be4efb..a11451396bc 100644 --- a/src/Processors/QueryPlan/AggregatingStep.cpp +++ b/src/Processors/QueryPlan/AggregatingStep.cpp @@ -235,6 +235,7 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B j, merge_threads, temporary_data_merge_threads, + should_produce_results_in_order_of_bucket_number, skip_merging); // For each input stream we have `grouping_sets_size` copies, so port index // for transform #j should skip ports of first (j-1) streams. @@ -437,7 +438,14 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B [&](const Block & header) { return std::make_shared( - header, transform_params, many_data, counter++, merge_threads, temporary_data_merge_threads, skip_merging); + header, + transform_params, + many_data, + counter++, + merge_threads, + temporary_data_merge_threads, + should_produce_results_in_order_of_bucket_number, + skip_merging); }); pipeline.resize(should_produce_results_in_order_of_bucket_number ? 1 : params.max_threads, true /* force */); diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index 27c69adaf14..4633750eba3 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -498,7 +498,14 @@ private: AggregatingTransform::AggregatingTransform(Block header, AggregatingTransformParamsPtr params_) : AggregatingTransform( - std::move(header), std::move(params_), std::make_unique(1), 0, 1, 1, false /* skip_merging */) + std::move(header), + std::move(params_), + std::make_unique(1), + 0, + 1, + 1, + true /* should_produce_results_in_order_of_bucket_number */, + false /* skip_merging */) { } @@ -509,6 +516,7 @@ AggregatingTransform::AggregatingTransform( size_t current_variant, size_t max_threads_, size_t temporary_data_merge_threads_, + bool should_produce_results_in_order_of_bucket_number_, bool skip_merging_) : IProcessor({std::move(header)}, {params_->getHeader()}) , params(std::move(params_)) @@ -518,6 +526,7 @@ AggregatingTransform::AggregatingTransform( , variants(*many_data->variants[current_variant]) , max_threads(std::min(many_data->variants.size(), max_threads_)) , temporary_data_merge_threads(temporary_data_merge_threads_) + , should_produce_results_in_order_of_bucket_number(should_produce_results_in_order_of_bucket_number_) , skip_merging(skip_merging_) { } @@ -707,10 +716,19 @@ void AggregatingTransform::initGenerate() for (auto & variant : prepared_data) pipes.emplace_back(std::make_shared(params, variant)); Pipe pipe = Pipe::unitePipes(std::move(pipes)); - /// Groups chunks with the same bucket_id and outputs them (as a vector of chunks) in order of bucket_id. - pipe.addTransform(std::make_shared(pipe.getHeader(), pipe.numOutputPorts(), params)); - /// Outputs one chunk from group at a time in order of bucket_id. - pipe.addTransform(std::make_shared(pipe.getHeader(), params->getHeader())); + if (should_produce_results_in_order_of_bucket_number) + { + /// Groups chunks with the same bucket_id and outputs them (as a vector of chunks) in order of bucket_id. + pipe.addTransform(std::make_shared(pipe.getHeader(), pipe.numOutputPorts(), params)); + /// Outputs one chunk from group at a time in order of bucket_id. + pipe.addTransform(std::make_shared(pipe.getHeader(), params->getHeader())); + } + else + { + /// AggregatingTransform::expandPipeline expects single output port. + /// It's not a big problem because we do resize() to max_threads after AggregatingTransform. + pipe.resize(1); + } processors = Pipe::detachProcessors(std::move(pipe)); } } diff --git a/src/Processors/Transforms/AggregatingTransform.h b/src/Processors/Transforms/AggregatingTransform.h index 2cc915d1ac7..7c8872e2953 100644 --- a/src/Processors/Transforms/AggregatingTransform.h +++ b/src/Processors/Transforms/AggregatingTransform.h @@ -150,6 +150,7 @@ public: size_t current_variant, size_t max_threads, size_t temporary_data_merge_threads, + bool should_produce_results_in_order_of_bucket_number_ = true, bool skip_merging_ = false); ~AggregatingTransform() override; @@ -182,7 +183,8 @@ private: AggregatedDataVariants & variants; size_t max_threads = 1; size_t temporary_data_merge_threads = 1; - bool skip_merging = false; /// if we aggregate partitioned data merging is not needed + bool should_produce_results_in_order_of_bucket_number = true; /// Currently makes difference only if skip_merging == true. + bool skip_merging = false; /// If we aggregate partitioned data merging is not needed. /// TODO: calculate time only for aggregation. Stopwatch watch; From 962f163dbdc59eb494bdc83c0136e9971f01a3c0 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Mon, 16 Jan 2023 22:59:14 +0000 Subject: [PATCH 068/717] add settings --- src/Core/Settings.h | 3 ++ .../QueryPlan/Optimizations/Optimizations.h | 4 ++- .../QueryPlanOptimizationSettings.cpp | 1 + .../QueryPlanOptimizationSettings.h | 2 ++ .../QueryPlan/ReadFromMergeTree.cpp | 36 ++++++++++++++++--- 5 files changed, 40 insertions(+), 6 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 274722a00d4..d2bdaa6fefa 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -549,6 +549,9 @@ class IColumn; \ M(Bool, kafka_disable_num_consumers_limit, false, "Disable limit on kafka_num_consumers that depends on the number of available CPU cores", 0) \ M(Bool, enable_software_prefetch_in_aggregation, true, "Enable use of software prefetch in aggregation", 0) \ + M(Bool, allow_aggregate_partitions_independently, false, "Enable independent aggregation of partitions on separate threads when partition key suits group by key. Benefitial when number of partitions close to number of cores and partitions have roughly the same size", 0) \ + M(Bool, force_aggregate_partitions_independently, false, "Force the use of optimization when it is applicable, but heuristics decided not to use it", 0) \ + M(UInt64, max_number_of_partitions_for_independent_aggregation, 128, "Maximal number of partitions in table to apply optimization", 0) \ /** Experimental feature for moving data between shards. */ \ \ M(Bool, allow_experimental_query_deduplication, false, "Experimental data deduplication for SELECT queries based on part UUIDs", 0) \ diff --git a/src/Processors/QueryPlan/Optimizations/Optimizations.h b/src/Processors/QueryPlan/Optimizations/Optimizations.h index e171e94a670..81f02da973c 100644 --- a/src/Processors/QueryPlan/Optimizations/Optimizations.h +++ b/src/Processors/QueryPlan/Optimizations/Optimizations.h @@ -83,7 +83,9 @@ inline const auto & getOptimizations() "reuseStorageOrderingForWindowFunctions", &QueryPlanOptimizationSettings::optimize_plan}, {tryLiftUpUnion, "liftUpUnion", &QueryPlanOptimizationSettings::optimize_plan}, - {tryAggregatePartitionsIndependently, "aggregatePartitionsIndependently", &QueryPlanOptimizationSettings::optimize_plan}, + {tryAggregatePartitionsIndependently, + "aggregatePartitionsIndependently", + &QueryPlanOptimizationSettings::aggregate_partitions_independently}, }}; return optimizations; diff --git a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp index a2a69ae1f69..c6280789370 100644 --- a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp +++ b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp @@ -15,6 +15,7 @@ QueryPlanOptimizationSettings QueryPlanOptimizationSettings::fromSettings(const settings.read_in_order = from.optimize_read_in_order && from.query_plan_read_in_order; settings.aggregation_in_order = from.optimize_aggregation_in_order && from.query_plan_aggregation_in_order; settings.remove_redundant_sorting = from.query_plan_remove_redundant_sorting; + settings.aggregate_partitions_independently = from.allow_aggregate_partitions_independently; return settings; } diff --git a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h index b894e5caf1d..0626fa81147 100644 --- a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h +++ b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h @@ -33,6 +33,8 @@ struct QueryPlanOptimizationSettings /// If removing redundant sorting is enabled, for example, ORDER BY clauses in subqueries bool remove_redundant_sorting = true; + bool aggregate_partitions_independently = false; + static QueryPlanOptimizationSettings fromSettings(const Settings & from); static QueryPlanOptimizationSettings fromContext(ContextPtr from); }; diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index cd66b8bc0e4..c1748972c7b 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -46,7 +46,8 @@ namespace template size_t countPartitions(const Container & parts, Getter get_partition_id) { - assert(!parts_with_ranges.empty()); + if (parts.empty()) + return 1; String cur_partition_id = get_partition_id(parts[0]); size_t unique_partitions = 1; for (size_t i = 1; i < parts.size(); ++i) @@ -1317,14 +1318,39 @@ bool ReadFromMergeTree::requestOutputEachPartitionThroughSeparatePort() const auto & settings = context->getSettingsRef(); const auto partitions_cnt = countPartitions(prepared_parts); - if (partitions_cnt < settings.max_threads / 20000) + if (!settings.force_aggregate_partitions_independently && (partitions_cnt == 1 || partitions_cnt < settings.max_threads / 2)) { - LOG_TRACE(log, "no cartoons"); + LOG_TRACE( + log, + "Independent aggregation by partitions won't be used because there are too few of them: {}. You can set " + "force_aggregate_partitions_independently to suppress this check", + partitions_cnt); return false; } - /* size_t min_marks_in_partition = -1; */ - /* size_t max_marks_in_partition = 0; */ + if (!settings.force_aggregate_partitions_independently + && (partitions_cnt > settings.max_number_of_partitions_for_independent_aggregation)) + { + LOG_TRACE( + log, + "Independent aggregation by partitions won't be used because there are too many of them: {}. You can increase " + "max_number_of_partitions_for_independent_aggregation (current value is {}) or set " + "force_aggregate_partitions_independently to suppress this check", + settings.max_number_of_partitions_for_independent_aggregation, + partitions_cnt); + return false; + } + + /// todo: check how big is the difference in the number of rows between parts and disable optimization if it is > 2. + /* if (!settings.force_aggregate_partitions_independently) */ + /* { */ + /* LOG_TRACE( */ + /* log, */ + /* "Independent aggregation by partitions won't be used because there are too big skew in the number of rows between partitions: " */ + /* "{} {} ." */ + /* "You can set force_aggregate_partitions_independently to suppress this check"); */ + /* return false; */ + /* } */ return output_each_partition_through_separate_port = true; } From 52fe7edbd9e6598bf35bdb05c05e810a9187d3b9 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Fri, 13 Jan 2023 22:52:02 +0000 Subject: [PATCH 069/717] better key analysis --- src/Processors/QueryPlan/AggregatingStep.cpp | 4 +- .../useDataParallelAggregation.cpp | 231 ++++++++++++++++-- .../performance/aggregation_by_partitions.xml | 4 +- .../02521_aggregation_by_partitions.reference | 6 + .../02521_aggregation_by_partitions.sql | 64 +++++ 5 files changed, 286 insertions(+), 23 deletions(-) diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp index a11451396bc..db78f236a61 100644 --- a/src/Processors/QueryPlan/AggregatingStep.cpp +++ b/src/Processors/QueryPlan/AggregatingStep.cpp @@ -465,11 +465,12 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B void AggregatingStep::describeActions(FormatSettings & settings) const { params.explain(settings.out, settings.offset); + String prefix(settings.offset, settings.indent_char); if (!sort_description_for_merging.empty()) { - String prefix(settings.offset, settings.indent_char); settings.out << prefix << "Order: " << dumpSortDescription(sort_description_for_merging) << '\n'; } + settings.out << prefix << "Skip merging: " << skip_merging << '\n'; } void AggregatingStep::describeActions(JSONBuilder::JSONMap & map) const @@ -477,6 +478,7 @@ void AggregatingStep::describeActions(JSONBuilder::JSONMap & map) const params.explain(map); if (!sort_description_for_merging.empty()) map.add("Order", dumpSortDescription(sort_description_for_merging)); + map.add("Skip merging", skip_merging); } void AggregatingStep::describePipeline(FormatSettings & settings) const diff --git a/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp b/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp index 445e9e4806f..9d520ac5c35 100644 --- a/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp +++ b/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp @@ -2,38 +2,216 @@ #include #include +#include +#include #include +#include +#include + using namespace DB; namespace { -bool isPartitionKeySuitsGroupByKey(const ReadFromMergeTree & reading, const AggregatingStep & aggregating) -{ - const auto & gb_keys = aggregating.getParams().keys; - if (aggregating.isGroupingSets() || gb_keys.size() != 1) - return false; +using NodeSet = std::unordered_set; +using NodeMap = std::unordered_map; - const auto & pkey_nodes = reading.getStorageMetadata()->getPartitionKey().expression->getActionsDAG().getNodes(); - LOG_DEBUG(&Poco::Logger::get("debug"), "{}", reading.getStorageMetadata()->getPartitionKey().expression->getActionsDAG().dumpDAG()); - if (!pkey_nodes.empty()) +struct Frame +{ + const ActionsDAG::Node * node = nullptr; + size_t next_child = 0; +}; + +auto print_node = [](const ActionsDAG::Node * node_) +{ + String children; + for (const auto & child : node_->children) + children += fmt::format("{}, ", static_cast(child)); + LOG_DEBUG( + &Poco::Logger::get("debug"), + "current node {} {} {} {}", + static_cast(node_), + node_->result_name, + node_->type, + children); +}; + +bool isInjectiveFunction(const ActionsDAG::Node * node) +{ + if (node->function_base->isInjective({})) + return true; + + size_t fixed_args = 0; + for (const auto & child : node->children) + if (child->type == ActionsDAG::ActionType::COLUMN) + ++fixed_args; + static const std::vector injective = {"plus", "minus"}; + return (fixed_args + 1 >= node->children.size()) && (std::ranges::find(injective, node->function_base->getName()) != injective.end()); +} + +void removeInjectiveColumnsFromResultsRecursively( + const ActionsDAGPtr & actions, const ActionsDAG::Node * cur_node, NodeSet & irreducible, NodeSet & visited) +{ + if (visited.contains(cur_node)) + return; + visited.insert(cur_node); + + print_node(cur_node); + + switch (cur_node->type) { - const auto & func_node = pkey_nodes.back(); - LOG_DEBUG(&Poco::Logger::get("debug"), "{} {} {}", func_node.type, func_node.is_deterministic, func_node.children.size()); - if (func_node.type == ActionsDAG::ActionType::FUNCTION && func_node.function->getName() == "modulo" - && func_node.children.size() == 2) + case ActionsDAG::ActionType::ALIAS: + assert(cur_node->children.size() == 1); + removeInjectiveColumnsFromResultsRecursively(actions, cur_node->children.at(0), irreducible, visited); + break; + case ActionsDAG::ActionType::ARRAY_JOIN: + break; + case ActionsDAG::ActionType::COLUMN: + irreducible.insert(cur_node); + break; + case ActionsDAG::ActionType::FUNCTION: + LOG_DEBUG(&Poco::Logger::get("debug"), "{} {}", __LINE__, isInjectiveFunction(cur_node)); + if (!isInjectiveFunction(cur_node)) + irreducible.insert(cur_node); + else + for (const auto & child : cur_node->children) + removeInjectiveColumnsFromResultsRecursively(actions, child, irreducible, visited); + break; + case ActionsDAG::ActionType::INPUT: + irreducible.insert(cur_node); + break; + } +} + +/// Removes injective functions recursively from result columns until it is no longer possible. +NodeSet removeInjectiveColumnsFromResultsRecursively(ActionsDAGPtr actions) +{ + NodeSet irreducible; + NodeSet visited; + + for (const auto & node : actions->getOutputs()) + removeInjectiveColumnsFromResultsRecursively(actions, node, irreducible, visited); + + LOG_DEBUG(&Poco::Logger::get("debug"), "irreducible nodes:"); + for (const auto & node : irreducible) + print_node(node); + + return irreducible; +} + +bool allOutputsCovered( + const ActionsDAGPtr & partition_actions, + const NodeSet & irreducible_nodes, + const MatchedTrees::Matches & matches, + const ActionsDAG::Node * cur_node, + NodeMap & visited) +{ + if (visited.contains(cur_node)) + return visited[cur_node]; + + auto has_match_in_group_by_actions = [&irreducible_nodes, &matches, &cur_node]() + { + if (matches.contains(cur_node)) { - const auto & arg1 = func_node.children.front(); - const auto & arg2 = func_node.children.back(); - LOG_DEBUG(&Poco::Logger::get("debug"), "{} {} {}", arg1->type, arg1->result_name, arg2->type); - if (arg1->type == ActionsDAG::ActionType::INPUT && arg1->result_name == gb_keys[0] - && arg2->type == ActionsDAG::ActionType::COLUMN && typeid_cast(arg2->column.get())) - return true; + if (const auto * node_in_gb_actions = matches.at(cur_node).node; + node_in_gb_actions && node_in_gb_actions->type == cur_node->type) + { + return irreducible_nodes.contains(node_in_gb_actions); + } + } + return false; + }; + + bool res = has_match_in_group_by_actions(); + if (!res) + { + switch (cur_node->type) + { + case ActionsDAG::ActionType::ALIAS: + assert(cur_node->children.size() == 1); + res = allOutputsCovered(partition_actions, irreducible_nodes, matches, cur_node->children.at(0), visited); + break; + case ActionsDAG::ActionType::ARRAY_JOIN: + break; + case ActionsDAG::ActionType::COLUMN: + /// Constants doesn't matter, so let's always consider them matched. + res = true; + break; + case ActionsDAG::ActionType::FUNCTION: + res = true; + for (const auto & child : cur_node->children) + res &= allOutputsCovered(partition_actions, irreducible_nodes, matches, child, visited); + break; + case ActionsDAG::ActionType::INPUT: + break; } } + print_node(cur_node); + LOG_DEBUG(&Poco::Logger::get("debug"), "res={}", res); + visited[cur_node] = res; + return res; +} - return false; +bool allOutputsCovered(ActionsDAGPtr partition_actions, const NodeSet & irreducible_nodes, const MatchedTrees::Matches & matches) +{ + NodeMap visited; + + bool res = true; + for (const auto & node : partition_actions->getOutputs()) + if (node->type != ActionsDAG::ActionType::INPUT) + res &= allOutputsCovered(partition_actions, irreducible_nodes, matches, node, visited); + return res; +} + +bool isPartitionKeySuitsGroupByKey(const ReadFromMergeTree & reading, ActionsDAGPtr group_by_actions, const AggregatingStep & aggregating) +{ + /// 0. Partition key columns should be a subset of group by key columns. + /// 1. Optimization is applicable if partition by expression is a deterministic function of col1, ..., coln and group by keys are injective functions of some of col1, ..., coln. + + if (aggregating.isGroupingSets() || group_by_actions->hasArrayJoin() || group_by_actions->hasStatefulFunctions()) + return false; + + /// Check that PK columns is a subset of GBK columns. + const auto partition_actions = reading.getStorageMetadata()->getPartitionKey().expression->getActionsDAG().clone(); + + /// We are interested only in calculations required to obtain group by keys. + group_by_actions->removeUnusedActions(aggregating.getParams().keys); + const auto & gb_keys = group_by_actions->getRequiredColumnsNames(); + + LOG_DEBUG(&Poco::Logger::get("debug"), "group by req cols: {}", fmt::join(gb_keys, ", ")); + LOG_DEBUG(&Poco::Logger::get("debug"), "partition by cols: {}", fmt::join(partition_actions->getRequiredColumnsNames(), ", ")); + + for (const auto & col : partition_actions->getRequiredColumnsNames()) + if (std::ranges::find(gb_keys, col) == gb_keys.end()) + return false; + + /* /// PK is always a deterministic expression without constants. No need to check. */ + + /* /// We will work only with subexpression that depends on partition key columns. */ + LOG_DEBUG(&Poco::Logger::get("debug"), "group by actions before:\n{}", group_by_actions->dumpDAG()); + LOG_DEBUG(&Poco::Logger::get("debug"), "partition by actions before:\n{}", partition_actions->dumpDAG()); + + LOG_DEBUG(&Poco::Logger::get("debug"), "group by actions after:\n{}", group_by_actions->dumpDAG()); + LOG_DEBUG(&Poco::Logger::get("debug"), "partition by actions after:\n{}", partition_actions->dumpDAG()); + + /// For cases like `partition by col + group by col+1` or `partition by hash(col) + group by hash(col)` + const auto irreducibe_nodes = removeInjectiveColumnsFromResultsRecursively(group_by_actions); + + const auto matches = matchTrees(*group_by_actions, *partition_actions); + LOG_DEBUG(&Poco::Logger::get("debug"), "matches:"); + for (const auto & match : matches) + { + if (match.first) + print_node(match.first); + if (match.second.node) + print_node(match.second.node); + LOG_DEBUG(&Poco::Logger::get("debug"), "----------------"); + } + + const bool res = allOutputsCovered(partition_actions, irreducibe_nodes, matches); + LOG_DEBUG(&Poco::Logger::get("debug"), "result={}", res); + return res; } } @@ -50,15 +228,26 @@ size_t tryAggregatePartitionsIndependently(QueryPlan::Node * node, QueryPlan::No return 0; const auto * expression_node = node->children.front(); - if (expression_node->children.size() != 1 || !typeid_cast(expression_node->step.get())) + const auto * expression_step = typeid_cast(expression_node->step.get()); + if (expression_node->children.size() != 1 || !expression_step) return 0; auto * reading_step = expression_node->children.front()->step.get(); + + if (const auto * filter = typeid_cast(reading_step)) + { + const auto * filter_node = expression_node->children.front(); + if (filter_node->children.size() != 1 || !filter_node->children.front()->step) + return 0; + reading_step = filter_node->children.front()->step.get(); + } + auto * reading = typeid_cast(reading_step); if (!reading) return 0; - if (!reading->willOutputEachPartitionThroughSeparatePort() && isPartitionKeySuitsGroupByKey(*reading, *aggregating_step)) + if (!reading->willOutputEachPartitionThroughSeparatePort() + && isPartitionKeySuitsGroupByKey(*reading, expression_step->getExpression()->clone(), *aggregating_step)) { if (reading->requestOutputEachPartitionThroughSeparatePort()) aggregating_step->skipMerging(); diff --git a/tests/performance/aggregation_by_partitions.xml b/tests/performance/aggregation_by_partitions.xml index e24f589508e..7ef4c7742c1 100644 --- a/tests/performance/aggregation_by_partitions.xml +++ b/tests/performance/aggregation_by_partitions.xml @@ -1,6 +1,8 @@ - + 1 + 1 + 4096 0 4096 diff --git a/tests/queries/0_stateless/02521_aggregation_by_partitions.reference b/tests/queries/0_stateless/02521_aggregation_by_partitions.reference index 3e396bafa38..85870f54483 100644 --- a/tests/queries/0_stateless/02521_aggregation_by_partitions.reference +++ b/tests/queries/0_stateless/02521_aggregation_by_partitions.reference @@ -195,3 +195,9 @@ ExpressionTransform × 16 ExpressionTransform × 2 MergeTreeInOrder × 2 0 → 1 1000000 +Skip merging: 1 +Skip merging: 1 +Skip merging: 0 +Skip merging: 1 +Skip merging: 0 +Skip merging: 1 diff --git a/tests/queries/0_stateless/02521_aggregation_by_partitions.sql b/tests/queries/0_stateless/02521_aggregation_by_partitions.sql index 870116d4a23..c788c6e238a 100644 --- a/tests/queries/0_stateless/02521_aggregation_by_partitions.sql +++ b/tests/queries/0_stateless/02521_aggregation_by_partitions.sql @@ -1,4 +1,7 @@ set max_threads = 16; +set allow_aggregate_partitions_independently = 1; +set force_aggregate_partitions_independently = 1; +set allow_experimental_projection_optimization = 0; create table t1(a UInt32) engine=MergeTree order by tuple() partition by a % 4; @@ -86,3 +89,64 @@ explain pipeline select a from t6 group by a settings read_in_order_two_level_me select count() from (select throwIf(count() != 2) from t6 group by a); drop table t6; + +create table t7(a UInt32) engine=MergeTree order by a partition by intDiv(a, 2); + +insert into t7 select number from numbers_mt(100); + +select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from ( + explain actions=1 select intDiv(a, 2) as a1 from t7 group by a1 +) where explain like '%Skip merging: %'; + +drop table t7; + +create table t8(a UInt32) engine=MergeTree order by a partition by intDiv(a, 2) * 2 + 1; + +insert into t8 select number from numbers_mt(100); + +select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from ( + explain actions=1 select intDiv(a, 2) + 1 as a1 from t8 group by a1 +) where explain like '%Skip merging: %'; + +drop table t8; + +create table t9(a UInt32) engine=MergeTree order by a partition by intDiv(a, 2); + +insert into t9 select number from numbers_mt(100); + +select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from ( + explain actions=1 select intDiv(a, 3) as a1 from t9 group by a1 +) where explain like '%Skip merging: %'; + +drop table t9; + +create table t10(a UInt32, b UInt32) engine=MergeTree order by a partition by (intDiv(a, 2), intDiv(b, 3)); + +insert into t10 select number, number from numbers_mt(100); + +select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from ( + explain actions=1 select intDiv(a, 2) + 1 as a1, intDiv(b, 3) as b1 from t10 group by a1, b1, pi() +) where explain like '%Skip merging: %'; + +drop table t10; + +-- multiplication by 2 is not injective, so optimization is not applicable +create table t11(a UInt32, b UInt32) engine=MergeTree order by a partition by (intDiv(a, 2), intDiv(b, 3)); + +insert into t11 select number, number from numbers_mt(100); + +select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from ( + explain actions=1 select intDiv(a, 2) + 1 as a1, intDiv(b, 3) * 2 as b1 from t11 group by a1, b1, pi() +) where explain like '%Skip merging: %'; + +drop table t11; + +create table t12(a UInt32, b UInt32) engine=MergeTree order by a partition by a % 16; + +insert into t12 select number, number from numbers_mt(100); + +select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from ( + explain actions=1 select a, b from t12 group by a, b, pi() +) where explain like '%Skip merging: %'; + +drop table t12; From 4d7a8bc6094c2956d7da5b8ecca0e0c015c6f9be Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Tue, 17 Jan 2023 14:50:38 +0000 Subject: [PATCH 070/717] more tests --- .../02521_aggregation_by_partitions.reference | 2 ++ .../02521_aggregation_by_partitions.sql | 20 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/tests/queries/0_stateless/02521_aggregation_by_partitions.reference b/tests/queries/0_stateless/02521_aggregation_by_partitions.reference index 85870f54483..121ebd52e5a 100644 --- a/tests/queries/0_stateless/02521_aggregation_by_partitions.reference +++ b/tests/queries/0_stateless/02521_aggregation_by_partitions.reference @@ -201,3 +201,5 @@ Skip merging: 0 Skip merging: 1 Skip merging: 0 Skip merging: 1 +Skip merging: 0 +Skip merging: 1 diff --git a/tests/queries/0_stateless/02521_aggregation_by_partitions.sql b/tests/queries/0_stateless/02521_aggregation_by_partitions.sql index c788c6e238a..919ac374024 100644 --- a/tests/queries/0_stateless/02521_aggregation_by_partitions.sql +++ b/tests/queries/0_stateless/02521_aggregation_by_partitions.sql @@ -150,3 +150,23 @@ select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from ( ) where explain like '%Skip merging: %'; drop table t12; + +create table t13(a UInt32, b UInt32) engine=MergeTree order by a partition by (intDiv(a, 2), intDiv(b, 3)); + +insert into t13 select number, number from numbers_mt(100); + +select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from ( + explain actions=1 select s from t13 group by intDiv(a, 2) + intDiv(b, 3) as s, pi() +) where explain like '%Skip merging: %'; + +drop table t13; + +create table t14(a UInt32, b UInt32) engine=MergeTree order by a partition by intDiv(a, 2) + intDiv(b, 3); + +insert into t14 select number, number from numbers_mt(100); + +select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from ( + explain actions=1 select intDiv(a, 2) as a1, intDiv(b, 3) as b1 from t14 group by a1, b1, pi() +) where explain like '%Skip merging: %'; + +drop table t14; From 3ce725023c7c8f48ad5d3fbc3947a23008512534 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Tue, 17 Jan 2023 15:33:21 +0000 Subject: [PATCH 071/717] fix style & build --- src/Core/Settings.h | 2 +- src/Processors/QueryPlan/AggregatingStep.cpp | 2 +- .../useDataParallelAggregation.cpp | 1 + src/Processors/QueryPlan/ReadFromMergeTree.cpp | 18 +++++++++--------- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index d2bdaa6fefa..c3f3c527cf4 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -549,7 +549,7 @@ class IColumn; \ M(Bool, kafka_disable_num_consumers_limit, false, "Disable limit on kafka_num_consumers that depends on the number of available CPU cores", 0) \ M(Bool, enable_software_prefetch_in_aggregation, true, "Enable use of software prefetch in aggregation", 0) \ - M(Bool, allow_aggregate_partitions_independently, false, "Enable independent aggregation of partitions on separate threads when partition key suits group by key. Benefitial when number of partitions close to number of cores and partitions have roughly the same size", 0) \ + M(Bool, allow_aggregate_partitions_independently, false, "Enable independent aggregation of partitions on separate threads when partition key suits group by key. Beneficial when number of partitions close to number of cores and partitions have roughly the same size", 0) \ M(Bool, force_aggregate_partitions_independently, false, "Force the use of optimization when it is applicable, but heuristics decided not to use it", 0) \ M(UInt64, max_number_of_partitions_for_independent_aggregation, 128, "Maximal number of partitions in table to apply optimization", 0) \ /** Experimental feature for moving data between shards. */ \ diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp index db78f236a61..c1028d40a32 100644 --- a/src/Processors/QueryPlan/AggregatingStep.cpp +++ b/src/Processors/QueryPlan/AggregatingStep.cpp @@ -427,7 +427,7 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B if (pipeline.getNumStreams() > 1) { /// Add resize transform to uniformly distribute data between aggregating streams. - /// But not if we execute aggregation over partitoned data in which case data streams shouldn't be mixed. + /// But not if we execute aggregation over partitioned data in which case data streams shouldn't be mixed. if (!storage_has_evenly_distributed_read && !skip_merging) pipeline.resize(pipeline.getNumStreams(), true, true); diff --git a/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp b/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp index 9d520ac5c35..a626fa47012 100644 --- a/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp +++ b/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index c1748972c7b..f8b41104130 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -1342,15 +1342,15 @@ bool ReadFromMergeTree::requestOutputEachPartitionThroughSeparatePort() } /// todo: check how big is the difference in the number of rows between parts and disable optimization if it is > 2. - /* if (!settings.force_aggregate_partitions_independently) */ - /* { */ - /* LOG_TRACE( */ - /* log, */ - /* "Independent aggregation by partitions won't be used because there are too big skew in the number of rows between partitions: " */ - /* "{} {} ." */ - /* "You can set force_aggregate_partitions_independently to suppress this check"); */ - /* return false; */ - /* } */ + /*if (!settings.force_aggregate_partitions_independently) + { + LOG_TRACE( + log, + "Independent aggregation by partitions won't be used because there are too big skew in the number of rows between partitions: " + "{} {} ." + "You can set force_aggregate_partitions_independently to suppress this check"); + return false; + }*/ return output_each_partition_through_separate_port = true; } From 47f57b7a5d6a57fd8d8ee67d46c671fdfbb9e5e8 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Wed, 18 Jan 2023 14:22:32 +0000 Subject: [PATCH 072/717] few heuristics to disable optimisation --- .../QueryPlan/ReadFromMergeTree.cpp | 40 ++++++++++++++----- .../02521_aggregation_by_partitions.reference | 3 ++ .../02521_aggregation_by_partitions.sql | 36 +++++++++++++++++ 3 files changed, 68 insertions(+), 11 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index f8b41104130..107d2d5517e 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -1,10 +1,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -1336,21 +1338,37 @@ bool ReadFromMergeTree::requestOutputEachPartitionThroughSeparatePort() "Independent aggregation by partitions won't be used because there are too many of them: {}. You can increase " "max_number_of_partitions_for_independent_aggregation (current value is {}) or set " "force_aggregate_partitions_independently to suppress this check", - settings.max_number_of_partitions_for_independent_aggregation, - partitions_cnt); + partitions_cnt, + settings.max_number_of_partitions_for_independent_aggregation); return false; } - /// todo: check how big is the difference in the number of rows between parts and disable optimization if it is > 2. - /*if (!settings.force_aggregate_partitions_independently) + if (!settings.force_aggregate_partitions_independently) { - LOG_TRACE( - log, - "Independent aggregation by partitions won't be used because there are too big skew in the number of rows between partitions: " - "{} {} ." - "You can set force_aggregate_partitions_independently to suppress this check"); - return false; - }*/ + std::unordered_map partition_rows; + for (const auto & part : prepared_parts) + partition_rows[part->info.partition_id] += part->rows_count; + size_t sum_rows = 0; + size_t max_rows = 0; + for (const auto & [_, rows] : partition_rows) + { + sum_rows += rows; + max_rows = std::max(max_rows, rows); + } + + /// Merging shouldn't take more time than preaggregation in normal cases. And exec time is proportional to the amount of data. + /// We assume that exec time of independent aggr is proportional to the maximum of sizes and + /// exec time of ordinary aggr is proportional to sum of sizes divided by number of threads and multiplied by two (preaggregation + merging). + const size_t avg_rows_in_partition = sum_rows / settings.max_threads; + if (max_rows > avg_rows_in_partition * 2) + { + LOG_TRACE( + log, + "Independent aggregation by partitions won't be used because there are too big skew in the number of rows between " + "partitions. You can set force_aggregate_partitions_independently to suppress this check"); + return false; + } + } return output_each_partition_through_separate_port = true; } diff --git a/tests/queries/0_stateless/02521_aggregation_by_partitions.reference b/tests/queries/0_stateless/02521_aggregation_by_partitions.reference index 121ebd52e5a..4a0967fb8ba 100644 --- a/tests/queries/0_stateless/02521_aggregation_by_partitions.reference +++ b/tests/queries/0_stateless/02521_aggregation_by_partitions.reference @@ -203,3 +203,6 @@ Skip merging: 0 Skip merging: 1 Skip merging: 0 Skip merging: 1 +Skip merging: 0 +Skip merging: 0 +Skip merging: 0 diff --git a/tests/queries/0_stateless/02521_aggregation_by_partitions.sql b/tests/queries/0_stateless/02521_aggregation_by_partitions.sql index 919ac374024..1c139e94591 100644 --- a/tests/queries/0_stateless/02521_aggregation_by_partitions.sql +++ b/tests/queries/0_stateless/02521_aggregation_by_partitions.sql @@ -170,3 +170,39 @@ select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from ( ) where explain like '%Skip merging: %'; drop table t14; + +-- to few partitions -- +create table t15(a UInt32, b UInt32) engine=MergeTree order by a partition by a < 90; + +insert into t15 select number, number from numbers_mt(100); + +select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from ( + explain actions=1 select a from t15 group by a +) where explain like '%Skip merging: %' +settings force_aggregate_partitions_independently = 0; + +drop table t15; + +-- to many partitions -- +create table t16(a UInt32, b UInt32) engine=MergeTree order by a partition by a % 16; + +insert into t16 select number, number from numbers_mt(100); + +select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from ( + explain actions=1 select a from t16 group by a +) where explain like '%Skip merging: %' +settings force_aggregate_partitions_independently = 0, max_number_of_partitions_for_independent_aggregation = 4; + +drop table t16; + +-- to big skew -- +create table t17(a UInt32, b UInt32) engine=MergeTree order by a partition by a < 90; + +insert into t17 select number, number from numbers_mt(100); + +select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from ( + explain actions=1 select a from t17 group by a +) where explain like '%Skip merging: %' +settings force_aggregate_partitions_independently = 0, max_threads = 4; + +drop table t17; From d8c191f53b78e467956347008153be458f98213a Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Wed, 18 Jan 2023 18:36:41 +0000 Subject: [PATCH 073/717] add docs --- .../custom-partitioning-key.md | 47 ++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/docs/en/engines/table-engines/mergetree-family/custom-partitioning-key.md b/docs/en/engines/table-engines/mergetree-family/custom-partitioning-key.md index 262571572e1..76996cae939 100644 --- a/docs/en/engines/table-engines/mergetree-family/custom-partitioning-key.md +++ b/docs/en/engines/table-engines/mergetree-family/custom-partitioning-key.md @@ -7,7 +7,7 @@ sidebar_label: Custom Partitioning Key # Custom Partitioning Key :::warning -In most cases you do not need a partition key, and in most other cases you do not need a partition key more granular than by months. Partitioning does not speed up queries (in contrast to the ORDER BY expression). +In most cases you do not need a partition key, and in most other cases you do not need a partition key more granular than by months. You should never use too granular of partitioning. Don't partition your data by client identifiers or names. Instead, make a client identifier or name the first column in the ORDER BY expression. ::: @@ -133,3 +133,48 @@ The `detached` directory contains parts that were detached from the table using Note that on the operating server, you cannot manually change the set of parts or their data on the file system, since the server will not know about it. For non-replicated tables, you can do this when the server is stopped, but it isn’t recommended. For replicated tables, the set of parts cannot be changed in any case. ClickHouse allows you to perform operations with the partitions: delete them, copy from one table to another, or create a backup. See the list of all operations in the section [Manipulations With Partitions and Parts](../../../sql-reference/statements/alter/partition.md#alter_manipulations-with-partitions). + +## Group By optimisation using partition key + +For some combinations of table's partition key and query's group by key it might be possible to execute aggregation for each partition independently. +Then we'll not have to merge partially aggregated data from all execution threads at the end, +because we provided with the guarantee that each group by key value cannot appear in working sets of two different threads. + +The typical example is: + +``` sql +CREATE TABLE session_log +( + UserID UInt64, + SessionID UUID +) +ENGINE = MergeTree +PARTITION BY sipHash64(UserID) % 16 +ORDER BY tuple(); + +SELECT + UserID, + COUNT() +FROM session_log +GROUP BY UserID; +``` + +:::warning +Performance of such a query heavily depends on the table layout. Because of that it is not enabled by default. +::: + +The key factors are: + +- number of partitions involved in the query should be sufficiently large (at least `max_threads / 2`), otherwise query will underutilize the machine +- number of partitions involved in the query should be not too big, so batch processing won't degenerate into row-by-row processing +- partitions should be comparable in size, so all threads will do roughly the same amount of work + +:::info +It's recommended to apply some hash function to columns in `partition by` clause in order to distribute data evenly between partitions. +::: + +Relevant settings are: + +- `allow_aggregate_partitions_independently` - controls if the use of optimisation is enabled +- `force_aggregate_partitions_independently` - forces its use when it's applicable from the correctness standpoint, but getting disabled by internal logic that estimates its expediency +- `max_number_of_partitions_for_independent_aggregation` - hard limit on the maximal number of partitions table could have From 3e2ed6bbb1048c11d2f7f741d7e684672765c4ce Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Wed, 18 Jan 2023 18:37:24 +0000 Subject: [PATCH 074/717] mark as long --- tests/queries/0_stateless/02521_aggregation_by_partitions.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/02521_aggregation_by_partitions.sql b/tests/queries/0_stateless/02521_aggregation_by_partitions.sql index 1c139e94591..35d75ff513b 100644 --- a/tests/queries/0_stateless/02521_aggregation_by_partitions.sql +++ b/tests/queries/0_stateless/02521_aggregation_by_partitions.sql @@ -1,3 +1,5 @@ +-- Tags: long + set max_threads = 16; set allow_aggregate_partitions_independently = 1; set force_aggregate_partitions_independently = 1; From 0d393c7a7f0c9e82d915fbfc06aed025187017f6 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Wed, 18 Jan 2023 23:11:12 +0000 Subject: [PATCH 075/717] cosmetics --- .../useDataParallelAggregation.cpp | 163 +++++++----------- .../Transforms/AggregatingTransform.cpp | 8 +- 2 files changed, 66 insertions(+), 105 deletions(-) diff --git a/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp b/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp index a626fa47012..2a5619cf27f 100644 --- a/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp +++ b/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp @@ -24,20 +24,6 @@ struct Frame size_t next_child = 0; }; -auto print_node = [](const ActionsDAG::Node * node_) -{ - String children; - for (const auto & child : node_->children) - children += fmt::format("{}, ", static_cast(child)); - LOG_DEBUG( - &Poco::Logger::get("debug"), - "current node {} {} {} {}", - static_cast(node_), - node_->result_name, - node_->type, - children); -}; - bool isInjectiveFunction(const ActionsDAG::Node * node) { if (node->function_base->isInjective({})) @@ -47,76 +33,70 @@ bool isInjectiveFunction(const ActionsDAG::Node * node) for (const auto & child : node->children) if (child->type == ActionsDAG::ActionType::COLUMN) ++fixed_args; - static const std::vector injective = {"plus", "minus"}; + static const std::vector injective = {"plus", "minus", "negate", "tuple"}; return (fixed_args + 1 >= node->children.size()) && (std::ranges::find(injective, node->function_base->getName()) != injective.end()); } -void removeInjectiveColumnsFromResultsRecursively( - const ActionsDAGPtr & actions, const ActionsDAG::Node * cur_node, NodeSet & irreducible, NodeSet & visited) +void removeInjectiveFunctionsFromResultsRecursively(const ActionsDAG::Node * node, NodeSet & irreducible, NodeSet & visited) { - if (visited.contains(cur_node)) + if (visited.contains(node)) return; - visited.insert(cur_node); + visited.insert(node); - print_node(cur_node); - - switch (cur_node->type) + switch (node->type) { case ActionsDAG::ActionType::ALIAS: - assert(cur_node->children.size() == 1); - removeInjectiveColumnsFromResultsRecursively(actions, cur_node->children.at(0), irreducible, visited); + assert(node->children.size() == 1); + removeInjectiveFunctionsFromResultsRecursively(node->children.at(0), irreducible, visited); break; case ActionsDAG::ActionType::ARRAY_JOIN: - break; + UNREACHABLE(); case ActionsDAG::ActionType::COLUMN: - irreducible.insert(cur_node); + irreducible.insert(node); break; case ActionsDAG::ActionType::FUNCTION: - LOG_DEBUG(&Poco::Logger::get("debug"), "{} {}", __LINE__, isInjectiveFunction(cur_node)); - if (!isInjectiveFunction(cur_node)) - irreducible.insert(cur_node); + if (!isInjectiveFunction(node)) + { + irreducible.insert(node); + } else - for (const auto & child : cur_node->children) - removeInjectiveColumnsFromResultsRecursively(actions, child, irreducible, visited); + { + for (const auto & child : node->children) + removeInjectiveFunctionsFromResultsRecursively(child, irreducible, visited); + } break; case ActionsDAG::ActionType::INPUT: - irreducible.insert(cur_node); + irreducible.insert(node); break; } } -/// Removes injective functions recursively from result columns until it is no longer possible. -NodeSet removeInjectiveColumnsFromResultsRecursively(ActionsDAGPtr actions) +/// Our objective is to replace injective function nodes in `actions` results with its children +/// until only the irreducible subset of nodes remains. Against these set of nodes we will match partition key expression +/// to determine if it maps all rows with the same value of group by key to the same partition. +NodeSet removeInjectiveFunctionsFromResultsRecursively(const ActionsDAGPtr & actions) { NodeSet irreducible; NodeSet visited; - for (const auto & node : actions->getOutputs()) - removeInjectiveColumnsFromResultsRecursively(actions, node, irreducible, visited); - - LOG_DEBUG(&Poco::Logger::get("debug"), "irreducible nodes:"); - for (const auto & node : irreducible) - print_node(node); - + removeInjectiveFunctionsFromResultsRecursively(node, irreducible, visited); return irreducible; } -bool allOutputsCovered( - const ActionsDAGPtr & partition_actions, - const NodeSet & irreducible_nodes, - const MatchedTrees::Matches & matches, - const ActionsDAG::Node * cur_node, - NodeMap & visited) +bool allOutputsDependsOnlyOnAllowedNodes( + const NodeSet & irreducible_nodes, const MatchedTrees::Matches & matches, const ActionsDAG::Node * node, NodeMap & visited) { - if (visited.contains(cur_node)) - return visited[cur_node]; + if (visited.contains(node)) + return visited[node]; - auto has_match_in_group_by_actions = [&irreducible_nodes, &matches, &cur_node]() + auto has_match_in_group_by_actions = [&irreducible_nodes, &matches, &node]() { - if (matches.contains(cur_node)) + /// `matches` maps partition key nodes into nodes in group by actions + if (matches.contains(node)) { - if (const auto * node_in_gb_actions = matches.at(cur_node).node; - node_in_gb_actions && node_in_gb_actions->type == cur_node->type) + if (const auto * node_in_gb_actions = matches.at(node).node; + /// Double-check is needed because function can be mapped into its arg (see matchTrees) + node_in_gb_actions && node_in_gb_actions->result_name == node->result_name) { return irreducible_nodes.contains(node_in_gb_actions); } @@ -127,92 +107,73 @@ bool allOutputsCovered( bool res = has_match_in_group_by_actions(); if (!res) { - switch (cur_node->type) + switch (node->type) { case ActionsDAG::ActionType::ALIAS: - assert(cur_node->children.size() == 1); - res = allOutputsCovered(partition_actions, irreducible_nodes, matches, cur_node->children.at(0), visited); + assert(node->children.size() == 1); + res = allOutputsDependsOnlyOnAllowedNodes(irreducible_nodes, matches, node->children.at(0), visited); break; case ActionsDAG::ActionType::ARRAY_JOIN: - break; + UNREACHABLE(); case ActionsDAG::ActionType::COLUMN: /// Constants doesn't matter, so let's always consider them matched. res = true; break; case ActionsDAG::ActionType::FUNCTION: res = true; - for (const auto & child : cur_node->children) - res &= allOutputsCovered(partition_actions, irreducible_nodes, matches, child, visited); + for (const auto & child : node->children) + res &= allOutputsDependsOnlyOnAllowedNodes(irreducible_nodes, matches, child, visited); break; case ActionsDAG::ActionType::INPUT: break; } } - print_node(cur_node); - LOG_DEBUG(&Poco::Logger::get("debug"), "res={}", res); - visited[cur_node] = res; + visited[node] = res; return res; } -bool allOutputsCovered(ActionsDAGPtr partition_actions, const NodeSet & irreducible_nodes, const MatchedTrees::Matches & matches) +/// Here we check that partition key expression is a deterministic function of the reduced set of group by key nodes. +/// No need to explicitly check that each function is deterministic, because it is a guaranteed property of partition key expression (checked on table creation). +/// So it is left only to check that each output node depends only on the allowed set of nodes (`irreducible_nodes`). +bool allOutputsDependsOnlyOnAllowedNodes( + const ActionsDAGPtr & partition_actions, const NodeSet & irreducible_nodes, const MatchedTrees::Matches & matches) { NodeMap visited; - bool res = true; for (const auto & node : partition_actions->getOutputs()) if (node->type != ActionsDAG::ActionType::INPUT) - res &= allOutputsCovered(partition_actions, irreducible_nodes, matches, node, visited); + res &= allOutputsDependsOnlyOnAllowedNodes(irreducible_nodes, matches, node, visited); return res; } +/// 0. Partition key columns should be a subset of group by key columns. +/// 1. Optimization is applicable if partition by expression is a deterministic function of col1, ..., coln and group by key is injective functions of these col1, ..., coln. +/// 2. To find col1, ..., coln we apply removeInjectiveFunctionsFromResultsRecursively to group by key actions. +/// 3. We match partition key actions with group by key actions to find col1', ..., coln' in partition key actions. +/// 4. We check that partition key is indeed a deterministic function of col1', ..., coln'. bool isPartitionKeySuitsGroupByKey(const ReadFromMergeTree & reading, ActionsDAGPtr group_by_actions, const AggregatingStep & aggregating) { - /// 0. Partition key columns should be a subset of group by key columns. - /// 1. Optimization is applicable if partition by expression is a deterministic function of col1, ..., coln and group by keys are injective functions of some of col1, ..., coln. - - if (aggregating.isGroupingSets() || group_by_actions->hasArrayJoin() || group_by_actions->hasStatefulFunctions()) + if (aggregating.isGroupingSets()) return false; - /// Check that PK columns is a subset of GBK columns. - const auto partition_actions = reading.getStorageMetadata()->getPartitionKey().expression->getActionsDAG().clone(); - /// We are interested only in calculations required to obtain group by keys. group_by_actions->removeUnusedActions(aggregating.getParams().keys); + if (group_by_actions->hasArrayJoin() || group_by_actions->hasStatefulFunctions() /* || group_by_actions->assertDeterministic() */) + return false; const auto & gb_keys = group_by_actions->getRequiredColumnsNames(); - LOG_DEBUG(&Poco::Logger::get("debug"), "group by req cols: {}", fmt::join(gb_keys, ", ")); - LOG_DEBUG(&Poco::Logger::get("debug"), "partition by cols: {}", fmt::join(partition_actions->getRequiredColumnsNames(), ", ")); + const auto partition_actions = reading.getStorageMetadata()->getPartitionKey().expression->getActionsDAG().clone(); + /// Check that PK columns is a subset of GBK columns. for (const auto & col : partition_actions->getRequiredColumnsNames()) if (std::ranges::find(gb_keys, col) == gb_keys.end()) return false; - /* /// PK is always a deterministic expression without constants. No need to check. */ - - /* /// We will work only with subexpression that depends on partition key columns. */ - LOG_DEBUG(&Poco::Logger::get("debug"), "group by actions before:\n{}", group_by_actions->dumpDAG()); - LOG_DEBUG(&Poco::Logger::get("debug"), "partition by actions before:\n{}", partition_actions->dumpDAG()); - - LOG_DEBUG(&Poco::Logger::get("debug"), "group by actions after:\n{}", group_by_actions->dumpDAG()); - LOG_DEBUG(&Poco::Logger::get("debug"), "partition by actions after:\n{}", partition_actions->dumpDAG()); - - /// For cases like `partition by col + group by col+1` or `partition by hash(col) + group by hash(col)` - const auto irreducibe_nodes = removeInjectiveColumnsFromResultsRecursively(group_by_actions); + const auto irreducibe_nodes = removeInjectiveFunctionsFromResultsRecursively(group_by_actions); const auto matches = matchTrees(*group_by_actions, *partition_actions); - LOG_DEBUG(&Poco::Logger::get("debug"), "matches:"); - for (const auto & match : matches) - { - if (match.first) - print_node(match.first); - if (match.second.node) - print_node(match.second.node); - LOG_DEBUG(&Poco::Logger::get("debug"), "----------------"); - } - const bool res = allOutputsCovered(partition_actions, irreducibe_nodes, matches); - LOG_DEBUG(&Poco::Logger::get("debug"), "result={}", res); - return res; + return allOutputsDependsOnlyOnAllowedNodes(partition_actions, irreducibe_nodes, matches); } } @@ -233,17 +194,17 @@ size_t tryAggregatePartitionsIndependently(QueryPlan::Node * node, QueryPlan::No if (expression_node->children.size() != 1 || !expression_step) return 0; - auto * reading_step = expression_node->children.front()->step.get(); + auto * maybe_reading_step = expression_node->children.front()->step.get(); - if (const auto * filter = typeid_cast(reading_step)) + if (const auto * filter = typeid_cast(maybe_reading_step)) { const auto * filter_node = expression_node->children.front(); if (filter_node->children.size() != 1 || !filter_node->children.front()->step) return 0; - reading_step = filter_node->children.front()->step.get(); + maybe_reading_step = filter_node->children.front()->step.get(); } - auto * reading = typeid_cast(reading_step); + auto * reading = typeid_cast(maybe_reading_step); if (!reading) return 0; diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index 4633750eba3..ea4d1d062f4 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -136,6 +136,7 @@ private: Arena * arena; }; +/// Asks Aggregator to convert accumulated aggregation state into blocks (without merging) and pushes them to later steps. class ConvertingAggregatedToChunksSource : public ISource { public: @@ -171,7 +172,7 @@ private: BlocksList blocks; }; -/// Reads chunks from GroupingAggregatedTransform and outputs them. +/// Reads chunks from GroupingAggregatedTransform (stored in ChunksToMerge structure) and outputs them. class FlattenChunksToMergeTransform : public IProcessor { public: @@ -187,7 +188,7 @@ private: { } - void process(Chunk chunk) + void process(Chunk && chunk) { if (!chunk.hasChunkInfo()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected chunk with chunk info in {}", getName()); @@ -245,7 +246,6 @@ private: Chunk chunk = input.pull(true /* set_not_needed */); process(std::move(chunk)); - /// Now transform. return Status::Ready; } @@ -711,9 +711,9 @@ void AggregatingTransform::initGenerate() else { auto prepared_data = params->aggregator.prepareVariantsToMerge(many_data->variants); - /// Converts hash tables to blocks with data (finalized or not). Pipes pipes; for (auto & variant : prepared_data) + /// Converts hash tables to blocks with data (finalized or not). pipes.emplace_back(std::make_shared(params, variant)); Pipe pipe = Pipe::unitePipes(std::move(pipes)); if (should_produce_results_in_order_of_bucket_number) From ac778081339626761be142021af0ba6bab47e86d Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Sun, 22 Jan 2023 19:36:11 +0000 Subject: [PATCH 076/717] fix perf test --- .../performance/aggregation_by_partitions.xml | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/tests/performance/aggregation_by_partitions.xml b/tests/performance/aggregation_by_partitions.xml index 7ef4c7742c1..c8f88530db1 100644 --- a/tests/performance/aggregation_by_partitions.xml +++ b/tests/performance/aggregation_by_partitions.xml @@ -2,38 +2,35 @@ 1 1 - 4096 0 - 4096 + 256 + 256 + 16 size - 10000 + 100000 1000000 10000000 - 100000000 + partitions - 2 - 4 - 8 + + + 16 32 64 - 128 - 256 - - - - + + From a18343773f6dc62619bbe98b6ba5847b816b411c Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Fri, 20 Jan 2023 23:05:49 +0000 Subject: [PATCH 077/717] improve perf --- src/Interpreters/Aggregator.cpp | 15 +++++ src/Interpreters/Aggregator.h | 2 + .../QueryPlan/ReadFromMergeTree.cpp | 14 ++--- .../Transforms/AggregatingTransform.cpp | 59 ++++++++++++------- 4 files changed, 61 insertions(+), 29 deletions(-) diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index da4442f99d3..cb758826357 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -1695,6 +1695,21 @@ Block Aggregator::mergeAndConvertOneBucketToBlock( return block; } +Block Aggregator::convertOneBucketToBlock(AggregatedDataVariants & variants, Arena * arena, bool final, Int32 bucket) const +{ + auto method = variants.type; + Block block; + + if (false) {} // NOLINT +#define M(NAME) \ + else if (method == AggregatedDataVariants::Type::NAME) \ + block = convertOneBucketToBlock(variants, *variants.NAME, arena, final, bucket); \ + + APPLY_FOR_VARIANTS_TWO_LEVEL(M) +#undef M + + return block; +} template void Aggregator::writeToTemporaryFileImpl( diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h index 65d84b9111a..5f487da627b 100644 --- a/src/Interpreters/Aggregator.h +++ b/src/Interpreters/Aggregator.h @@ -1304,6 +1304,8 @@ private: bool final, Int32 bucket) const; + Block convertOneBucketToBlock(AggregatedDataVariants & variants, Arena * arena, bool final, Int32 bucket) const; + Block mergeAndConvertOneBucketToBlock( ManyAggregatedDataVariants & variants, Arena * arena, diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 107d2d5517e..3ac6344509c 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -88,23 +88,23 @@ Pipe outputPerPartitionIfRequested( } else { + const size_t partitions_per_stream = std::max(1, countPartitions(parts_with_ranges) / num_streams); num_streams = std::max(1, num_streams / countPartitions(parts_with_ranges)); Pipes pipes; - for (auto begin = parts_with_ranges.begin(); begin != parts_with_ranges.end();) + for (auto begin = parts_with_ranges.begin(), end = begin; end != parts_with_ranges.end(); begin = end) { - const auto end = std::find_if( - begin, - parts_with_ranges.end(), - [&begin](const auto & part) { return begin->data_part->info.partition_id != part.data_part->info.partition_id; }); + for (size_t i = 0; i < partitions_per_stream; ++i) + end = std::find_if( + end, + parts_with_ranges.end(), + [&end](const auto & part) { return end->data_part->info.partition_id != part.data_part->info.partition_id; }); RangesInDataParts partition_parts{std::make_move_iterator(begin), std::make_move_iterator(end)}; pipes.emplace_back(read(std::move(partition_parts), num_streams)); if (!pipes.back().empty()) pipes.back().resize(1); - - begin = end; } return Pipe::unitePipes(std::move(pipes)); diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index ea4d1d062f4..8570191c4e9 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -6,6 +6,8 @@ #include #include +#include + namespace ProfileEvents { extern const Event ExternalAggregationMerge; @@ -150,26 +152,33 @@ public: protected: Chunk generate() override { - if (!convertion_is_done) + if (variant->isTwoLevel()) { - blocks = params->aggregator.convertToBlocks(*variant, params->final, 1 /* max_threads */); - convertion_is_done = true; + if (current_bucket_num < NUM_BUCKETS) + { + Arena * arena = variant->aggregates_pool; + Block block = params->aggregator.convertOneBucketToBlock(*variant, arena, params->final, current_bucket_num++); + return convertToChunk(block); + } + } + else if (!single_level_converted) + { + Block block = params->aggregator.prepareBlockAndFillSingleLevel(*variant, params->final); + single_level_converted = true; + return convertToChunk(block); } - if (blocks.empty()) - return {}; - - auto res = convertToChunk(blocks.front()); - blocks.pop_front(); - return res; + return {}; } private: + static constexpr UInt32 NUM_BUCKETS = 256; + AggregatingTransformParamsPtr params; AggregatedDataVariantsPtr variant; - bool convertion_is_done = false; - BlocksList blocks; + UInt32 current_bucket_num = 0; + bool single_level_converted = false; }; /// Reads chunks from GroupingAggregatedTransform (stored in ChunksToMerge structure) and outputs them. @@ -716,18 +725,24 @@ void AggregatingTransform::initGenerate() /// Converts hash tables to blocks with data (finalized or not). pipes.emplace_back(std::make_shared(params, variant)); Pipe pipe = Pipe::unitePipes(std::move(pipes)); - if (should_produce_results_in_order_of_bucket_number) + if (!pipe.empty()) { - /// Groups chunks with the same bucket_id and outputs them (as a vector of chunks) in order of bucket_id. - pipe.addTransform(std::make_shared(pipe.getHeader(), pipe.numOutputPorts(), params)); - /// Outputs one chunk from group at a time in order of bucket_id. - pipe.addTransform(std::make_shared(pipe.getHeader(), params->getHeader())); - } - else - { - /// AggregatingTransform::expandPipeline expects single output port. - /// It's not a big problem because we do resize() to max_threads after AggregatingTransform. - pipe.resize(1); + if (should_produce_results_in_order_of_bucket_number) + { + /// Groups chunks with the same bucket_id and outputs them (as a vector of chunks) in order of bucket_id. + pipe.addTransform(std::make_shared(pipe.getHeader(), pipe.numOutputPorts(), params)); + /// Outputs one chunk from group at a time in order of bucket_id. + pipe.addTransform(std::make_shared(pipe.getHeader(), params->getHeader())); + } + else + { + pipe.addSimpleTransform( + [this](const Block & header) + { return std::make_shared(header, params->params.max_block_size, 0); }); + /// AggregatingTransform::expandPipeline expects single output port. + /// It's not a big problem because we do resize() to max_threads after AggregatingTransform. + pipe.resize(1); + } } processors = Pipe::detachProcessors(std::move(pipe)); } From 581f31ad3d2986d7ad5cd2c1516ecf4595981d0e Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Sun, 22 Jan 2023 19:39:24 +0000 Subject: [PATCH 078/717] better --- .../custom-partitioning-key.md | 8 ++++---- src/Interpreters/Aggregator.cpp | 2 +- src/Processors/QueryPlan/AggregatingStep.h | 2 +- .../Optimizations/actionsDAGUtils.cpp | 4 ++-- .../useDataParallelAggregation.cpp | 8 ++++---- .../QueryPlan/ReadFromMergeTree.cpp | 8 ++++++-- .../Transforms/AggregatingTransform.h | 2 +- .../02521_aggregation_by_partitions.reference | 2 ++ .../02521_aggregation_by_partitions.sql | 20 +++++++++++++++++++ 9 files changed, 41 insertions(+), 15 deletions(-) diff --git a/docs/en/engines/table-engines/mergetree-family/custom-partitioning-key.md b/docs/en/engines/table-engines/mergetree-family/custom-partitioning-key.md index 76996cae939..b1e79c4c3fd 100644 --- a/docs/en/engines/table-engines/mergetree-family/custom-partitioning-key.md +++ b/docs/en/engines/table-engines/mergetree-family/custom-partitioning-key.md @@ -160,13 +160,13 @@ GROUP BY UserID; ``` :::warning -Performance of such a query heavily depends on the table layout. Because of that it is not enabled by default. +Performance of such a query heavily depends on the table layout. Because of that the optimisation is not enabled by default. ::: -The key factors are: +The key factors for a good performance: -- number of partitions involved in the query should be sufficiently large (at least `max_threads / 2`), otherwise query will underutilize the machine -- number of partitions involved in the query should be not too big, so batch processing won't degenerate into row-by-row processing +- number of partitions involved in the query should be sufficiently large (more than `max_threads / 2`), otherwise query will underutilize the machine +- partitions shouldn't be too small, so batch processing won't degenerate into row-by-row processing - partitions should be comparable in size, so all threads will do roughly the same amount of work :::info diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index cb758826357..55e49eb8d19 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -1697,7 +1697,7 @@ Block Aggregator::mergeAndConvertOneBucketToBlock( Block Aggregator::convertOneBucketToBlock(AggregatedDataVariants & variants, Arena * arena, bool final, Int32 bucket) const { - auto method = variants.type; + const auto method = variants.type; Block block; if (false) {} // NOLINT diff --git a/src/Processors/QueryPlan/AggregatingStep.h b/src/Processors/QueryPlan/AggregatingStep.h index c0729809ccf..de52918386f 100644 --- a/src/Processors/QueryPlan/AggregatingStep.h +++ b/src/Processors/QueryPlan/AggregatingStep.h @@ -84,7 +84,7 @@ private: SortDescription group_by_sort_description; /// These settings are used to determine if we should resize pipeline to 1 at the end. - bool should_produce_results_in_order_of_bucket_number; + const bool should_produce_results_in_order_of_bucket_number; bool memory_bound_merging_of_aggregation_results_enabled; Processors aggregating_in_order; diff --git a/src/Processors/QueryPlan/Optimizations/actionsDAGUtils.cpp b/src/Processors/QueryPlan/Optimizations/actionsDAGUtils.cpp index 5d02436444f..643e93146f4 100644 --- a/src/Processors/QueryPlan/Optimizations/actionsDAGUtils.cpp +++ b/src/Processors/QueryPlan/Optimizations/actionsDAGUtils.cpp @@ -81,7 +81,7 @@ MatchedTrees::Matches matchTrees(const ActionsDAG & inner_dag, const ActionsDAG continue; /// Create an empty match for current node. - /// natch.node will be set if match is found. + /// match.node will be set if match is found. auto & match = matches[frame.node]; if (frame.node->type == ActionsDAG::ActionType::INPUT || frame.node->type == ActionsDAG::ActionType::COLUMN) @@ -153,7 +153,7 @@ MatchedTrees::Matches matchTrees(const ActionsDAG & inner_dag, const ActionsDAG { bool all_children_matched = true; for (size_t i = 0; all_children_matched && i < num_children; ++i) - all_children_matched &= frame.mapped_children[i] == children[i]; + all_children_matched = frame.mapped_children[i] == children[i]; if (all_children_matched) { diff --git a/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp b/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp index 2a5619cf27f..bf4622bebb1 100644 --- a/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp +++ b/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp @@ -89,7 +89,7 @@ bool allOutputsDependsOnlyOnAllowedNodes( if (visited.contains(node)) return visited[node]; - auto has_match_in_group_by_actions = [&irreducible_nodes, &matches, &node]() + auto has_match_in_group_by_actions_that_is_also_irreducible = [&irreducible_nodes, &matches, &node]() { /// `matches` maps partition key nodes into nodes in group by actions if (matches.contains(node)) @@ -104,7 +104,7 @@ bool allOutputsDependsOnlyOnAllowedNodes( return false; }; - bool res = has_match_in_group_by_actions(); + bool res = has_match_in_group_by_actions_that_is_also_irreducible(); if (!res) { switch (node->type) @@ -160,13 +160,13 @@ bool isPartitionKeySuitsGroupByKey(const ReadFromMergeTree & reading, ActionsDAG group_by_actions->removeUnusedActions(aggregating.getParams().keys); if (group_by_actions->hasArrayJoin() || group_by_actions->hasStatefulFunctions() /* || group_by_actions->assertDeterministic() */) return false; - const auto & gb_keys = group_by_actions->getRequiredColumnsNames(); + const auto & gb_key_required_columns = group_by_actions->getRequiredColumnsNames(); const auto partition_actions = reading.getStorageMetadata()->getPartitionKey().expression->getActionsDAG().clone(); /// Check that PK columns is a subset of GBK columns. for (const auto & col : partition_actions->getRequiredColumnsNames()) - if (std::ranges::find(gb_keys, col) == gb_keys.end()) + if (std::ranges::find(gb_key_required_columns, col) == gb_key_required_columns.end()) return false; const auto irreducibe_nodes = removeInjectiveFunctionsFromResultsRecursively(group_by_actions); diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 3ac6344509c..d0e3e39bb54 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -88,8 +88,9 @@ Pipe outputPerPartitionIfRequested( } else { - const size_t partitions_per_stream = std::max(1, countPartitions(parts_with_ranges) / num_streams); - num_streams = std::max(1, num_streams / countPartitions(parts_with_ranges)); + const size_t partitions_cnt = countPartitions(parts_with_ranges); + const size_t partitions_per_stream = std::max(1, partitions_cnt / num_streams); + num_streams = std::max(1, num_streams / partitions_cnt); Pipes pipes; for (auto begin = parts_with_ranges.begin(), end = begin; end != parts_with_ranges.end(); begin = end) @@ -718,6 +719,9 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrderImpl( Pipe pipe = Pipe::unitePipes(std::move(pipes)); + /// In contrast with usual aggregation in order that allocates separate AggregatingTransform for each data part, + /// aggregation of partitioned data uses the same AggregatingTransform for all parts of the same partition. + /// Thus we need to merge all partition parts into a single sorted stream. if (output_each_partition_through_separate_port) add_merge(pipe); diff --git a/src/Processors/Transforms/AggregatingTransform.h b/src/Processors/Transforms/AggregatingTransform.h index 7c8872e2953..83dfc01e6b0 100644 --- a/src/Processors/Transforms/AggregatingTransform.h +++ b/src/Processors/Transforms/AggregatingTransform.h @@ -183,7 +183,7 @@ private: AggregatedDataVariants & variants; size_t max_threads = 1; size_t temporary_data_merge_threads = 1; - bool should_produce_results_in_order_of_bucket_number = true; /// Currently makes difference only if skip_merging == true. + bool should_produce_results_in_order_of_bucket_number = true; bool skip_merging = false; /// If we aggregate partitioned data merging is not needed. /// TODO: calculate time only for aggregation. diff --git a/tests/queries/0_stateless/02521_aggregation_by_partitions.reference b/tests/queries/0_stateless/02521_aggregation_by_partitions.reference index 4a0967fb8ba..690cd168d22 100644 --- a/tests/queries/0_stateless/02521_aggregation_by_partitions.reference +++ b/tests/queries/0_stateless/02521_aggregation_by_partitions.reference @@ -206,3 +206,5 @@ Skip merging: 1 Skip merging: 0 Skip merging: 0 Skip merging: 0 +Skip merging: 1 +Skip merging: 0 diff --git a/tests/queries/0_stateless/02521_aggregation_by_partitions.sql b/tests/queries/0_stateless/02521_aggregation_by_partitions.sql index 35d75ff513b..a17e1a9ced2 100644 --- a/tests/queries/0_stateless/02521_aggregation_by_partitions.sql +++ b/tests/queries/0_stateless/02521_aggregation_by_partitions.sql @@ -208,3 +208,23 @@ select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from ( settings force_aggregate_partitions_independently = 0, max_threads = 4; drop table t17; + +create table t18(a UInt32, b UInt32) engine=MergeTree order by a partition by a; + +insert into t18 select number, number from numbers_mt(50); + +select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from ( + explain actions=1 select a1 from t18 group by intDiv(a, 2) as a1 +) where explain like '%Skip merging: %'; + +drop table t18; + +create table t19(a UInt32, b UInt32) engine=MergeTree order by a partition by a; + +insert into t19 select number, number from numbers_mt(50); + +select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from ( + explain actions=1 select a1 from t19 group by blockNumber() as a1 +) where explain like '%Skip merging: %'; + +drop table t19; From 3e3d1d15a8b01cee7fbae9c7b0df9fcfa08bd157 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Sun, 22 Jan 2023 20:30:20 +0000 Subject: [PATCH 079/717] properly handle case when there is nondeterministic functions --- src/Interpreters/ActionsDAG.cpp | 8 ++++++++ src/Interpreters/ActionsDAG.h | 1 + .../Optimizations/useDataParallelAggregation.cpp | 2 +- .../02521_aggregation_by_partitions.reference | 1 + .../0_stateless/02521_aggregation_by_partitions.sql | 10 ++++++++++ 5 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index 5f1398fed39..6c69ca1974e 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -1055,6 +1055,14 @@ void ActionsDAG::assertDeterministic() const "Expression must be deterministic but it contains non-deterministic part `{}`", node.result_name); } +bool ActionsDAG::hasNonDeterministic() const +{ + for (const auto & node : nodes) + if (!node.is_deterministic) + return true; + return false; +} + void ActionsDAG::addMaterializingOutputActions() { for (auto & output_node : outputs) diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index 40bc76fe057..89e83e24dc2 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -225,6 +225,7 @@ public: bool hasStatefulFunctions() const; bool trivial() const; /// If actions has no functions or array join. void assertDeterministic() const; /// Throw if not isDeterministic. + bool hasNonDeterministic() const; #if USE_EMBEDDED_COMPILER void compileExpressions(size_t min_count_to_compile_expression, const std::unordered_set & lazy_executed_nodes = {}); diff --git a/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp b/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp index bf4622bebb1..b0cb3eaac0c 100644 --- a/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp +++ b/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp @@ -158,7 +158,7 @@ bool isPartitionKeySuitsGroupByKey(const ReadFromMergeTree & reading, ActionsDAG /// We are interested only in calculations required to obtain group by keys. group_by_actions->removeUnusedActions(aggregating.getParams().keys); - if (group_by_actions->hasArrayJoin() || group_by_actions->hasStatefulFunctions() /* || group_by_actions->assertDeterministic() */) + if (group_by_actions->hasArrayJoin() || group_by_actions->hasStatefulFunctions() || group_by_actions->hasNonDeterministic()) return false; const auto & gb_key_required_columns = group_by_actions->getRequiredColumnsNames(); diff --git a/tests/queries/0_stateless/02521_aggregation_by_partitions.reference b/tests/queries/0_stateless/02521_aggregation_by_partitions.reference index 690cd168d22..abb5bb339e9 100644 --- a/tests/queries/0_stateless/02521_aggregation_by_partitions.reference +++ b/tests/queries/0_stateless/02521_aggregation_by_partitions.reference @@ -208,3 +208,4 @@ Skip merging: 0 Skip merging: 0 Skip merging: 1 Skip merging: 0 +Skip merging: 0 diff --git a/tests/queries/0_stateless/02521_aggregation_by_partitions.sql b/tests/queries/0_stateless/02521_aggregation_by_partitions.sql index a17e1a9ced2..99323e9b2d0 100644 --- a/tests/queries/0_stateless/02521_aggregation_by_partitions.sql +++ b/tests/queries/0_stateless/02521_aggregation_by_partitions.sql @@ -228,3 +228,13 @@ select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from ( ) where explain like '%Skip merging: %'; drop table t19; + +create table t20(a UInt32, b UInt32) engine=MergeTree order by a partition by a; + +insert into t20 select number, number from numbers_mt(50); + +select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from ( + explain actions=1 select a1 from t20 group by rand(a) as a1 +) where explain like '%Skip merging: %'; + +drop table t20; From 8b51bd84c07a8f5c3ae728418d610754cf500a30 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Mon, 23 Jan 2023 13:09:50 +0000 Subject: [PATCH 080/717] improve perf --- .../Transforms/AggregatingTransform.cpp | 13 ++++++-- .../Transforms/SquashingChunksTransform.cpp | 30 +++++++++++++++++++ .../Transforms/SquashingChunksTransform.h | 22 +++++++++++++- .../02521_aggregation_by_partitions.sql | 10 +++++++ 4 files changed, 71 insertions(+), 4 deletions(-) diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index 8570191c4e9..922b5c3969a 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -736,9 +736,16 @@ void AggregatingTransform::initGenerate() } else { - pipe.addSimpleTransform( - [this](const Block & header) - { return std::make_shared(header, params->params.max_block_size, 0); }); + /// If this is a final stage, we no longer have to keep chunks from different buckets into different chunks. + /// So now we can insert transform that will keep chunks size under control. It makes few times difference in exec time in some cases. + if (params->final) + { + pipe.addSimpleTransform( + [this](const Block & header) + { + return std::make_shared(header, params->params.max_block_size, 1024 * 1024); + }); + } /// AggregatingTransform::expandPipeline expects single output port. /// It's not a big problem because we do resize() to max_threads after AggregatingTransform. pipe.resize(1); diff --git a/src/Processors/Transforms/SquashingChunksTransform.cpp b/src/Processors/Transforms/SquashingChunksTransform.cpp index 76d0338d7fc..e89aec31655 100644 --- a/src/Processors/Transforms/SquashingChunksTransform.cpp +++ b/src/Processors/Transforms/SquashingChunksTransform.cpp @@ -50,4 +50,34 @@ void SquashingChunksTransform::work() } } +SimpleSquashingChunksTransform::SimpleSquashingChunksTransform( + const Block & header, size_t min_block_size_rows, size_t min_block_size_bytes) + : ISimpleTransform(header, header, true), squashing(min_block_size_rows, min_block_size_bytes) +{ +} + +void SimpleSquashingChunksTransform::transform(Chunk & chunk) +{ + if (!finished) + { + if (auto block = squashing.add(getInputPort().getHeader().cloneWithColumns(chunk.detachColumns()))) + chunk.setColumns(block.getColumns(), block.rows()); + } + else + { + auto block = squashing.add({}); + chunk.setColumns(block.getColumns(), block.rows()); + } +} + +IProcessor::Status SimpleSquashingChunksTransform::prepare() +{ + if (!finished && input.isFinished()) + { + finished = true; + return Status::Ready; + } + return ISimpleTransform::prepare(); +} + } diff --git a/src/Processors/Transforms/SquashingChunksTransform.h b/src/Processors/Transforms/SquashingChunksTransform.h index 96c89003726..df13f539b90 100644 --- a/src/Processors/Transforms/SquashingChunksTransform.h +++ b/src/Processors/Transforms/SquashingChunksTransform.h @@ -1,7 +1,8 @@ #pragma once -#include #include +#include +#include namespace DB { @@ -28,4 +29,23 @@ private: Chunk finish_chunk; }; +/// Doesn't care about propagating exceptions and thus doesn't throw LOGICAL_ERROR if the following transform closes its input port. +class SimpleSquashingChunksTransform : public ISimpleTransform +{ +public: + explicit SimpleSquashingChunksTransform(const Block & header, size_t min_block_size_rows, size_t min_block_size_bytes); + + String getName() const override { return "SimpleSquashingTransform"; } + +protected: + void transform(Chunk &) override; + + IProcessor::Status prepare() override; + +private: + SquashingTransform squashing; + + /// When consumption is finished we need to release the final chunk regardless of its size. + bool finished = false; +}; } diff --git a/tests/queries/0_stateless/02521_aggregation_by_partitions.sql b/tests/queries/0_stateless/02521_aggregation_by_partitions.sql index 99323e9b2d0..375a323efd9 100644 --- a/tests/queries/0_stateless/02521_aggregation_by_partitions.sql +++ b/tests/queries/0_stateless/02521_aggregation_by_partitions.sql @@ -92,6 +92,8 @@ select count() from (select throwIf(count() != 2) from t6 group by a); drop table t6; +set optimize_aggregation_in_order = 0; + create table t7(a UInt32) engine=MergeTree order by a partition by intDiv(a, 2); insert into t7 select number from numbers_mt(100); @@ -238,3 +240,11 @@ select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from ( ) where explain like '%Skip merging: %'; drop table t20; + +create table t21(a UInt64, b UInt64) engine=MergeTree order by a partition by a % 16; + +insert into t21 select number, number from numbers_mt(1e6); + +select a from t21 group by a limit 10 format Null; + +drop table t21; From 8f2d680cd3e4a071a393e3693b79e41ad7726d74 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Mon, 23 Jan 2023 18:35:42 +0000 Subject: [PATCH 081/717] test with final --- .../02521_aggregation_by_partitions.reference | 1 + .../0_stateless/02521_aggregation_by_partitions.sql | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/tests/queries/0_stateless/02521_aggregation_by_partitions.reference b/tests/queries/0_stateless/02521_aggregation_by_partitions.reference index abb5bb339e9..baa85ed5346 100644 --- a/tests/queries/0_stateless/02521_aggregation_by_partitions.reference +++ b/tests/queries/0_stateless/02521_aggregation_by_partitions.reference @@ -209,3 +209,4 @@ Skip merging: 0 Skip merging: 1 Skip merging: 0 Skip merging: 0 +Skip merging: 0 diff --git a/tests/queries/0_stateless/02521_aggregation_by_partitions.sql b/tests/queries/0_stateless/02521_aggregation_by_partitions.sql index 375a323efd9..0bb4445a05a 100644 --- a/tests/queries/0_stateless/02521_aggregation_by_partitions.sql +++ b/tests/queries/0_stateless/02521_aggregation_by_partitions.sql @@ -248,3 +248,13 @@ insert into t21 select number, number from numbers_mt(1e6); select a from t21 group by a limit 10 format Null; drop table t21; + +create table t22(a UInt32, b UInt32) engine=SummingMergeTree order by a partition by a % 16; + +insert into t22 select number, number from numbers_mt(1e6); + +select replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') from ( + explain actions=1 select a from t22 final group by a +) where explain like '%Skip merging: %'; + +drop table t22; From e7ca90adabc2cb98dbf5740a3cee2f2bdd4bfcd2 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Tue, 24 Jan 2023 14:33:39 +0000 Subject: [PATCH 082/717] fix perf test --- tests/performance/aggregation_by_partitions.xml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/performance/aggregation_by_partitions.xml b/tests/performance/aggregation_by_partitions.xml index c8f88530db1..dbeaf3ce6aa 100644 --- a/tests/performance/aggregation_by_partitions.xml +++ b/tests/performance/aggregation_by_partitions.xml @@ -1,11 +1,11 @@ - 1 - 1 + + + + 0 - 256 256 - 16 @@ -35,11 +35,11 @@ - create table t_{size}_{partitions}(a UInt64) engine=MergeTree order by a partition by a % {partitions} + create table t_{size}_{partitions}(a UInt64) engine=MergeTree order by a partition by sipHash64(a) % {partitions} insert into t_{size}_{partitions} select * from numbers_mt({size}) - + optimize table t_{size}_{partitions} final select a from t_{size}_{partitions} group by a format Null From 8223fc6ab94af41b4b50db1d5974d0bbf37d644c Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Mon, 30 Jan 2023 19:49:54 +0000 Subject: [PATCH 083/717] stash --- .../QueryPlan/ReadFromMergeTree.cpp | 333 ++++++++---------- src/Processors/QueryPlan/ReadFromMergeTree.h | 21 +- .../02521_aggregation_by_partitions.reference | 141 ++++---- 3 files changed, 219 insertions(+), 276 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index d0e3e39bb54..e0a12166d40 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -75,42 +75,6 @@ size_t countPartitions(const MergeTreeData::DataPartsVector & prepared_parts) return countPartitions(prepared_parts, get_partition_id); } -template -Pipe outputPerPartitionIfRequested( - RangesInDataParts && parts_with_ranges, size_t num_streams, bool output_each_partition_through_separate_port, ReadFunc read) -{ - if (parts_with_ranges.empty()) - return {}; - - if (!output_each_partition_through_separate_port) - { - return read(std::move(parts_with_ranges), num_streams); - } - else - { - const size_t partitions_cnt = countPartitions(parts_with_ranges); - const size_t partitions_per_stream = std::max(1, partitions_cnt / num_streams); - num_streams = std::max(1, num_streams / partitions_cnt); - - Pipes pipes; - for (auto begin = parts_with_ranges.begin(), end = begin; end != parts_with_ranges.end(); begin = end) - { - for (size_t i = 0; i < partitions_per_stream; ++i) - end = std::find_if( - end, - parts_with_ranges.end(), - [&end](const auto & part) { return end->data_part->info.partition_id != part.data_part->info.partition_id; }); - - RangesInDataParts partition_parts{std::make_move_iterator(begin), std::make_move_iterator(end)}; - - pipes.emplace_back(read(std::move(partition_parts), num_streams)); - if (!pipes.back().empty()) - pipes.back().resize(1); - } - - return Pipe::unitePipes(std::move(pipes)); - } -} } namespace ProfileEvents @@ -491,8 +455,7 @@ struct PartRangesReadInfo } -Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsImpl( - RangesInDataParts && parts_with_ranges, const Names & column_names, size_t num_streams) +Pipe ReadFromMergeTree::spreadMarkRangesAmongStreams(RangesInDataParts && parts_with_ranges, size_t num_streams, const Names & column_names) { const auto & settings = context->getSettingsRef(); const auto data_settings = data.getSettings(); @@ -506,27 +469,11 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsImpl( { /// Reduce the number of num_streams if the data is small. if (info.sum_marks < num_streams * info.min_marks_for_concurrent_read && parts_with_ranges.size() < num_streams) - num_streams = std::max( - (info.sum_marks + info.min_marks_for_concurrent_read - 1) / info.min_marks_for_concurrent_read, parts_with_ranges.size()); + num_streams = std::max((info.sum_marks + info.min_marks_for_concurrent_read - 1) / info.min_marks_for_concurrent_read, parts_with_ranges.size()); } - return read( - std::move(parts_with_ranges), - column_names, - ReadType::Default, - num_streams, - info.min_marks_for_concurrent_read, - info.use_uncompressed_cache); -} - -Pipe ReadFromMergeTree::spreadMarkRangesAmongStreams(RangesInDataParts && parts_with_ranges, const Names & column_names) -{ - auto read = [this, &column_names](RangesInDataParts && parts_with_ranges_, size_t num_streams_) - { - return spreadMarkRangesAmongStreamsImpl(std::move(parts_with_ranges_), column_names, num_streams_); - }; - return outputPerPartitionIfRequested( - std::move(parts_with_ranges), requested_num_streams, output_each_partition_through_separate_port, read); + return read(std::move(parts_with_ranges), column_names, ReadType::Default, + num_streams, info.min_marks_for_concurrent_read, info.use_uncompressed_cache); } static ActionsDAGPtr createProjection(const Block & header) @@ -537,24 +484,45 @@ static ActionsDAGPtr createProjection(const Block & header) return projection; } -Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrderImpl( +Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder( RangesInDataParts && parts_with_ranges, - const Names & column_names, - const InputOrderInfoPtr & input_order_info, size_t num_streams, - bool need_preliminary_merge) + const Names & column_names, + ActionsDAGPtr & out_projection, + const InputOrderInfoPtr & input_order_info) { const auto & settings = context->getSettingsRef(); const auto data_settings = data.getSettings(); PartRangesReadInfo info(parts_with_ranges, settings, *data_settings); + Pipes res; + if (info.sum_marks == 0) return {}; + /// PREWHERE actions can remove some input columns (which are needed only for prewhere condition). + /// In case of read-in-order, PREWHERE is executed before sorting. But removed columns could be needed for sorting key. + /// To fix this, we prohibit removing any input in prewhere actions. Instead, projection actions will be added after sorting. + /// See 02354_read_in_order_prewhere.sql as an example. + bool have_input_columns_removed_after_prewhere = false; + if (prewhere_info && prewhere_info->prewhere_actions) + { + auto & outputs = prewhere_info->prewhere_actions->getOutputs(); + std::unordered_set outputs_set(outputs.begin(), outputs.end()); + for (const auto * input : prewhere_info->prewhere_actions->getInputs()) + { + if (!outputs_set.contains(input)) + { + outputs.push_back(input); + have_input_columns_removed_after_prewhere = true; + } + } + } + /// Let's split ranges to avoid reading much data. - auto split_ranges = [rows_granularity = data_settings->index_granularity, max_block_size = max_block_size] - (const auto & ranges, int direction) + auto split_ranges + = [rows_granularity = data_settings->index_granularity, max_block_size = max_block_size](const auto & ranges, int direction) { MarkRanges new_ranges; const size_t max_marks_in_range = (max_block_size + rows_granularity - 1) / rows_granularity; @@ -599,10 +567,11 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrderImpl( }; const size_t min_marks_per_stream = (info.sum_marks - 1) / num_streams + 1; + bool need_preliminary_merge = (parts_with_ranges.size() > settings.read_in_order_two_level_merge_threshold); Pipes pipes; - for (size_t i = 0; i < requested_num_streams && !parts_with_ranges.empty(); ++i) + for (size_t i = 0; i < num_streams && !parts_with_ranges.empty(); ++i) { size_t need_marks = min_marks_per_stream; RangesInDataParts new_parts; @@ -617,21 +586,18 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrderImpl( size_t & marks_in_part = info.sum_marks_in_parts.back(); /// We will not take too few rows from a part. - if (marks_in_part >= info.min_marks_for_concurrent_read && - need_marks < info.min_marks_for_concurrent_read) + if (marks_in_part >= info.min_marks_for_concurrent_read && need_marks < info.min_marks_for_concurrent_read) need_marks = info.min_marks_for_concurrent_read; /// Do not leave too few rows in the part. - if (marks_in_part > need_marks && - marks_in_part - need_marks < info.min_marks_for_concurrent_read) + if (marks_in_part > need_marks && marks_in_part - need_marks < info.min_marks_for_concurrent_read) need_marks = marks_in_part; MarkRanges ranges_to_get_from_part; /// We take full part if it contains enough marks or /// if we know limit and part contains less than 'limit' rows. - bool take_full_part = marks_in_part <= need_marks - || (input_order_info->limit && input_order_info->limit < part.getRowsCount()); + bool take_full_part = marks_in_part <= need_marks || (input_order_info->limit && input_order_info->limit < part.getRowsCount()); /// We take the whole part if it is small enough. if (take_full_part) @@ -668,12 +634,11 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrderImpl( new_parts.emplace_back(part.data_part, part.part_index_in_query, std::move(ranges_to_get_from_part)); } - auto read_type = input_order_info->direction == 1 - ? ReadFromMergeTree::ReadType::InOrder - : ReadFromMergeTree::ReadType::InReverseOrder; + auto read_type + = input_order_info->direction == 1 ? ReadFromMergeTree::ReadType::InOrder : ReadFromMergeTree::ReadType::InReverseOrder; - pipes.emplace_back(readInOrder(std::move(new_parts), column_names, read_type, - info.use_uncompressed_cache, input_order_info->limit)); + pipes.emplace_back( + readInOrder(std::move(new_parts), column_names, read_type, info.use_uncompressed_cache, input_order_info->limit)); } Block pipe_header; @@ -699,7 +664,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrderImpl( auto sorting_key_expr = std::make_shared(sorting_key_prefix_expr); - auto add_merge = [&](Pipe & pipe) + auto merge_streams = [&](Pipe & pipe) { pipe.addSimpleTransform([sorting_key_expr](const Block & header) { return std::make_shared(header, sorting_key_expr); }); @@ -713,71 +678,28 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrderImpl( } }; - if (!output_each_partition_through_separate_port) - for (auto & pipe : pipes) - add_merge(pipe); + if (!pipes.empty() && output_each_partition_through_separate_port) + { + /// In contrast with usual aggregation in order that allocates separate AggregatingTransform for each data part, + /// aggregation of partitioned data uses the same AggregatingTransform for all parts of the same partition. + /// Thus we need to merge all partition parts into a single sorted stream. + Pipe pipe = Pipe::unitePipes(std::move(pipes)); + merge_streams(pipe); + out_projection = createProjection(pipe_header); + return pipe; + } - Pipe pipe = Pipe::unitePipes(std::move(pipes)); - - /// In contrast with usual aggregation in order that allocates separate AggregatingTransform for each data part, - /// aggregation of partitioned data uses the same AggregatingTransform for all parts of the same partition. - /// Thus we need to merge all partition parts into a single sorted stream. - if (output_each_partition_through_separate_port) - add_merge(pipe); - - return pipe; + for (auto & pipe : pipes) + merge_streams(pipe); } + if (!pipes.empty() && (need_preliminary_merge || have_input_columns_removed_after_prewhere)) + /// Drop temporary columns, added by 'sorting_key_prefix_expr' + out_projection = createProjection(pipe_header); + return Pipe::unitePipes(std::move(pipes)); } -Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder( - RangesInDataParts && parts_with_ranges, - const Names & column_names, - ActionsDAGPtr & out_projection, - const InputOrderInfoPtr & input_order_info) -{ - if (parts_with_ranges.empty()) - return {}; - - /// PREWHERE actions can remove some input columns (which are needed only for prewhere condition). - /// In case of read-in-order, PREWHERE is executed before sorting. But removed columns could be needed for sorting key. - /// To fix this, we prohibit removing any input in prewhere actions. Instead, projection actions will be added after sorting. - /// See 02354_read_in_order_prewhere.sql as an example. - bool have_input_columns_removed_after_prewhere = false; - if (prewhere_info && prewhere_info->prewhere_actions) - { - auto & outputs = prewhere_info->prewhere_actions->getOutputs(); - std::unordered_set outputs_set(outputs.begin(), outputs.end()); - for (const auto * input : prewhere_info->prewhere_actions->getInputs()) - { - if (!outputs_set.contains(input)) - { - outputs.push_back(input); - have_input_columns_removed_after_prewhere = true; - } - } - } - - const auto & settings = context->getSettingsRef(); - bool need_preliminary_merge = (parts_with_ranges.size() > settings.read_in_order_two_level_merge_threshold); - - auto read - = [this, &column_names, &input_order_info, need_preliminary_merge](RangesInDataParts && parts_with_ranges_, size_t num_streams_) - { - return spreadMarkRangesAmongStreamsWithOrderImpl( - std::move(parts_with_ranges_), column_names, input_order_info, num_streams_, need_preliminary_merge); - }; - Pipe pipe = outputPerPartitionIfRequested( - std::move(parts_with_ranges), requested_num_streams, output_each_partition_through_separate_port, read); - - if (!pipe.empty() && (need_preliminary_merge || have_input_columns_removed_after_prewhere)) - /// Drop temporary columns, added by 'sorting_key_prefix_expr' - out_projection = createProjection(pipe.getHeader()); - - return pipe; -} - static void addMergingFinal( Pipe & pipe, const SortDescription & sort_description, @@ -831,16 +753,14 @@ static void addMergingFinal( Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( - RangesInDataParts && parts_with_ranges, - const Names & column_names, - ActionsDAGPtr & out_projection) + RangesInDataParts && parts_with_ranges, size_t num_streams, const Names & column_names, ActionsDAGPtr & out_projection) { const auto & settings = context->getSettingsRef(); const auto data_settings = data.getSettings(); PartRangesReadInfo info(parts_with_ranges, settings, *data_settings); - size_t num_streams = requested_num_streams; + assert(num_streams == requested_num_streams); if (num_streams > settings.max_final_threads) num_streams = settings.max_final_threads; @@ -1395,6 +1315,84 @@ bool ReadFromMergeTree::isQueryWithFinal() const return select.final(); } +Pipe ReadFromMergeTree::spreadMarkRanges( + RangesInDataParts && parts_with_ranges, size_t num_streams, AnalysisResult & result, ActionsDAGPtr & result_projection) +{ + bool final = isQueryWithFinal(); + const auto & input_order_info = query_info.getInputOrderInfo(); + + Names column_names_to_read = result.column_names_to_read; + + if (!final && result.sampling.use_sampling) + { + /// Add columns needed for `sample_by_ast` to `column_names_to_read`. + /// Skip this if final was used, because such columns were already added from PK. + std::vector add_columns = result.sampling.filter_expression->getRequiredColumns().getNames(); + column_names_to_read.insert(column_names_to_read.end(), add_columns.begin(), add_columns.end()); + ::sort(column_names_to_read.begin(), column_names_to_read.end()); + column_names_to_read.erase(std::unique(column_names_to_read.begin(), column_names_to_read.end()), column_names_to_read.end()); + } + + if (final) + { + if (output_each_partition_through_separate_port) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Optimisation isn't supposed to be used for queries with final"); + + /// Add columns needed to calculate the sorting expression and the sign. + std::vector add_columns = metadata_for_reading->getColumnsRequiredForSortingKey(); + column_names_to_read.insert(column_names_to_read.end(), add_columns.begin(), add_columns.end()); + + if (!data.merging_params.sign_column.empty()) + column_names_to_read.push_back(data.merging_params.sign_column); + if (!data.merging_params.version_column.empty()) + column_names_to_read.push_back(data.merging_params.version_column); + + ::sort(column_names_to_read.begin(), column_names_to_read.end()); + column_names_to_read.erase(std::unique(column_names_to_read.begin(), column_names_to_read.end()), column_names_to_read.end()); + + return spreadMarkRangesAmongStreamsFinal(std::move(parts_with_ranges), num_streams, column_names_to_read, result_projection); + } + else if (input_order_info) + { + return spreadMarkRangesAmongStreamsWithOrder( + std::move(parts_with_ranges), num_streams, column_names_to_read, result_projection, input_order_info); + } + else + { + return spreadMarkRangesAmongStreams(std::move(parts_with_ranges), num_streams, column_names_to_read); + } +} + +Pipe ReadFromMergeTree::groupStreamsByPartition(AnalysisResult & result, ActionsDAGPtr & result_projection) +{ + auto && parts_with_ranges = std::move(result.parts_with_ranges); + + if (parts_with_ranges.empty()) + return {}; + + const size_t partitions_cnt = countPartitions(parts_with_ranges); + const size_t partitions_per_stream = std::max(1, partitions_cnt / requested_num_streams); + const size_t num_streams = std::max(1, requested_num_streams / partitions_cnt); + + Pipes pipes; + for (auto begin = parts_with_ranges.begin(), end = begin; end != parts_with_ranges.end(); begin = end) + { + for (size_t i = 0; i < partitions_per_stream; ++i) + end = std::find_if( + end, + parts_with_ranges.end(), + [&end](const auto & part) { return end->data_part->info.partition_id != part.data_part->info.partition_id; }); + + RangesInDataParts partition_parts{std::make_move_iterator(begin), std::make_move_iterator(end)}; + + pipes.emplace_back(spreadMarkRanges(std::move(partition_parts), num_streams, result, result_projection)); + if (!pipes.back().empty()) + pipes.back().resize(1); + } + + return Pipe::unitePipes(std::move(pipes)); +} + void ReadFromMergeTree::initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) { auto result = getAnalysisResult(); @@ -1429,60 +1427,9 @@ void ReadFromMergeTree::initializePipeline(QueryPipelineBuilder & pipeline, cons /// NOTE: It may lead to double computation of expressions. ActionsDAGPtr result_projection; - Names column_names_to_read = std::move(result.column_names_to_read); - bool final = isQueryWithFinal(); - - if (!final && result.sampling.use_sampling) - { - /// Add columns needed for `sample_by_ast` to `column_names_to_read`. - /// Skip this if final was used, because such columns were already added from PK. - std::vector add_columns = result.sampling.filter_expression->getRequiredColumns().getNames(); - column_names_to_read.insert(column_names_to_read.end(), add_columns.begin(), add_columns.end()); - ::sort(column_names_to_read.begin(), column_names_to_read.end()); - column_names_to_read.erase(std::unique(column_names_to_read.begin(), column_names_to_read.end()), - column_names_to_read.end()); - } - - Pipe pipe; - - const auto & input_order_info = query_info.getInputOrderInfo(); - - if (final) - { - if (output_each_partition_through_separate_port) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Optimisation isn't supposed to be used for queries with final"); - - /// Add columns needed to calculate the sorting expression and the sign. - std::vector add_columns = metadata_for_reading->getColumnsRequiredForSortingKey(); - column_names_to_read.insert(column_names_to_read.end(), add_columns.begin(), add_columns.end()); - - if (!data.merging_params.sign_column.empty()) - column_names_to_read.push_back(data.merging_params.sign_column); - if (!data.merging_params.version_column.empty()) - column_names_to_read.push_back(data.merging_params.version_column); - - ::sort(column_names_to_read.begin(), column_names_to_read.end()); - column_names_to_read.erase(std::unique(column_names_to_read.begin(), column_names_to_read.end()), column_names_to_read.end()); - - pipe = spreadMarkRangesAmongStreamsFinal( - std::move(result.parts_with_ranges), - column_names_to_read, - result_projection); - } - else if (input_order_info) - { - pipe = spreadMarkRangesAmongStreamsWithOrder( - std::move(result.parts_with_ranges), - column_names_to_read, - result_projection, - input_order_info); - } - else - { - pipe = spreadMarkRangesAmongStreams( - std::move(result.parts_with_ranges), - column_names_to_read); - } + Pipe pipe = output_each_partition_through_separate_port + ? groupStreamsByPartition(result, result_projection) + : spreadMarkRanges(std::move(result.parts_with_ranges), requested_num_streams, result, result_projection); for (const auto & processor : pipe.getProcessors()) processor->setStorageLimits(query_info.storage_limits); diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.h b/src/Processors/QueryPlan/ReadFromMergeTree.h index 1e7a1696a30..c53b22873fb 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.h +++ b/src/Processors/QueryPlan/ReadFromMergeTree.h @@ -226,29 +226,22 @@ private: template ProcessorPtr createSource(const RangesInDataPart & part, const Names & required_columns, bool use_uncompressed_cache, bool has_limit_below_one_block); - Pipe spreadMarkRangesAmongStreams( - RangesInDataParts && parts_with_ranges, - const Names & column_names); + Pipe spreadMarkRanges( + RangesInDataParts && parts_with_ranges, size_t num_streams, AnalysisResult & result, ActionsDAGPtr & result_projection); - Pipe spreadMarkRangesAmongStreamsImpl(RangesInDataParts && parts_with_ranges, const Names & column_names, size_t num_streams); + Pipe groupStreamsByPartition(AnalysisResult & result, ActionsDAGPtr & result_projection); + + Pipe spreadMarkRangesAmongStreams(RangesInDataParts && parts_with_ranges, size_t num_streams, const Names & column_names); Pipe spreadMarkRangesAmongStreamsWithOrder( RangesInDataParts && parts_with_ranges, + size_t num_streams, const Names & column_names, ActionsDAGPtr & out_projection, const InputOrderInfoPtr & input_order_info); - Pipe spreadMarkRangesAmongStreamsWithOrderImpl( - RangesInDataParts && parts_with_ranges, - const Names & column_names, - const InputOrderInfoPtr & input_order_info, - size_t num_streams, - bool need_preliminary_merge); - Pipe spreadMarkRangesAmongStreamsFinal( - RangesInDataParts && parts, - const Names & column_names, - ActionsDAGPtr & out_projection); + RangesInDataParts && parts, size_t num_streams, const Names & column_names, ActionsDAGPtr & out_projection); MergeTreeDataSelectAnalysisResultPtr selectRangesToRead(MergeTreeData::DataPartsVector parts) const; ReadFromMergeTree::AnalysisResult getAnalysisResult() const; diff --git a/tests/queries/0_stateless/02521_aggregation_by_partitions.reference b/tests/queries/0_stateless/02521_aggregation_by_partitions.reference index baa85ed5346..188a6889002 100644 --- a/tests/queries/0_stateless/02521_aggregation_by_partitions.reference +++ b/tests/queries/0_stateless/02521_aggregation_by_partitions.reference @@ -91,18 +91,19 @@ ExpressionTransform × 16 (Expression) ExpressionTransform × 4 (ReadFromMergeTree) - MergingSortedTransform 2 → 1 - ExpressionTransform × 2 - MergeTreeInOrder × 2 0 → 1 - MergingSortedTransform 2 → 1 - ExpressionTransform × 2 - MergeTreeInOrder × 2 0 → 1 - MergingSortedTransform 2 → 1 - ExpressionTransform × 2 - MergeTreeInOrder × 2 0 → 1 - MergingSortedTransform 2 → 1 - ExpressionTransform × 2 - MergeTreeInOrder × 2 0 → 1 + ExpressionTransform × 4 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 1000000 (Expression) ExpressionTransform × 16 @@ -113,6 +114,41 @@ ExpressionTransform × 16 (Expression) ExpressionTransform × 8 (ReadFromMergeTree) + ExpressionTransform × 8 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 +1000000 +(Expression) +ExpressionTransform × 16 + (Aggregating) + FinalizeAggregatedTransform × 16 + AggregatingInOrderTransform × 16 + (Expression) + ExpressionTransform × 16 + (ReadFromMergeTree) + ExpressionTransform × 16 MergingSortedTransform 2 → 1 ExpressionTransform × 2 MergeTreeInOrder × 2 0 → 1 @@ -137,63 +173,30 @@ ExpressionTransform × 16 MergingSortedTransform 2 → 1 ExpressionTransform × 2 MergeTreeInOrder × 2 0 → 1 -1000000 -(Expression) -ExpressionTransform × 16 - (Aggregating) - FinalizeAggregatedTransform × 16 - AggregatingInOrderTransform × 16 - (Expression) - ExpressionTransform × 16 - (ReadFromMergeTree) - MergingSortedTransform 2 → 1 - ExpressionTransform × 2 - MergeTreeInOrder × 2 0 → 1 - MergingSortedTransform 2 → 1 - ExpressionTransform × 2 - MergeTreeInOrder × 2 0 → 1 - MergingSortedTransform 2 → 1 - ExpressionTransform × 2 - MergeTreeInOrder × 2 0 → 1 - MergingSortedTransform 2 → 1 - ExpressionTransform × 2 - MergeTreeInOrder × 2 0 → 1 - MergingSortedTransform 2 → 1 - ExpressionTransform × 2 - MergeTreeInOrder × 2 0 → 1 - MergingSortedTransform 2 → 1 - ExpressionTransform × 2 - MergeTreeInOrder × 2 0 → 1 - MergingSortedTransform 2 → 1 - ExpressionTransform × 2 - MergeTreeInOrder × 2 0 → 1 - MergingSortedTransform 2 → 1 - ExpressionTransform × 2 - MergeTreeInOrder × 2 0 → 1 - MergingSortedTransform 2 → 1 - ExpressionTransform × 2 - MergeTreeInOrder × 2 0 → 1 - MergingSortedTransform 2 → 1 - ExpressionTransform × 2 - MergeTreeInOrder × 2 0 → 1 - MergingSortedTransform 2 → 1 - ExpressionTransform × 2 - MergeTreeInOrder × 2 0 → 1 - MergingSortedTransform 2 → 1 - ExpressionTransform × 2 - MergeTreeInOrder × 2 0 → 1 - MergingSortedTransform 2 → 1 - ExpressionTransform × 2 - MergeTreeInOrder × 2 0 → 1 - MergingSortedTransform 2 → 1 - ExpressionTransform × 2 - MergeTreeInOrder × 2 0 → 1 - MergingSortedTransform 2 → 1 - ExpressionTransform × 2 - MergeTreeInOrder × 2 0 → 1 - MergingSortedTransform 2 → 1 - ExpressionTransform × 2 - MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 + MergingSortedTransform 2 → 1 + ExpressionTransform × 2 + MergeTreeInOrder × 2 0 → 1 1000000 Skip merging: 1 Skip merging: 1 From fdb9336b0273c0aaaec13380a0c7fe6c88a5ef48 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Mon, 30 Jan 2023 23:08:26 +0000 Subject: [PATCH 084/717] Respect aliases when compare DISTINCT columns --- src/Interpreters/ActionsDAG.cpp | 26 ++++++ src/Interpreters/ActionsDAG.h | 3 + .../Optimizations/removeRedundantDistinct.cpp | 76 ++++++++++++---- .../02500_remove_redundant_distinct.reference | 91 ++++++++++++------- .../02500_remove_redundant_distinct.sh | 17 +++- 5 files changed, 159 insertions(+), 54 deletions(-) diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index 5f1398fed39..d437bc73b9a 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -2260,4 +2260,30 @@ ActionsDAGPtr ActionsDAG::buildFilterActionsDAG( return result_dag; } +const ActionsDAG::Node * ActionsDAG::getOriginalNodeForOutputAlias(const String & output_name) +{ + const Node * output_alias = nullptr; + for (const auto * node : outputs) + { + if (node->result_name == output_name && node->type == ActionsDAG::ActionType::ALIAS) + { + output_alias = node; + break; + } + } + if (!output_alias) + return nullptr; + + const Node * node = output_alias; + while (node && node->type == ActionsDAG::ActionType::ALIAS) + { + chassert(!node->children.empty()); + node = node->children.front(); + } + if (node->type != ActionsDAG::ActionType::INPUT) + return nullptr; + + return node; +} + } diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index 40bc76fe057..1b1ea15deb7 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -347,6 +347,9 @@ public: const std::unordered_map & node_name_to_input_node_column, const ContextPtr & context); + /// Return origial node (input) for alias in output if exists + const Node* getOriginalNodeForOutputAlias(const String & output_name); + private: NodeRawConstPtrs getParents(const Node * target) const; diff --git a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp index bed30c0fc21..9cf5c27006b 100644 --- a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp +++ b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp @@ -1,5 +1,8 @@ #include +#include #include +#include +#include #include #include #include @@ -9,30 +12,30 @@ namespace DB::QueryPlanOptimizations { -// template -// FMT_NODISCARD FMT_INLINE auto format(format_string fmt, T&&... args) constexpr bool debug_logging_enabled = true; -template -void logDebug(const char * format, Args &&... args) +void logDebug(const String & prefix, const String & message) { if constexpr (debug_logging_enabled) { - LOG_DEBUG(&Poco::Logger::get("redundantDistinct"), format, args...); + LOG_DEBUG(&Poco::Logger::get("redundantDistinct"), "{}: {}", prefix, message); } } -static std::set getDistinctColumns(const DistinctStep * distinct) +namespace { - /// find non-const columns in DISTINCT - const ColumnsWithTypeAndName & distinct_columns = distinct->getOutputStream().header.getColumnsWithTypeAndName(); - std::set non_const_columns; - for (const auto & column : distinct_columns) + std::set getDistinctColumns(const DistinctStep * distinct) { - if (!isColumnConst(*column.column)) - non_const_columns.emplace(column.name); + /// find non-const columns in DISTINCT + const ColumnsWithTypeAndName & distinct_columns = distinct->getOutputStream().header.getColumnsWithTypeAndName(); + std::set non_const_columns; + for (const auto & column : distinct_columns) + { + if (!isColumnConst(*column.column)) + non_const_columns.emplace(column.name); + } + return non_const_columns; } - return non_const_columns; } size_t tryRemoveRedundantDistinct(QueryPlan::Node * parent_node, QueryPlan::Nodes & /* nodes*/) @@ -48,16 +51,22 @@ size_t tryRemoveRedundantDistinct(QueryPlan::Node * parent_node, QueryPlan::Node distinct_node = parent_node->children.front(); + std::vector dag_stack; const DistinctStep * inner_distinct_step = nullptr; QueryPlan::Node * node = distinct_node; while (!node->children.empty()) { - const IQueryPlanStep* current_step = node->step.get(); + const IQueryPlanStep * current_step = node->step.get(); /// don't try to remove DISTINCT after union or join - if (typeid_cast(current_step) || typeid_cast(current_step)) + if (typeid_cast(current_step) || typeid_cast(current_step)) break; + if (const auto * const expr = typeid_cast(current_step); expr) + dag_stack.push_back(expr->getExpression()); + if (const auto * const filter = typeid_cast(current_step); filter) + dag_stack.push_back(filter->getExpression()); + node = node->children.front(); inner_distinct_step = typeid_cast(node->step.get()); if (inner_distinct_step) @@ -74,13 +83,42 @@ size_t tryRemoveRedundantDistinct(QueryPlan::Node * parent_node, QueryPlan::Node if (inner_distinct_step->isPreliminary()) return 0; - /// try to remove outer distinct step - if (getDistinctColumns(distinct_step) != getDistinctColumns(inner_distinct_step)) + const auto distinct_columns = getDistinctColumns(distinct_step); + auto inner_distinct_columns = getDistinctColumns(inner_distinct_step); + if (distinct_columns.size() != inner_distinct_columns.size()) return 0; - chassert(!distinct_node->children.empty()); + /// build actions DAG to compare DISTINCT columns + ActionsDAGPtr path_actions; + if (!dag_stack.empty()) + { + path_actions = dag_stack.back(); + dag_stack.pop_back(); + } + while (!dag_stack.empty()) + { + ActionsDAGPtr clone = dag_stack.back()->clone(); + dag_stack.pop_back(); + path_actions->mergeInplace(std::move(*clone)); + } - /// delete current distinct + logDebug("mergedDAG\n", path_actions->dumpDAG()); + + for (const auto & column : distinct_columns) + { + const auto * alias_node = path_actions->getOriginalNodeForOutputAlias(String(column)); + if (!alias_node) + return 0; + + auto it = inner_distinct_columns.find(alias_node->result_name); + if (it == inner_distinct_columns.end()) + return 0; + + inner_distinct_columns.erase(it); + } + + /// remove current distinct + chassert(!distinct_node->children.empty()); parent_node->children[0] = distinct_node->children.front(); return 1; diff --git a/tests/queries/0_stateless/02500_remove_redundant_distinct.reference b/tests/queries/0_stateless/02500_remove_redundant_distinct.reference index bb1480153db..e9d8c63771a 100644 --- a/tests/queries/0_stateless/02500_remove_redundant_distinct.reference +++ b/tests/queries/0_stateless/02500_remove_redundant_distinct.reference @@ -1,14 +1,14 @@ -- Disabled query_plan_remove_redundant_distinct -Expression (Projection) - Distinct +Expression (Project names) + Distinct (DISTINCT) Distinct (Preliminary DISTINCT) - Expression ((Before ORDER BY + Projection)) - Distinct + Expression ((Projection + (Change column names to column identifiers + Project names))) + Distinct (DISTINCT) Distinct (Preliminary DISTINCT) - Expression ((Before ORDER BY + Projection)) - Distinct + Expression ((Projection + (Change column names to column identifiers + Project names))) + Distinct (DISTINCT) Distinct (Preliminary DISTINCT) - Expression (Before ORDER BY) + Expression ((Projection + Change column names to column identifiers)) ReadFromStorage (SystemNumbers) -- Enabled query_plan_remove_redundant_distinct -- DISTINCT is only in most inner subquery @@ -24,10 +24,10 @@ FROM ) ) -- explain -Expression ((Projection + (Before ORDER BY + (Projection + (Before ORDER BY + Projection))))) - Distinct +Expression ((Project names + (Projection + (Change column names to column identifiers + (Project names + (Projection + (Change column names to column identifiers + Project names))))))) + Distinct (DISTINCT) Distinct (Preliminary DISTINCT) - Expression (Before ORDER BY) + Expression ((Projection + Change column names to column identifiers)) ReadFromStorage (SystemNumbers) -- execute 0 @@ -43,21 +43,22 @@ SELECT DISTINCT number FROM ) ORDER BY number -- explain -Expression (Projection) - Distinct +Expression (Project names) + Distinct (DISTINCT) Sorting (Sorting for ORDER BY) - Distinct (Preliminary DISTINCT) - Union - Expression ((Before ORDER BY + Projection)) - Distinct - Distinct (Preliminary DISTINCT) - Expression (Before ORDER BY) - ReadFromStorage (SystemNumbers) - Expression (( + Projection)) - Distinct - Distinct (Preliminary DISTINCT) - Expression (Before ORDER BY) - ReadFromStorage (SystemNumbers) + Expression (Before ORDER BY) + Distinct (Preliminary DISTINCT) + Union + Expression ((Projection + (Change column names to column identifiers + Project names))) + Distinct (DISTINCT) + Distinct (Preliminary DISTINCT) + Expression ((Projection + Change column names to column identifiers)) + ReadFromStorage (SystemNumbers) + Expression (( + ( + Project names))) + Distinct (DISTINCT) + Distinct (Preliminary DISTINCT) + Expression ((Projection + Change column names to column identifiers)) + ReadFromStorage (SystemNumbers) -- execute 0 1 @@ -74,21 +75,45 @@ FROM FROM numbers(2) ) SETTINGS joined_subquery_requires_alias=0 -- explain -Expression (Projection) - Distinct +Expression (Project names) + Distinct (DISTINCT) Distinct (Preliminary DISTINCT) - Expression (Before ORDER BY) + Expression ((Projection + DROP unused columns after JOIN)) Join (JOIN FillRightFirst) - Expression ((Before JOIN + Projection)) - Distinct + Expression ((Change column names to column identifiers + Project names)) + Distinct (DISTINCT) Distinct (Preliminary DISTINCT) - Expression (Before ORDER BY) + Expression ((Projection + Change column names to column identifiers)) ReadFromStorage (SystemNumbers) - Expression ((Joined actions + (Rename joined columns + Projection))) - Distinct + Expression ((Change column names to column identifiers + Project names)) + Distinct (DISTINCT) Distinct (Preliminary DISTINCT) - Expression (Before ORDER BY) + Expression ((Projection + Change column names to column identifiers)) ReadFromStorage (SystemNumbers) -- execute 0 0 +0 1 +1 0 1 1 +-- DISTINCT duplicates with several columns +-- query +SELECT DISTINCT * +FROM +( + SELECT DISTINCT * + FROM + ( + SELECT DISTINCT number as a, 2*number as b + FROM numbers(3) + ) +) +-- explain +Expression ((Project names + (Projection + (Change column names to column identifiers + (Project names + (Projection + (Change column names to column identifiers + Project names))))))) + Distinct (DISTINCT) + Distinct (Preliminary DISTINCT) + Expression ((Projection + Change column names to column identifiers)) + ReadFromStorage (SystemNumbers) +-- execute +0 0 +1 2 +2 4 diff --git a/tests/queries/0_stateless/02500_remove_redundant_distinct.sh b/tests/queries/0_stateless/02500_remove_redundant_distinct.sh index 2788ded9e2d..bcbe6d7a651 100755 --- a/tests/queries/0_stateless/02500_remove_redundant_distinct.sh +++ b/tests/queries/0_stateless/02500_remove_redundant_distinct.sh @@ -5,8 +5,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh OPTIMIZATION_SETTING="query_plan_remove_redundant_distinct" -DISABLE_OPTIMIZATION="SET $OPTIMIZATION_SETTING=0;SET optimize_duplicate_order_by_and_distinct=0" -ENABLE_OPTIMIZATION="SET $OPTIMIZATION_SETTING=1;SET optimize_duplicate_order_by_and_distinct=0" +DISABLE_OPTIMIZATION="SET allow_experimental_analyzer=1;SET $OPTIMIZATION_SETTING=0;SET optimize_duplicate_order_by_and_distinct=0" +ENABLE_OPTIMIZATION="SET allow_experimental_analyzer=1;SET $OPTIMIZATION_SETTING=1;SET optimize_duplicate_order_by_and_distinct=0" echo "-- Disabled $OPTIMIZATION_SETTING" query="SELECT DISTINCT * @@ -57,3 +57,16 @@ FROM FROM numbers(2) ) SETTINGS joined_subquery_requires_alias=0" run_query "$query" + +echo "-- DISTINCT duplicates with several columns" +query="SELECT DISTINCT * +FROM +( + SELECT DISTINCT * + FROM + ( + SELECT DISTINCT number as a, 2*number as b + FROM numbers(3) + ) +)" +run_query "$query" From f2fab484409e49b77fcbcf45dcc3f52ff75d7522 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Mon, 30 Jan 2023 20:33:18 -0300 Subject: [PATCH 085/717] add join on mix of tables supporting/ not supporting final --- ...02420_force_select_final_setting.reference | 17 +++++++++++++++ .../02420_force_select_final_setting.sql | 21 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/tests/queries/0_stateless/02420_force_select_final_setting.reference b/tests/queries/0_stateless/02420_force_select_final_setting.reference index 8be3da3545c..1fe561cd267 100644 --- a/tests/queries/0_stateless/02420_force_select_final_setting.reference +++ b/tests/queries/0_stateless/02420_force_select_final_setting.reference @@ -40,3 +40,20 @@ create view nv_regular_mt_table AS SELECT * FROM mv_regular_mt_table; set force_select_final=1; select count() from nv_regular_mt_table; 2 +-- join on mix of tables that support / do not support select final +create table if not exists left_table (x String) engine=ReplacingMergeTree() ORDER BY x; +create table if not exists middle_table (x String) engine=MergeTree() ORDER BY x; +create table if not exists right_table (x String) engine=ReplacingMergeTree() ORDER BY x; +insert into left_table values ('abc'); +insert into left_table values ('abc'); +insert into left_table values ('abc'); +insert into middle_table values ('abc'); +insert into middle_table values ('abc'); +insert into right_table values ('abc'); +insert into right_table values ('abc'); +insert into right_table values ('abc'); +-- Expected output is 2 because middle table does not support final +select count() from left_table + inner join middle_table on left_table.x = middle_table.x + inner join right_table on middle_table.x = right_table.x; +2 diff --git a/tests/queries/0_stateless/02420_force_select_final_setting.sql b/tests/queries/0_stateless/02420_force_select_final_setting.sql index 84d4c40445a..c074d1a8926 100644 --- a/tests/queries/0_stateless/02420_force_select_final_setting.sql +++ b/tests/queries/0_stateless/02420_force_select_final_setting.sql @@ -46,3 +46,24 @@ create view nv_regular_mt_table AS SELECT * FROM mv_regular_mt_table; set force_select_final=1; select count() from nv_regular_mt_table; + +-- join on mix of tables that support / do not support select final +create table if not exists left_table (x String) engine=ReplacingMergeTree() ORDER BY x; +create table if not exists middle_table (x String) engine=MergeTree() ORDER BY x; +create table if not exists right_table (x String) engine=ReplacingMergeTree() ORDER BY x; + +insert into left_table values ('abc'); +insert into left_table values ('abc'); +insert into left_table values ('abc'); + +insert into middle_table values ('abc'); +insert into middle_table values ('abc'); + +insert into right_table values ('abc'); +insert into right_table values ('abc'); +insert into right_table values ('abc'); + +-- Expected output is 2 because middle table does not support final +select count() from left_table + inner join middle_table on left_table.x = middle_table.x + inner join right_table on middle_table.x = right_table.x; From c022e2fd72fbac952cd411c95105411c7491e036 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 31 Jan 2023 10:14:09 +0000 Subject: [PATCH 086/717] Fix typo --- src/Interpreters/ActionsDAG.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index 1b1ea15deb7..fa394c5bde6 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -347,7 +347,7 @@ public: const std::unordered_map & node_name_to_input_node_column, const ContextPtr & context); - /// Return origial node (input) for alias in output if exists + /// Return original node (input) for alias in output if exists const Node* getOriginalNodeForOutputAlias(const String & output_name); private: From f194b452e7c5120c31d1108ef17957565aad24b5 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 31 Jan 2023 09:13:57 -0300 Subject: [PATCH 087/717] auto -> bool --- src/Interpreters/InterpreterSelectQuery.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 250d92cbe4c..12b6f6f3297 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -3010,9 +3010,9 @@ void InterpreterSelectQuery::ignoreWithTotals() bool InterpreterSelectQuery::forceSelectFinalOnSelectQuery(ASTSelectQuery & query) { // query.tables() is required because not all queries have tables in it, it could be a function. - auto is_force_select_final_setting_on = context->getSettingsRef().force_select_final; - auto is_final_supported = storage && storage->supportsFinal() && !storage->isRemote() && query.tables(); - auto is_query_already_final = query.final(); + bool is_force_select_final_setting_on = context->getSettingsRef().force_select_final; + bool is_final_supported = storage && storage->supportsFinal() && !storage->isRemote() && query.tables(); + bool is_query_already_final = query.final(); return is_force_select_final_setting_on && !is_query_already_final && is_final_supported; } From fd1ee98183315fb050768a1f1983791b2ec0acbe Mon Sep 17 00:00:00 2001 From: flynn Date: Tue, 31 Jan 2023 14:40:48 +0000 Subject: [PATCH 088/717] fix style --- src/Storages/StorageS3.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/Storages/StorageS3.h b/src/Storages/StorageS3.h index 5d98663f1d0..f0465d6623c 100644 --- a/src/Storages/StorageS3.h +++ b/src/Storages/StorageS3.h @@ -20,12 +20,8 @@ #include #include #include -#include -#include #include #include -#include -#include namespace Aws::S3 { From aef4154b8b51f2ce34c9c11de7727b232cbf2a42 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 31 Jan 2023 12:32:26 -0300 Subject: [PATCH 089/717] add and improve tests, mix of join, distributed and subqueries --- ...02420_force_select_final_setting.reference | 144 +++++++++++++++--- .../02420_force_select_final_setting.sql | 100 ++++++++++-- 2 files changed, 211 insertions(+), 33 deletions(-) diff --git a/tests/queries/0_stateless/02420_force_select_final_setting.reference b/tests/queries/0_stateless/02420_force_select_final_setting.reference index 1fe561cd267..7194b90f775 100644 --- a/tests/queries/0_stateless/02420_force_select_final_setting.reference +++ b/tests/queries/0_stateless/02420_force_select_final_setting.reference @@ -40,20 +40,130 @@ create view nv_regular_mt_table AS SELECT * FROM mv_regular_mt_table; set force_select_final=1; select count() from nv_regular_mt_table; 2 --- join on mix of tables that support / do not support select final -create table if not exists left_table (x String) engine=ReplacingMergeTree() ORDER BY x; -create table if not exists middle_table (x String) engine=MergeTree() ORDER BY x; -create table if not exists right_table (x String) engine=ReplacingMergeTree() ORDER BY x; -insert into left_table values ('abc'); -insert into left_table values ('abc'); -insert into left_table values ('abc'); -insert into middle_table values ('abc'); -insert into middle_table values ('abc'); -insert into right_table values ('abc'); -insert into right_table values ('abc'); -insert into right_table values ('abc'); --- Expected output is 2 because middle table does not support final -select count() from left_table - inner join middle_table on left_table.x = middle_table.x - inner join right_table on middle_table.x = right_table.x; -2 +-- join on mix of tables that support / do not support select final with explain +create table if not exists left_table (id UInt64, val_left String) engine=ReplacingMergeTree() ORDER BY id; +create table if not exists middle_table (id UInt64, val_middle String) engine=MergeTree() ORDER BY id; +create table if not exists right_table (id UInt64, val_right String) engine=ReplacingMergeTree() ORDER BY id; +insert into left_table values (1,'a'); +insert into left_table values (1,'b'); +insert into left_table values (1,'c'); +insert into middle_table values (1,'a'); +insert into middle_table values (1,'b'); +insert into right_table values (1,'a'); +insert into right_table values (1,'b'); +insert into right_table values (1,'c'); +-- expected output +-- 1 c a c +-- 1 c b c +select left_table.id,val_left, val_middle, val_right from left_table + inner join middle_table on left_table.id = middle_table.id + inner join right_table on middle_table.id = right_table.id +ORDER BY left_table.id, val_left, val_middle, val_right; +1 c a c +1 c b c +explain syntax select left_table.id,val_left, val_middle, val_right from left_table + inner join middle_table on left_table.id = middle_table.id + inner join right_table on middle_table.id = right_table.id +ORDER BY left_table.id, val_left, val_middle, val_right; +SELECT + `--left_table.id` AS `left_table.id`, + val_left, + val_middle, + val_right +FROM +( + SELECT + val_left, + id AS `--left_table.id`, + val_middle, + middle_table.id AS `--middle_table.id` + FROM left_table + FINAL + ALL INNER JOIN + ( + SELECT + val_middle, + id + FROM middle_table + ) AS middle_table ON `--left_table.id` = `--middle_table.id` +) AS `--.s` +ALL INNER JOIN +( + SELECT + val_right, + id + FROM right_table + FINAL +) AS right_table ON `--middle_table.id` = id +ORDER BY + `--left_table.id` ASC, + val_left ASC, + val_middle ASC, + val_right ASC +-- extra: same with subquery +select left_table.id,val_left, val_middle, val_right from left_table + inner join middle_table on left_table.id = middle_table.id + inner join (SELECT * FROM right_table WHERE id = 1) r on middle_table.id = r.id +ORDER BY left_table.id, val_left, val_middle, val_right; +1 c a c +1 c b c +-- distributed tables +drop table if exists left_table; +drop table if exists middle_table; +drop table if exists right_table; +create table if not exists left_table (id UInt64, val_left String) engine=ReplacingMergeTree() ORDER BY id; +create table if not exists middle_table (id UInt64, val_middle String) engine=MergeTree() ORDER BY id; +create table if not exists right_table_local (id UInt64, val_right String) engine=ReplacingMergeTree() ORDER BY id; +create table if not exists right_table engine=Distributed('test_shard_localhost', currentDatabase(), right_table_local) AS right_table_local; +insert into left_table values (1,'a'); +insert into left_table values (1,'b'); +insert into left_table values (1,'c'); +insert into middle_table values (1,'a'); +insert into middle_table values (1,'b'); +insert into right_table_local values (1,'a'); +insert into right_table_local values (1,'b'); +insert into right_table_local values (1,'c'); +SET prefer_localhost_replica=0; +-- expected output: +-- 1 c 1 a 1 c +-- 1 c 1 b 1 c +select left_table.*,middle_table.*, right_table.* from left_table + inner join middle_table on left_table.id = middle_table.id + inner join right_table on middle_table.id = right_table.id +ORDER BY left_table.id, val_left, val_middle, val_right; +1 c 1 a 1 c +1 c 1 b 1 c +SET prefer_localhost_replica=1; +-- expected output: +-- 1 c 1 a 1 c +-- 1 c 1 b 1 c +select left_table.*,middle_table.*, right_table.* from left_table + inner join middle_table on left_table.id = middle_table.id + inner join right_table on middle_table.id = right_table.id +ORDER BY left_table.id, val_left, val_middle, val_right; +1 c 1 a 1 c +1 c 1 b 1 c +-- Quite exotic with Merge engine +DROP TABLE IF EXISTS table_to_merge_a; +DROP TABLE IF EXISTS table_to_merge_b; +DROP TABLE IF EXISTS table_to_merge_c; +DROP TABLE IF EXISTS merge_table; +create table if not exists table_to_merge_a (id UInt64, val String) engine=ReplacingMergeTree() ORDER BY id; +create table if not exists table_to_merge_b (id UInt64, val String) engine=MergeTree() ORDER BY id; +create table if not exists table_to_merge_c (id UInt64, val String) engine=ReplacingMergeTree() ORDER BY id; +CREATE TABLE merge_table Engine=Merge(currentDatabase(), '^(table_to_merge_[a-z])$') AS table_to_merge_a; +insert into table_to_merge_a values (1,'a'); +insert into table_to_merge_a values (1,'b'); +insert into table_to_merge_a values (1,'c'); +insert into table_to_merge_b values (2,'a'); +insert into table_to_merge_b values (2,'b'); +insert into table_to_merge_c values (3,'a'); +insert into table_to_merge_c values (3,'b'); +insert into table_to_merge_c values (3,'c'); +-- expected output: +-- 1 c, 2 a, 2 b, 3 c +SELECT * FROM merge_table ORDER BY id, val; +1 c +2 a +2 b +3 c diff --git a/tests/queries/0_stateless/02420_force_select_final_setting.sql b/tests/queries/0_stateless/02420_force_select_final_setting.sql index c074d1a8926..c60bc3c7f53 100644 --- a/tests/queries/0_stateless/02420_force_select_final_setting.sql +++ b/tests/queries/0_stateless/02420_force_select_final_setting.sql @@ -47,23 +47,91 @@ create view nv_regular_mt_table AS SELECT * FROM mv_regular_mt_table; set force_select_final=1; select count() from nv_regular_mt_table; --- join on mix of tables that support / do not support select final -create table if not exists left_table (x String) engine=ReplacingMergeTree() ORDER BY x; -create table if not exists middle_table (x String) engine=MergeTree() ORDER BY x; -create table if not exists right_table (x String) engine=ReplacingMergeTree() ORDER BY x; +-- join on mix of tables that support / do not support select final with explain +create table if not exists left_table (id UInt64, val_left String) engine=ReplacingMergeTree() ORDER BY id; +create table if not exists middle_table (id UInt64, val_middle String) engine=MergeTree() ORDER BY id; +create table if not exists right_table (id UInt64, val_right String) engine=ReplacingMergeTree() ORDER BY id; +insert into left_table values (1,'a'); +insert into left_table values (1,'b'); +insert into left_table values (1,'c'); +insert into middle_table values (1,'a'); +insert into middle_table values (1,'b'); +insert into right_table values (1,'a'); +insert into right_table values (1,'b'); +insert into right_table values (1,'c'); +-- expected output +-- 1 c a c +-- 1 c b c +select left_table.id,val_left, val_middle, val_right from left_table + inner join middle_table on left_table.id = middle_table.id + inner join right_table on middle_table.id = right_table.id +ORDER BY left_table.id, val_left, val_middle, val_right; -insert into left_table values ('abc'); -insert into left_table values ('abc'); -insert into left_table values ('abc'); +explain syntax select left_table.id,val_left, val_middle, val_right from left_table + inner join middle_table on left_table.id = middle_table.id + inner join right_table on middle_table.id = right_table.id +ORDER BY left_table.id, val_left, val_middle, val_right; -insert into middle_table values ('abc'); -insert into middle_table values ('abc'); +-- extra: same with subquery +select left_table.id,val_left, val_middle, val_right from left_table + inner join middle_table on left_table.id = middle_table.id + inner join (SELECT * FROM right_table WHERE id = 1) r on middle_table.id = r.id +ORDER BY left_table.id, val_left, val_middle, val_right; -insert into right_table values ('abc'); -insert into right_table values ('abc'); -insert into right_table values ('abc'); +-- distributed tables +drop table if exists left_table; +drop table if exists middle_table; +drop table if exists right_table; +create table if not exists left_table (id UInt64, val_left String) engine=ReplacingMergeTree() ORDER BY id; +create table if not exists middle_table (id UInt64, val_middle String) engine=MergeTree() ORDER BY id; +create table if not exists right_table_local (id UInt64, val_right String) engine=ReplacingMergeTree() ORDER BY id; +create table if not exists right_table engine=Distributed('test_shard_localhost', currentDatabase(), right_table_local) AS right_table_local; +insert into left_table values (1,'a'); +insert into left_table values (1,'b'); +insert into left_table values (1,'c'); +insert into middle_table values (1,'a'); +insert into middle_table values (1,'b'); +insert into right_table_local values (1,'a'); +insert into right_table_local values (1,'b'); +insert into right_table_local values (1,'c'); +SET prefer_localhost_replica=0; +-- expected output: +-- 1 c 1 a 1 c +-- 1 c 1 b 1 c +select left_table.*,middle_table.*, right_table.* from left_table + inner join middle_table on left_table.id = middle_table.id + inner join right_table on middle_table.id = right_table.id +ORDER BY left_table.id, val_left, val_middle, val_right; --- Expected output is 2 because middle table does not support final -select count() from left_table - inner join middle_table on left_table.x = middle_table.x - inner join right_table on middle_table.x = right_table.x; +SET prefer_localhost_replica=1; +-- expected output: +-- 1 c 1 a 1 c +-- 1 c 1 b 1 c +select left_table.*,middle_table.*, right_table.* from left_table + inner join middle_table on left_table.id = middle_table.id + inner join right_table on middle_table.id = right_table.id +ORDER BY left_table.id, val_left, val_middle, val_right; + +-- Quite exotic with Merge engine +DROP TABLE IF EXISTS table_to_merge_a; +DROP TABLE IF EXISTS table_to_merge_b; +DROP TABLE IF EXISTS table_to_merge_c; +DROP TABLE IF EXISTS merge_table; + +create table if not exists table_to_merge_a (id UInt64, val String) engine=ReplacingMergeTree() ORDER BY id; +create table if not exists table_to_merge_b (id UInt64, val String) engine=MergeTree() ORDER BY id; +create table if not exists table_to_merge_c (id UInt64, val String) engine=ReplacingMergeTree() ORDER BY id; +CREATE TABLE merge_table Engine=Merge(currentDatabase(), '^(table_to_merge_[a-z])$') AS table_to_merge_a; + +insert into table_to_merge_a values (1,'a'); +insert into table_to_merge_a values (1,'b'); +insert into table_to_merge_a values (1,'c'); +insert into table_to_merge_b values (2,'a'); +insert into table_to_merge_b values (2,'b'); +insert into table_to_merge_c values (3,'a'); +insert into table_to_merge_c values (3,'b'); +insert into table_to_merge_c values (3,'c'); + +-- expected output: +-- 1 c, 2 a, 2 b, 3 c +SELECT * FROM merge_table ORDER BY id, val; From 2c15ede37c422ff995a227ae64264610fa725e1b Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 31 Jan 2023 15:21:00 -0300 Subject: [PATCH 090/717] Change setting name from force_select_final to final --- src/Core/Settings.h | 2 +- src/Interpreters/InterpreterSelectQuery.cpp | 8 ++++---- src/Interpreters/InterpreterSelectQuery.h | 2 +- .../02420_force_select_final_setting.reference | 16 ++++++++-------- .../02420_force_select_final_setting.sql | 16 ++++++++-------- ...force_select_final_setting_analyzer.reference | 0 ...02420_force_select_final_setting_analyzer.sql | 0 7 files changed, 22 insertions(+), 22 deletions(-) create mode 100644 tests/queries/0_stateless/02420_force_select_final_setting_analyzer.reference create mode 100644 tests/queries/0_stateless/02420_force_select_final_setting_analyzer.sql diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 3dfbfa292d8..413d128564a 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -275,7 +275,7 @@ class IColumn; M(Milliseconds, stream_flush_interval_ms, 7500, "Timeout for flushing data from streaming storages.", 0) \ M(Milliseconds, stream_poll_timeout_ms, 500, "Timeout for polling data from/to streaming storages.", 0) \ \ - M(Bool, force_select_final, false, "Query with the FINAL modifier by default", 0) \ + M(Bool, final, false, "Query with the FINAL modifier by default", 0) \ \ /** Settings for testing hedged requests */ \ M(Milliseconds, sleep_in_send_tables_status_ms, 0, "Time to sleep in sending tables status response in TCPHandler", 0) \ diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 12b6f6f3297..ae4ccc34f78 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -503,7 +503,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( query_info.additional_filter_ast = parseAdditionalFilterConditionForTable( settings.additional_table_filters, joined_tables.tablesWithColumns().front().table, *context); - if (forceSelectFinalOnSelectQuery(query)) + if (autoFinalOnQuery(query)) { query.setFinal(); } @@ -3007,14 +3007,14 @@ void InterpreterSelectQuery::ignoreWithTotals() getSelectQuery().group_by_with_totals = false; } -bool InterpreterSelectQuery::forceSelectFinalOnSelectQuery(ASTSelectQuery & query) +bool InterpreterSelectQuery::autoFinalOnQuery(ASTSelectQuery & query) { // query.tables() is required because not all queries have tables in it, it could be a function. - bool is_force_select_final_setting_on = context->getSettingsRef().force_select_final; + bool is_auto_final_setting_on = context->getSettingsRef().final; bool is_final_supported = storage && storage->supportsFinal() && !storage->isRemote() && query.tables(); bool is_query_already_final = query.final(); - return is_force_select_final_setting_on && !is_query_already_final && is_final_supported; + return is_auto_final_setting_on && !is_query_already_final && is_final_supported; } void InterpreterSelectQuery::initSettings() diff --git a/src/Interpreters/InterpreterSelectQuery.h b/src/Interpreters/InterpreterSelectQuery.h index f679051532c..88db1360683 100644 --- a/src/Interpreters/InterpreterSelectQuery.h +++ b/src/Interpreters/InterpreterSelectQuery.h @@ -194,7 +194,7 @@ private: void executeDistinct(QueryPlan & query_plan, bool before_order, Names columns, bool pre_distinct); void executeExtremes(QueryPlan & query_plan); void executeSubqueriesInSetsAndJoins(QueryPlan & query_plan); - bool forceSelectFinalOnSelectQuery(ASTSelectQuery & select_query); + bool autoFinalOnQuery(ASTSelectQuery & select_query); enum class Modificator { diff --git a/tests/queries/0_stateless/02420_force_select_final_setting.reference b/tests/queries/0_stateless/02420_force_select_final_setting.reference index 7194b90f775..5b9780e27cd 100644 --- a/tests/queries/0_stateless/02420_force_select_final_setting.reference +++ b/tests/queries/0_stateless/02420_force_select_final_setting.reference @@ -4,11 +4,11 @@ SYSTEM STOP MERGES tbl; create table if not exists replacing_mt (x String) engine=ReplacingMergeTree() ORDER BY x; insert into replacing_mt values ('abc'); insert into replacing_mt values ('abc'); --- expected output is 2 because force_select_final is turned off +-- expected output is 2 because final is turned off select count() from replacing_mt; 2 -set force_select_final = 1; --- expected output is 1 because force_select_final is turned on +set final = 1; +-- expected output is 1 because final is turned on select count() from replacing_mt; 1 -- JOIN test cases @@ -18,16 +18,16 @@ insert into lhs values ('abc'); insert into lhs values ('abc'); insert into rhs values ('abc'); insert into rhs values ('abc'); -set force_select_final = 0; +set final = 0; -- expected output is 4 because select_final == 0 select count() from lhs inner join rhs on lhs.x = rhs.x; 4 -set force_select_final = 1; --- expected output is 1 because force_select_final == 1 +set final = 1; +-- expected output is 1 because final == 1 select count() from lhs inner join rhs on lhs.x = rhs.x; 1 -- regular non final table -set force_select_final = 1; +set final = 1; create table if not exists regular_mt_table (x String) engine=MergeTree() ORDER BY x; insert into regular_mt_table values ('abc'); insert into regular_mt_table values ('abc'); @@ -37,7 +37,7 @@ select count() from regular_mt_table; -- view test create materialized VIEW mv_regular_mt_table TO regular_mt_table AS SELECT * FROM regular_mt_table; create view nv_regular_mt_table AS SELECT * FROM mv_regular_mt_table; -set force_select_final=1; +set final=1; select count() from nv_regular_mt_table; 2 -- join on mix of tables that support / do not support select final with explain diff --git a/tests/queries/0_stateless/02420_force_select_final_setting.sql b/tests/queries/0_stateless/02420_force_select_final_setting.sql index c60bc3c7f53..d8ca24da3da 100644 --- a/tests/queries/0_stateless/02420_force_select_final_setting.sql +++ b/tests/queries/0_stateless/02420_force_select_final_setting.sql @@ -7,11 +7,11 @@ create table if not exists replacing_mt (x String) engine=ReplacingMergeTree() O insert into replacing_mt values ('abc'); insert into replacing_mt values ('abc'); --- expected output is 2 because force_select_final is turned off +-- expected output is 2 because final is turned off select count() from replacing_mt; -set force_select_final = 1; --- expected output is 1 because force_select_final is turned on +set final = 1; +-- expected output is 1 because final is turned on select count() from replacing_mt; -- JOIN test cases @@ -24,16 +24,16 @@ insert into lhs values ('abc'); insert into rhs values ('abc'); insert into rhs values ('abc'); -set force_select_final = 0; +set final = 0; -- expected output is 4 because select_final == 0 select count() from lhs inner join rhs on lhs.x = rhs.x; -set force_select_final = 1; --- expected output is 1 because force_select_final == 1 +set final = 1; +-- expected output is 1 because final == 1 select count() from lhs inner join rhs on lhs.x = rhs.x; -- regular non final table -set force_select_final = 1; +set final = 1; create table if not exists regular_mt_table (x String) engine=MergeTree() ORDER BY x; insert into regular_mt_table values ('abc'); insert into regular_mt_table values ('abc'); @@ -44,7 +44,7 @@ select count() from regular_mt_table; create materialized VIEW mv_regular_mt_table TO regular_mt_table AS SELECT * FROM regular_mt_table; create view nv_regular_mt_table AS SELECT * FROM mv_regular_mt_table; -set force_select_final=1; +set final=1; select count() from nv_regular_mt_table; -- join on mix of tables that support / do not support select final with explain diff --git a/tests/queries/0_stateless/02420_force_select_final_setting_analyzer.reference b/tests/queries/0_stateless/02420_force_select_final_setting_analyzer.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02420_force_select_final_setting_analyzer.sql b/tests/queries/0_stateless/02420_force_select_final_setting_analyzer.sql new file mode 100644 index 00000000000..e69de29bb2d From b9e46c02ddf92c12bf3211f4a23abe9975a63a09 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 31 Jan 2023 15:24:06 -0300 Subject: [PATCH 091/717] rename tests --- ...lect_final_setting.reference => 02420_final_setting.reference} | 0 ...420_force_select_final_setting.sql => 02420_final_setting.sql} | 0 .../02420_force_select_final_setting_analyzer.reference | 0 .../0_stateless/02420_force_select_final_setting_analyzer.sql | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename tests/queries/0_stateless/{02420_force_select_final_setting.reference => 02420_final_setting.reference} (100%) rename tests/queries/0_stateless/{02420_force_select_final_setting.sql => 02420_final_setting.sql} (100%) delete mode 100644 tests/queries/0_stateless/02420_force_select_final_setting_analyzer.reference delete mode 100644 tests/queries/0_stateless/02420_force_select_final_setting_analyzer.sql diff --git a/tests/queries/0_stateless/02420_force_select_final_setting.reference b/tests/queries/0_stateless/02420_final_setting.reference similarity index 100% rename from tests/queries/0_stateless/02420_force_select_final_setting.reference rename to tests/queries/0_stateless/02420_final_setting.reference diff --git a/tests/queries/0_stateless/02420_force_select_final_setting.sql b/tests/queries/0_stateless/02420_final_setting.sql similarity index 100% rename from tests/queries/0_stateless/02420_force_select_final_setting.sql rename to tests/queries/0_stateless/02420_final_setting.sql diff --git a/tests/queries/0_stateless/02420_force_select_final_setting_analyzer.reference b/tests/queries/0_stateless/02420_force_select_final_setting_analyzer.reference deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/queries/0_stateless/02420_force_select_final_setting_analyzer.sql b/tests/queries/0_stateless/02420_force_select_final_setting_analyzer.sql deleted file mode 100644 index e69de29bb2d..00000000000 From a8489fbef3ab7a7a27c4474f1b9bf37203f14adb Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 31 Jan 2023 15:29:30 -0300 Subject: [PATCH 092/717] improve docs --- src/Core/Settings.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 413d128564a..b2af210124d 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -275,7 +275,7 @@ class IColumn; M(Milliseconds, stream_flush_interval_ms, 7500, "Timeout for flushing data from streaming storages.", 0) \ M(Milliseconds, stream_poll_timeout_ms, 500, "Timeout for polling data from/to streaming storages.", 0) \ \ - M(Bool, final, false, "Query with the FINAL modifier by default", 0) \ + M(Bool, final, false, "Query with the FINAL modifier by default. If the engine does not support final, it does not have any effect. On queries with multiple tables final is applied only on those that support it. It also works on distributed tables", 0) \ \ /** Settings for testing hedged requests */ \ M(Milliseconds, sleep_in_send_tables_status_ms, 0, "Time to sleep in sending tables status response in TCPHandler", 0) \ From fc71acc76829252e7c41ccd629d4c441c79c2b19 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Wed, 1 Feb 2023 13:06:34 +0000 Subject: [PATCH 093/717] Fix DISTINCT with intersect/except --- .../Optimizations/removeRedundantDistinct.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp index 9cf5c27006b..dcac72dc00d 100644 --- a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp +++ b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -44,13 +45,11 @@ size_t tryRemoveRedundantDistinct(QueryPlan::Node * parent_node, QueryPlan::Node return 0; /// check if it is preliminary distinct node - QueryPlan::Node * distinct_node = nullptr; - DistinctStep * distinct_step = typeid_cast(parent_node->children.front()->step.get()); + QueryPlan::Node * distinct_node = parent_node->children.front(); + DistinctStep * distinct_step = typeid_cast(distinct_node->step.get()); if (!distinct_step) return 0; - distinct_node = parent_node->children.front(); - std::vector dag_stack; const DistinctStep * inner_distinct_step = nullptr; QueryPlan::Node * node = distinct_node; @@ -59,7 +58,8 @@ size_t tryRemoveRedundantDistinct(QueryPlan::Node * parent_node, QueryPlan::Node const IQueryPlanStep * current_step = node->step.get(); /// don't try to remove DISTINCT after union or join - if (typeid_cast(current_step) || typeid_cast(current_step)) + if (typeid_cast(current_step) || typeid_cast(current_step) + || typeid_cast(current_step)) break; if (const auto * const expr = typeid_cast(current_step); expr) From 8067f98103e6c366ecd5a976c85cfa69d199b715 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 1 Feb 2023 18:19:09 +0100 Subject: [PATCH 094/717] Add test which reproduce rename bug --- .../02538_alter_rename_sequence.reference | 0 .../02538_alter_rename_sequence.sql | 27 +++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 tests/queries/0_stateless/02538_alter_rename_sequence.reference create mode 100644 tests/queries/0_stateless/02538_alter_rename_sequence.sql diff --git a/tests/queries/0_stateless/02538_alter_rename_sequence.reference b/tests/queries/0_stateless/02538_alter_rename_sequence.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02538_alter_rename_sequence.sql b/tests/queries/0_stateless/02538_alter_rename_sequence.sql new file mode 100644 index 00000000000..efdb1d469eb --- /dev/null +++ b/tests/queries/0_stateless/02538_alter_rename_sequence.sql @@ -0,0 +1,27 @@ +DROP TABLE IF EXISTS wrong_metadata; + +CREATE TABLE wrong_metadata( + column1 UInt64, + column2 UInt64, + column3 UInt64 +) +ENGINE ReplicatedMergeTree('/test/tables/wrong_metadata', '1') +ORDER BY tuple(); + +INSERT INTO wrong_metadata VALUES (1, 2, 3); + +SYSTEM STOP REPLICATION QUEUES wrong_metadata; + +ALTER TABLE wrong_metadata RENAME COLUMN column1 TO column1_renamed SETTINGS replication_alter_partitions_sync = 0; + +INSERT INTO wrong_metadata VALUES (4, 5, 6); + +SYSTEM START REPLICATION QUEUES wrong_metadata; + +SYSTEM SYNC REPLICA wrong_metadata; + +ALTER TABLE wrong_metadata RENAME COLUMN column2 to column2_renamed SETTINGS replication_alter_partitions_sync = 2; + +SELECT * FROM wrong_metadata ORDER BY column1_renamed FORMAT JSONEachRow; + +DROP TABLE IF EXISTS wrong_metadata; From 7e7b1829bea49ab271158c7e87270b5e1e82e3cf Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Wed, 1 Feb 2023 14:29:59 -0300 Subject: [PATCH 095/717] temp --- src/Interpreters/AutoFinalOnQueryVisitor.cpp | 37 ++++++++++++++++++++ src/Interpreters/AutoFinalOnQueryVisitor.h | 31 ++++++++++++++++ src/Interpreters/InterpreterSelectQuery.cpp | 13 ++++--- 3 files changed, 77 insertions(+), 4 deletions(-) create mode 100644 src/Interpreters/AutoFinalOnQueryVisitor.cpp create mode 100644 src/Interpreters/AutoFinalOnQueryVisitor.h diff --git a/src/Interpreters/AutoFinalOnQueryVisitor.cpp b/src/Interpreters/AutoFinalOnQueryVisitor.cpp new file mode 100644 index 00000000000..f2b3eb147c1 --- /dev/null +++ b/src/Interpreters/AutoFinalOnQueryVisitor.cpp @@ -0,0 +1,37 @@ +#include "AutoFinalOnQueryVisitor.h" +#include +#include + +namespace DB +{ + +void AutoFinalOnQuery::visit(ASTPtr & query, Data & data) +{ + if (auto * select_query = query->as()) + visit(*select_query, data.storage, data.context); +} + +void AutoFinalOnQuery::visit(ASTSelectQuery & query, StoragePtr storage, ContextPtr context) +{ + if (autoFinalOnQuery(query, storage, context)) + { + query.setFinal(); + } +} + +bool AutoFinalOnQuery::needChildVisit(ASTPtr &, const ASTPtr &) +{ + return true; +} + +bool AutoFinalOnQuery::autoFinalOnQuery(ASTSelectQuery & query, StoragePtr storage, ContextPtr context) +{ + // query.tables() is required because not all queries have tables in it, it could be a function. + bool is_auto_final_setting_on = context->getSettingsRef().final; + bool is_final_supported = storage && storage->supportsFinal() && !storage->isRemote() && query.tables(); + bool is_query_already_final = query.final(); + + return is_auto_final_setting_on && !is_query_already_final && is_final_supported; +} + +} diff --git a/src/Interpreters/AutoFinalOnQueryVisitor.h b/src/Interpreters/AutoFinalOnQueryVisitor.h new file mode 100644 index 00000000000..850fb9fc0e4 --- /dev/null +++ b/src/Interpreters/AutoFinalOnQueryVisitor.h @@ -0,0 +1,31 @@ +#pragma once + +#include +#include +#include +#include + +namespace DB +{ + +class AutoFinalOnQuery +{ +public: + struct Data + { + StoragePtr storage; + ContextPtr context; + }; + + static bool needChildVisit(ASTPtr &, const ASTPtr &); + static void visit(ASTPtr & query, Data & data); + +private: + static void visit(ASTSelectQuery & select, StoragePtr storage, ContextPtr context); + static bool autoFinalOnQuery(ASTSelectQuery & select_query, StoragePtr storage, ContextPtr context); + +}; + +using AutoFinalOnQueryVisitor = InDepthNodeVisitor; + +} diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index ae4ccc34f78..8114b7a5375 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -38,6 +38,7 @@ #include #include #include +#include #include #include @@ -503,10 +504,14 @@ InterpreterSelectQuery::InterpreterSelectQuery( query_info.additional_filter_ast = parseAdditionalFilterConditionForTable( settings.additional_table_filters, joined_tables.tablesWithColumns().front().table, *context); - if (autoFinalOnQuery(query)) - { - query.setFinal(); - } + AutoFinalOnQuery::Data abc{storage, context}; + + AutoFinalOnQueryVisitor(abc).visit(query_ptr); + +// if (autoFinalOnQuery(query)) +// { +// query.setFinal(); +// } auto analyze = [&] (bool try_move_to_prewhere) { From 8f873d568256db6460decce5feda5eabacac1ded Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Wed, 1 Feb 2023 21:13:44 +0000 Subject: [PATCH 096/717] Disable optimize_duplicate_order_by_and_distinct + tests --- src/Core/Settings.h | 2 +- .../QueryPlanOptimizationSettings.cpp | 2 +- .../Optimizations/removeRedundantDistinct.cpp | 43 ++++++++++--------- ...istinct_optimize_for_distributed_table.sql | 4 +- .../02500_remove_redundant_distinct.reference | 22 ++++++++++ .../02500_remove_redundant_distinct.sh | 13 ++++++ 6 files changed, 62 insertions(+), 24 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 146b54bc575..025736a961a 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -491,7 +491,7 @@ class IColumn; M(Bool, convert_query_to_cnf, false, "Convert SELECT query to CNF", 0) \ M(Bool, optimize_or_like_chain, false, "Optimize multiple OR LIKE into multiMatchAny. This optimization should not be enabled by default, because it defies index analysis in some cases.", 0) \ M(Bool, optimize_arithmetic_operations_in_aggregate_functions, true, "Move arithmetic operations out of aggregation functions", 0) \ - M(Bool, optimize_duplicate_order_by_and_distinct, true, "Remove duplicate ORDER BY and DISTINCT if it's possible", 0) \ + M(Bool, optimize_duplicate_order_by_and_distinct, false, "Remove duplicate ORDER BY and DISTINCT if it's possible", 0) \ M(Bool, optimize_redundant_functions_in_order_by, true, "Remove functions from ORDER BY if its argument is also in ORDER BY", 0) \ M(Bool, optimize_if_chain_to_multiif, false, "Replace if(cond1, then1, if(cond2, ...)) chains to multiIf. Currently it's not beneficial for numeric types.", 0) \ M(Bool, optimize_multiif_to_if, true, "Replace 'multiIf' with only one condition to 'if'.", 0) \ diff --git a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp index 00f9240e692..2f3708e7b74 100644 --- a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp +++ b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp @@ -15,7 +15,7 @@ QueryPlanOptimizationSettings QueryPlanOptimizationSettings::fromSettings(const settings.read_in_order = from.optimize_read_in_order && from.query_plan_read_in_order; settings.aggregation_in_order = from.optimize_aggregation_in_order && from.query_plan_aggregation_in_order; settings.remove_redundant_sorting = from.query_plan_remove_redundant_sorting; - settings.remove_redundant_distinct = from.query_plan_remove_redundant_distinct && !from.distributed_group_by_no_merge; + settings.remove_redundant_distinct = from.query_plan_remove_redundant_distinct; return settings; } diff --git a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp index dcac72dc00d..3975f7efb3a 100644 --- a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp +++ b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp @@ -14,7 +14,6 @@ namespace DB::QueryPlanOptimizations { constexpr bool debug_logging_enabled = true; - void logDebug(const String & prefix, const String & message) { if constexpr (debug_logging_enabled) @@ -88,33 +87,37 @@ size_t tryRemoveRedundantDistinct(QueryPlan::Node * parent_node, QueryPlan::Node if (distinct_columns.size() != inner_distinct_columns.size()) return 0; - /// build actions DAG to compare DISTINCT columns ActionsDAGPtr path_actions; if (!dag_stack.empty()) { + /// build actions DAG to find original column names path_actions = dag_stack.back(); dag_stack.pop_back(); + while (!dag_stack.empty()) + { + ActionsDAGPtr clone = dag_stack.back()->clone(); + dag_stack.pop_back(); + path_actions->mergeInplace(std::move(*clone)); + } + + /// compare columns of two DISTINCTs + for (const auto & column : distinct_columns) + { + const auto * alias_node = path_actions->getOriginalNodeForOutputAlias(String(column)); + if (!alias_node) + return 0; + + auto it = inner_distinct_columns.find(alias_node->result_name); + if (it == inner_distinct_columns.end()) + return 0; + + inner_distinct_columns.erase(it); + } } - while (!dag_stack.empty()) + else { - ActionsDAGPtr clone = dag_stack.back()->clone(); - dag_stack.pop_back(); - path_actions->mergeInplace(std::move(*clone)); - } - - logDebug("mergedDAG\n", path_actions->dumpDAG()); - - for (const auto & column : distinct_columns) - { - const auto * alias_node = path_actions->getOriginalNodeForOutputAlias(String(column)); - if (!alias_node) + if (distinct_columns != inner_distinct_columns) return 0; - - auto it = inner_distinct_columns.find(alias_node->result_name); - if (it == inner_distinct_columns.end()) - return 0; - - inner_distinct_columns.erase(it); } /// remove current distinct diff --git a/tests/queries/0_stateless/01306_disable_duplicate_order_by_and_distinct_optimize_for_distributed_table.sql b/tests/queries/0_stateless/01306_disable_duplicate_order_by_and_distinct_optimize_for_distributed_table.sql index 06aa8672831..8ef1273c855 100644 --- a/tests/queries/0_stateless/01306_disable_duplicate_order_by_and_distinct_optimize_for_distributed_table.sql +++ b/tests/queries/0_stateless/01306_disable_duplicate_order_by_and_distinct_optimize_for_distributed_table.sql @@ -1,7 +1,7 @@ -- Tags: distributed -set query_plan_remove_redundant_distinct = 0; -set optimize_duplicate_order_by_and_distinct = 1; +set query_plan_remove_redundant_distinct = 1; +set optimize_duplicate_order_by_and_distinct = 0; SET distributed_group_by_no_merge = 0; SELECT DISTINCT number diff --git a/tests/queries/0_stateless/02500_remove_redundant_distinct.reference b/tests/queries/0_stateless/02500_remove_redundant_distinct.reference index e9d8c63771a..30da5d536f9 100644 --- a/tests/queries/0_stateless/02500_remove_redundant_distinct.reference +++ b/tests/queries/0_stateless/02500_remove_redundant_distinct.reference @@ -117,3 +117,25 @@ Expression ((Project names + (Projection + (Change column names to column identi 0 0 1 2 2 4 +-- DISTINCT duplicates with constant columns +-- query +SELECT DISTINCT 2, a, b +FROM +( + SELECT DISTINCT a, b + FROM + ( + SELECT DISTINCT 1, number as a, 2*number as b + FROM numbers(3) + ) +) +-- explain +Expression ((Project names + (Projection + (Change column names to column identifiers + (Project names + (Projection + (Change column names to column identifiers + Project names))))))) + Distinct (DISTINCT) + Distinct (Preliminary DISTINCT) + Expression ((Projection + Change column names to column identifiers)) + ReadFromStorage (SystemNumbers) +-- execute +2 0 0 +2 1 2 +2 2 4 diff --git a/tests/queries/0_stateless/02500_remove_redundant_distinct.sh b/tests/queries/0_stateless/02500_remove_redundant_distinct.sh index bcbe6d7a651..94d99242ead 100755 --- a/tests/queries/0_stateless/02500_remove_redundant_distinct.sh +++ b/tests/queries/0_stateless/02500_remove_redundant_distinct.sh @@ -70,3 +70,16 @@ FROM ) )" run_query "$query" + +echo "-- DISTINCT duplicates with constant columns" +query="SELECT DISTINCT 2, a, b +FROM +( + SELECT DISTINCT a, b + FROM + ( + SELECT DISTINCT 1, number as a, 2*number as b + FROM numbers(3) + ) +)" +run_query "$query" From 4fbbca075b61d9dfc704476d128904b832c8324c Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Wed, 1 Feb 2023 23:01:02 +0000 Subject: [PATCH 097/717] review fixes --- .../QueryPlan/Optimizations/useDataParallelAggregation.cpp | 2 +- src/Processors/QueryPlan/ReadFromMergeTree.cpp | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp b/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp index b0cb3eaac0c..681b8f78c3a 100644 --- a/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp +++ b/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp @@ -191,7 +191,7 @@ size_t tryAggregatePartitionsIndependently(QueryPlan::Node * node, QueryPlan::No const auto * expression_node = node->children.front(); const auto * expression_step = typeid_cast(expression_node->step.get()); - if (expression_node->children.size() != 1 || !expression_step) + if (!expression_step) return 0; auto * maybe_reading_step = expression_node->children.front()->step.get(); diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index e0a12166d40..44e81af259b 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -49,7 +49,8 @@ template size_t countPartitions(const Container & parts, Getter get_partition_id) { if (parts.empty()) - return 1; + return 0; + String cur_partition_id = get_partition_id(parts[0]); size_t unique_partitions = 1; for (size_t i = 1; i < parts.size(); ++i) @@ -1370,7 +1371,7 @@ Pipe ReadFromMergeTree::groupStreamsByPartition(AnalysisResult & result, Actions if (parts_with_ranges.empty()) return {}; - const size_t partitions_cnt = countPartitions(parts_with_ranges); + const size_t partitions_cnt = std::max(countPartitions(parts_with_ranges), 1); const size_t partitions_per_stream = std::max(1, partitions_cnt / requested_num_streams); const size_t num_streams = std::max(1, requested_num_streams / partitions_cnt); From 996b833faabe107308f0a30dd0a8b74c334dea6d Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Thu, 2 Feb 2023 11:14:26 +0000 Subject: [PATCH 098/717] Some polishing --- src/Interpreters/ActionsDAG.cpp | 2 ++ .../Optimizations/removeRedundantDistinct.cpp | 20 ++++++++++--------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index d437bc73b9a..8bc430ee825 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -2262,6 +2262,7 @@ ActionsDAGPtr ActionsDAG::buildFilterActionsDAG( const ActionsDAG::Node * ActionsDAG::getOriginalNodeForOutputAlias(const String & output_name) { + /// find alias in output const Node * output_alias = nullptr; for (const auto * node : outputs) { @@ -2274,6 +2275,7 @@ const ActionsDAG::Node * ActionsDAG::getOriginalNodeForOutputAlias(const String if (!output_alias) return nullptr; + /// find original(non alias) node it refers to const Node * node = output_alias; while (node && node->type == ActionsDAG::ActionType::ALIAS) { diff --git a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp index 3975f7efb3a..6e44067411f 100644 --- a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp +++ b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp @@ -13,17 +13,17 @@ namespace DB::QueryPlanOptimizations { -constexpr bool debug_logging_enabled = true; -void logDebug(const String & prefix, const String & message) -{ - if constexpr (debug_logging_enabled) - { - LOG_DEBUG(&Poco::Logger::get("redundantDistinct"), "{}: {}", prefix, message); - } -} - namespace { + constexpr bool debug_logging_enabled = false; + void logActionsDAG(const String & prefix, const ActionsDAGPtr & actions) + { + if constexpr (debug_logging_enabled) + { + LOG_DEBUG(&Poco::Logger::get("redundantDistinct"), "{}: {}", prefix, actions->dumpDAG()); + } + } + std::set getDistinctColumns(const DistinctStep * distinct) { /// find non-const columns in DISTINCT @@ -100,6 +100,8 @@ size_t tryRemoveRedundantDistinct(QueryPlan::Node * parent_node, QueryPlan::Node path_actions->mergeInplace(std::move(*clone)); } + logActionsDAG("merged DAG:\n{}", path_actions); + /// compare columns of two DISTINCTs for (const auto & column : distinct_columns) { From 32ab4dc6930693fc744fa16c51c1c861d12486c4 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Thu, 2 Feb 2023 11:46:13 +0000 Subject: [PATCH 099/717] Fix review comments --- .../Optimizations/removeRedundantDistinct.cpp | 178 ++++++++++-------- 1 file changed, 97 insertions(+), 81 deletions(-) diff --git a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp index 6e44067411f..6d14a8c3d9c 100644 --- a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp +++ b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp @@ -36,97 +36,113 @@ namespace } return non_const_columns; } + + bool canRemoveDistinct(const QueryPlan::Node * distinct_node) + { + const DistinctStep * distinct_step = typeid_cast(distinct_node->step.get()); + chassert(distinct_step); + + std::vector dag_stack; + const DistinctStep * inner_distinct_step = nullptr; + const QueryPlan::Node * node = distinct_node; + while (!node->children.empty()) + { + const IQueryPlanStep * current_step = node->step.get(); + + /// don't try to remove DISTINCT after union or join + if (typeid_cast(current_step) || typeid_cast(current_step) + || typeid_cast(current_step)) + break; + + if (const auto * const expr = typeid_cast(current_step); expr) + dag_stack.push_back(expr->getExpression()); + if (const auto * const filter = typeid_cast(current_step); filter) + dag_stack.push_back(filter->getExpression()); + + node = node->children.front(); + inner_distinct_step = typeid_cast(node->step.get()); + if (inner_distinct_step) + break; + } + if (!inner_distinct_step) + return false; + + /// possible cases (outer distinct -> inner distinct): + /// final -> preliminary => do nothing + /// preliminary -> final => try remove preliminary + /// final -> final => try remove final + /// preliminary -> preliminary => logical error? + if (inner_distinct_step->isPreliminary()) + return false; + + const auto distinct_columns = getDistinctColumns(distinct_step); + auto inner_distinct_columns = getDistinctColumns(inner_distinct_step); + if (distinct_columns.size() != inner_distinct_columns.size()) + return false; + + ActionsDAGPtr path_actions; + if (!dag_stack.empty()) + { + /// build actions DAG to find original column names + path_actions = dag_stack.back(); + dag_stack.pop_back(); + while (!dag_stack.empty()) + { + ActionsDAGPtr clone = dag_stack.back()->clone(); + dag_stack.pop_back(); + path_actions->mergeInplace(std::move(*clone)); + } + + logActionsDAG("merged DAG:\n{}", path_actions); + + /// compare columns of two DISTINCTs + for (const auto & column : distinct_columns) + { + const auto * alias_node = path_actions->getOriginalNodeForOutputAlias(String(column)); + if (!alias_node) + return false; + + auto it = inner_distinct_columns.find(alias_node->result_name); + if (it == inner_distinct_columns.end()) + return false; + + inner_distinct_columns.erase(it); + } + } + else + { + if (distinct_columns != inner_distinct_columns) + return false; + } + + return true; + } } +/// +/// DISTINCT is redundant if DISTINCT on the same columns was executed before +/// Trivial example: SELECT DISTINCT * FROM (SELECT DISTINCT * FROM numbers(3)) +/// size_t tryRemoveRedundantDistinct(QueryPlan::Node * parent_node, QueryPlan::Nodes & /* nodes*/) { - if (parent_node->children.empty()) - return 0; - - /// check if it is preliminary distinct node - QueryPlan::Node * distinct_node = parent_node->children.front(); - DistinctStep * distinct_step = typeid_cast(distinct_node->step.get()); - if (!distinct_step) - return 0; - - std::vector dag_stack; - const DistinctStep * inner_distinct_step = nullptr; - QueryPlan::Node * node = distinct_node; - while (!node->children.empty()) + bool applied = false; + for (const auto * node : parent_node->children) { - const IQueryPlanStep * current_step = node->step.get(); + /// check if it is distinct node + const DistinctStep * distinct_step = typeid_cast(node->step.get()); + if (!distinct_step) + continue; - /// don't try to remove DISTINCT after union or join - if (typeid_cast(current_step) || typeid_cast(current_step) - || typeid_cast(current_step)) - break; - - if (const auto * const expr = typeid_cast(current_step); expr) - dag_stack.push_back(expr->getExpression()); - if (const auto * const filter = typeid_cast(current_step); filter) - dag_stack.push_back(filter->getExpression()); - - node = node->children.front(); - inner_distinct_step = typeid_cast(node->step.get()); - if (inner_distinct_step) - break; - } - if (!inner_distinct_step) - return 0; - - /// possible cases (outer distinct -> inner distinct): - /// final -> preliminary => do nothing - /// preliminary -> final => try remove preliminary - /// final -> final => try remove final - /// preliminary -> preliminary => logical error? - if (inner_distinct_step->isPreliminary()) - return 0; - - const auto distinct_columns = getDistinctColumns(distinct_step); - auto inner_distinct_columns = getDistinctColumns(inner_distinct_step); - if (distinct_columns.size() != inner_distinct_columns.size()) - return 0; - - ActionsDAGPtr path_actions; - if (!dag_stack.empty()) - { - /// build actions DAG to find original column names - path_actions = dag_stack.back(); - dag_stack.pop_back(); - while (!dag_stack.empty()) + if (canRemoveDistinct(node)) { - ActionsDAGPtr clone = dag_stack.back()->clone(); - dag_stack.pop_back(); - path_actions->mergeInplace(std::move(*clone)); - } - - logActionsDAG("merged DAG:\n{}", path_actions); - - /// compare columns of two DISTINCTs - for (const auto & column : distinct_columns) - { - const auto * alias_node = path_actions->getOriginalNodeForOutputAlias(String(column)); - if (!alias_node) - return 0; - - auto it = inner_distinct_columns.find(alias_node->result_name); - if (it == inner_distinct_columns.end()) - return 0; - - inner_distinct_columns.erase(it); + /// remove current distinct + chassert(!node->children.empty()); + parent_node->children[0] = node->children.front(); + applied = true; } } - else - { - if (distinct_columns != inner_distinct_columns) - return 0; - } - /// remove current distinct - chassert(!distinct_node->children.empty()); - parent_node->children[0] = distinct_node->children.front(); - - return 1; + return applied; } } From 174bdba912565e05a5e6d41ce8d2f44c4848606f Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 2 Feb 2023 16:57:46 +0100 Subject: [PATCH 100/717] Fix bugs in alter rename column --- src/Storages/MergeTree/AlterConversions.h | 2 + src/Storages/MergeTree/MergeTreeData.cpp | 2 + src/Storages/MergeTree/MutateTask.cpp | 60 +++++++++++++++++-- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 11 ++-- ...1_alter_rename_and_other_renames.reference | 6 +- .../01281_alter_rename_and_other_renames.sql | 2 +- .../02538_alter_rename_sequence.reference | 8 +++ .../02538_alter_rename_sequence.sql | 32 ++++++++++ 8 files changed, 110 insertions(+), 13 deletions(-) diff --git a/src/Storages/MergeTree/AlterConversions.h b/src/Storages/MergeTree/AlterConversions.h index 0d58499d424..6f5d5157138 100644 --- a/src/Storages/MergeTree/AlterConversions.h +++ b/src/Storages/MergeTree/AlterConversions.h @@ -17,6 +17,8 @@ struct AlterConversions /// Rename map new_name -> old_name std::unordered_map rename_map; + bool columnHasNewName(const std::string & old_name) const; + std::string getColumnNewName(const std::string & old_name) const; bool isColumnRenamed(const std::string & new_name) const { return rename_map.count(new_name) > 0; } std::string getColumnOldName(const std::string & new_name) const { return rename_map.at(new_name); } }; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index b2e0c14489a..99e84a6c783 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -7526,11 +7526,13 @@ AlterConversions MergeTreeData::getAlterConversionsForPart(const MergeTreeDataPa AlterConversions result{}; for (const auto & command : commands) + { /// Currently we need explicit conversions only for RENAME alter /// all other conversions can be deduced from diff between part columns /// and columns in storage. if (command.type == MutationCommand::Type::RENAME_COLUMN) result.rename_map[command.rename_to] = command.column_name; + } return result; } diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index 47df0cfe42e..da87fcaad9a 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -109,13 +109,29 @@ static void splitMutationCommands( } } } + + auto alter_conversions = part->storage.getAlterConversionsForPart(part); /// If it's compact part, then we don't need to actually remove files /// from disk we just don't read dropped columns for (const auto & column : part->getColumns()) { if (!mutated_columns.contains(column.name)) - for_interpreter.emplace_back( - MutationCommand{.type = MutationCommand::Type::READ_COLUMN, .column_name = column.name, .data_type = column.type}); + { + if (alter_conversions.columnHasNewName(column.name)) + { + for_interpreter.push_back( + { + .type = MutationCommand::Type::READ_COLUMN, + .column_name = alter_conversions.getColumnNewName(column.name), + .data_type = column.type + }); + } + else + { + for_interpreter.emplace_back( + MutationCommand{.type = MutationCommand::Type::READ_COLUMN, .column_name = column.name, .data_type = column.type}); + } + } } } else @@ -147,6 +163,22 @@ static void splitMutationCommands( for_file_renames.push_back(command); } } + + auto alter_conversions = part->storage.getAlterConversionsForPart(part); + for (const auto & part_column : part_columns) + { + if (alter_conversions.columnHasNewName(part_column.name)) + { + auto new_column_name = alter_conversions.getColumnNewName(part_column.name); + for_file_renames.push_back({ + .type = MutationCommand::Type::RENAME_COLUMN, + .column_name = part_column.name, + .rename_to = new_column_name, + }); + + part_columns.rename(part_column.name, new_column_name); + } + } } } @@ -276,12 +308,30 @@ getColumnsForNewDataPart( /// Column was renamed and no other column renamed to it's name /// or column is dropped. if (!renamed_columns_to_from.contains(it->name) && (was_renamed || was_removed)) + { it = storage_columns.erase(it); + } else { - /// Take a type from source part column. - /// It may differ from column type in storage. - it->type = source_col->second; + + if (was_removed) + { /// DROP COLUMN xxx, RENAME COLUMN yyy TO xxx + auto renamed_from = renamed_columns_to_from.at(it->name); + auto maybe_name_and_type = source_columns.tryGetByName(renamed_from); + if (!maybe_name_and_type) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Got incorrect mutation commands, column {} was renamed from {}, but it doesn't exist in source columns {}", + it->name, renamed_from, source_columns.toString()); + + it->type = maybe_name_and_type->type; + } + else + { + /// Take a type from source part column. + /// It may differ from column type in storage. + it->type = source_col->second; + } ++it; } } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index e47dddb9795..e4f7f67ee02 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -1759,12 +1759,15 @@ MutationCommands ReplicatedMergeTreeQueue::getFirstAlterMutationCommandsForPart( if (in_partition == mutations_by_partition.end()) return MutationCommands{}; - Int64 part_version = part->info.getDataVersion(); + Int64 part_mutation_version = part->info.getMutationVersion(); + MutationCommands result; for (auto [mutation_version, mutation_status] : in_partition->second) - if (mutation_version > part_version && mutation_status->entry->alter_version != -1) - return mutation_status->entry->commands; + { + if (mutation_version > part_mutation_version && mutation_status->entry->alter_version != -1) + result.insert(result.end(), mutation_status->entry->commands.begin(), mutation_status->entry->commands.end()); + } - return MutationCommands{}; + return result; } MutationCommands ReplicatedMergeTreeQueue::getMutationCommands( diff --git a/tests/queries/0_stateless/01281_alter_rename_and_other_renames.reference b/tests/queries/0_stateless/01281_alter_rename_and_other_renames.reference index bf3358aea60..532b8ce8712 100644 --- a/tests/queries/0_stateless/01281_alter_rename_and_other_renames.reference +++ b/tests/queries/0_stateless/01281_alter_rename_and_other_renames.reference @@ -1,11 +1,11 @@ -CREATE TABLE default.rename_table_multiple\n(\n `key` Int32,\n `value1_string` String,\n `value2` Int32\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.rename_table_multiple\n(\n `key` Int32,\n `value1_string` String,\n `value2` Int32\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS min_bytes_for_wide_part = 0, index_granularity = 8192 key value1_string value2 1 2 3 -CREATE TABLE default.rename_table_multiple\n(\n `key` Int32,\n `value1_string` String,\n `value2_old` Int32,\n `value2` Int64 DEFAULT 7\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.rename_table_multiple\n(\n `key` Int32,\n `value1_string` String,\n `value2_old` Int32,\n `value2` Int64 DEFAULT 7\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS min_bytes_for_wide_part = 0, index_granularity = 8192 key value1_string value2_old value2 1 2 3 7 4 5 6 7 -CREATE TABLE default.rename_table_multiple\n(\n `key` Int32,\n `value1_string` String,\n `value2_old` Int64 DEFAULT 7\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.rename_table_multiple\n(\n `key` Int32,\n `value1_string` String,\n `value2_old` Int64 DEFAULT 7\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS min_bytes_for_wide_part = 0, index_granularity = 8192 key value1_string value2_old 1 2 7 4 5 7 diff --git a/tests/queries/0_stateless/01281_alter_rename_and_other_renames.sql b/tests/queries/0_stateless/01281_alter_rename_and_other_renames.sql index f9462f0478e..b0ccd7751ab 100644 --- a/tests/queries/0_stateless/01281_alter_rename_and_other_renames.sql +++ b/tests/queries/0_stateless/01281_alter_rename_and_other_renames.sql @@ -1,6 +1,6 @@ DROP TABLE IF EXISTS rename_table_multiple; -CREATE TABLE rename_table_multiple (key Int32, value1 String, value2 Int32) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE rename_table_multiple (key Int32, value1 String, value2 Int32) ENGINE = MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part=0; INSERT INTO rename_table_multiple VALUES (1, 2, 3); diff --git a/tests/queries/0_stateless/02538_alter_rename_sequence.reference b/tests/queries/0_stateless/02538_alter_rename_sequence.reference index e69de29bb2d..73aa1b7e8d8 100644 --- a/tests/queries/0_stateless/02538_alter_rename_sequence.reference +++ b/tests/queries/0_stateless/02538_alter_rename_sequence.reference @@ -0,0 +1,8 @@ +1 2 3 +4 5 6 +{"column1_renamed":"1","column2_renamed":"2","column3":"3"} +{"column1_renamed":"4","column2_renamed":"5","column3":"6"} +1 2 3 +4 5 6 +{"column1_renamed":"1","column2_renamed":"2","column3":"3"} +{"column1_renamed":"4","column2_renamed":"5","column3":"6"} diff --git a/tests/queries/0_stateless/02538_alter_rename_sequence.sql b/tests/queries/0_stateless/02538_alter_rename_sequence.sql index efdb1d469eb..0eb839ebe59 100644 --- a/tests/queries/0_stateless/02538_alter_rename_sequence.sql +++ b/tests/queries/0_stateless/02538_alter_rename_sequence.sql @@ -16,6 +16,8 @@ ALTER TABLE wrong_metadata RENAME COLUMN column1 TO column1_renamed SETTINGS rep INSERT INTO wrong_metadata VALUES (4, 5, 6); +SELECT * FROM wrong_metadata ORDER BY column1; + SYSTEM START REPLICATION QUEUES wrong_metadata; SYSTEM SYNC REPLICA wrong_metadata; @@ -25,3 +27,33 @@ ALTER TABLE wrong_metadata RENAME COLUMN column2 to column2_renamed SETTINGS rep SELECT * FROM wrong_metadata ORDER BY column1_renamed FORMAT JSONEachRow; DROP TABLE IF EXISTS wrong_metadata; + + +CREATE TABLE wrong_metadata_wide( + column1 UInt64, + column2 UInt64, + column3 UInt64 +) +ENGINE ReplicatedMergeTree('/test/tables/wrong_metadata_wide', '1') +ORDER BY tuple() +SETTINGS min_bytes_for_wide_part = 0; + +INSERT INTO wrong_metadata_wide VALUES (1, 2, 3); + +SYSTEM STOP REPLICATION QUEUES wrong_metadata_wide; + +ALTER TABLE wrong_metadata_wide RENAME COLUMN column1 TO column1_renamed SETTINGS replication_alter_partitions_sync = 0; + +INSERT INTO wrong_metadata_wide VALUES (4, 5, 6); + +SELECT * FROM wrong_metadata_wide ORDER by column1; + +SYSTEM START REPLICATION QUEUES wrong_metadata_wide; + +SYSTEM SYNC REPLICA wrong_metadata_wide; + +ALTER TABLE wrong_metadata_wide RENAME COLUMN column2 to column2_renamed SETTINGS replication_alter_partitions_sync = 2; + +SELECT * FROM wrong_metadata_wide ORDER BY column1_renamed FORMAT JSONEachRow; + +DROP TABLE IF EXISTS wrong_metadata_wide; From dea46e58fbc203c674f2322ce8936614e3acc98d Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 2 Feb 2023 17:03:24 +0100 Subject: [PATCH 101/717] Fixes for ordinary merge tree --- src/Storages/MergeTree/AlterConversions.cpp | 34 +++++++++++++++++++++ src/Storages/StorageMergeTree.cpp | 13 +++++--- 2 files changed, 43 insertions(+), 4 deletions(-) create mode 100644 src/Storages/MergeTree/AlterConversions.cpp diff --git a/src/Storages/MergeTree/AlterConversions.cpp b/src/Storages/MergeTree/AlterConversions.cpp new file mode 100644 index 00000000000..e2eea2e68f6 --- /dev/null +++ b/src/Storages/MergeTree/AlterConversions.cpp @@ -0,0 +1,34 @@ +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +bool AlterConversions::columnHasNewName(const std::string & old_name) const +{ + for (const auto & [new_name, prev_name] : rename_map) + { + if (old_name == prev_name) + return true; + } + + return false; +} + +std::string AlterConversions::getColumnNewName(const std::string & old_name) const +{ + for (const auto & [new_name, prev_name] : rename_map) + { + if (old_name == prev_name) + return new_name; + } + + throw Exception(ErrorCodes::LOGICAL_ERROR, "Column {} was not renamed", old_name); +} + +} diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 4ef34ae91d5..e3d4ca072ca 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -2060,10 +2060,15 @@ MutationCommands StorageMergeTree::getFirstAlterMutationCommandsForPart(const Da { std::lock_guard lock(currently_processing_in_background_mutex); - auto it = current_mutations_by_version.upper_bound(part->info.getDataVersion()); - if (it == current_mutations_by_version.end()) - return {}; - return it->second.commands; + Int64 part_mutation_version = part->info.getMutationVersion(); + + MutationCommands result; + for (const auto & current_mutation_by_version : current_mutations_by_version) + { + if (static_cast(current_mutation_by_version.first) > part_mutation_version) + result.insert(result.end(), current_mutation_by_version.second.commands.begin(), current_mutation_by_version.second.commands.end()); + } + return result; } void StorageMergeTree::startBackgroundMovesIfNeeded() From 8477b811308014674e9f5bfb16c1e3a6a40f0196 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 2 Feb 2023 17:17:57 +0100 Subject: [PATCH 102/717] Add working test --- .../02543_alter_rename_modify_stuck.reference | 1 + .../02543_alter_rename_modify_stuck.sh | 58 +++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 tests/queries/0_stateless/02543_alter_rename_modify_stuck.reference create mode 100755 tests/queries/0_stateless/02543_alter_rename_modify_stuck.sh diff --git a/tests/queries/0_stateless/02543_alter_rename_modify_stuck.reference b/tests/queries/0_stateless/02543_alter_rename_modify_stuck.reference new file mode 100644 index 00000000000..156128e3dd2 --- /dev/null +++ b/tests/queries/0_stateless/02543_alter_rename_modify_stuck.reference @@ -0,0 +1 @@ +{"v":"1","v2":"77"} diff --git a/tests/queries/0_stateless/02543_alter_rename_modify_stuck.sh b/tests/queries/0_stateless/02543_alter_rename_modify_stuck.sh new file mode 100755 index 00000000000..0e9b39d6dae --- /dev/null +++ b/tests/queries/0_stateless/02543_alter_rename_modify_stuck.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + + +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS table_to_rename" + +$CLICKHOUSE_CLIENT --query="CREATE TABLE table_to_rename(v UInt64, v1 UInt64)ENGINE = MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0" + +$CLICKHOUSE_CLIENT --query="INSERT INTO table_to_rename VALUES (1, 1)" + + +# we want to following mutations to stuck +# That is why we stop merges and wait in loops until they actually start +$CLICKHOUSE_CLIENT --query="SYSTEM STOP MERGES table_to_rename" + +$CLICKHOUSE_CLIENT --query="ALTER TABLE table_to_rename RENAME COLUMN v1 to v2" & + +counter=0 retries=60 + +I=0 +while [[ $counter -lt $retries ]]; do + I=$((I + 1)) + result=$($CLICKHOUSE_CLIENT --query "show create table table_to_rename") + if [[ $result == *"v2"* ]]; then + break; + fi + sleep 0.1 + ((++counter)) +done + + +$CLICKHOUSE_CLIENT --query="ALTER TABLE table_to_rename UPDATE v2 = 77 WHERE 1 = 1" & + +counter=0 retries=60 + +I=0 +while [[ $counter -lt $retries ]]; do + I=$((I + 1)) + result=$($CLICKHOUSE_CLIENT --query "SELECT count() from system.mutations where database='${CLICKHOUSE_DATABASE}' and table='table_to_rename'") + if [[ $result == "2" ]]; then + break; + fi + sleep 0.1 + ((++counter)) +done + + +$CLICKHOUSE_CLIENT --query="SYSTEM START MERGES table_to_rename" + +wait + +$CLICKHOUSE_CLIENT --query="SELECT * FROM table_to_rename FORMAT JSONEachRow" + + + $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS table_to_rename" From bdc530dead710a4dd108e5effb1c561f57913174 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 2 Feb 2023 17:30:38 +0100 Subject: [PATCH 103/717] Fix style --- src/Storages/MergeTree/MergeTreeData.cpp | 2 +- src/Storages/MergeTree/MergeTreeData.h | 2 +- src/Storages/MergeTree/MutateTask.cpp | 6 +----- src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp | 6 +++++- src/Storages/MergeTree/ReplicatedMergeTreeQueue.h | 6 +++--- src/Storages/StorageMergeTree.cpp | 2 +- src/Storages/StorageMergeTree.h | 2 +- src/Storages/StorageReplicatedMergeTree.cpp | 4 ++-- src/Storages/StorageReplicatedMergeTree.h | 2 +- 9 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 99e84a6c783..c8b1020bd82 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -7522,7 +7522,7 @@ bool MergeTreeData::canUsePolymorphicParts(const MergeTreeSettings & settings, S AlterConversions MergeTreeData::getAlterConversionsForPart(const MergeTreeDataPartPtr part) const { - MutationCommands commands = getFirstAlterMutationCommandsForPart(part); + MutationCommands commands = getAlterMutationCommandsForPart(part); AlterConversions result{}; for (const auto & command : commands) diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 7dcd0c40553..0dfeb44524e 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -1303,7 +1303,7 @@ protected: /// Used to receive AlterConversions for part and apply them on fly. This /// method has different implementations for replicated and non replicated /// MergeTree because they store mutations in different way. - virtual MutationCommands getFirstAlterMutationCommandsForPart(const DataPartPtr & part) const = 0; + virtual MutationCommands getAlterMutationCommandsForPart(const DataPartPtr & part) const = 0; /// Moves part to specified space, used in ALTER ... MOVE ... queries bool movePartsToSpace(const DataPartsVector & parts, SpacePtr space); diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index da87fcaad9a..44ec321681b 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -170,11 +170,7 @@ static void splitMutationCommands( if (alter_conversions.columnHasNewName(part_column.name)) { auto new_column_name = alter_conversions.getColumnNewName(part_column.name); - for_file_renames.push_back({ - .type = MutationCommand::Type::RENAME_COLUMN, - .column_name = part_column.name, - .rename_to = new_column_name, - }); + for_file_renames.push_back({.type = MutationCommand::Type::RENAME_COLUMN, .column_name = part_column.name, .rename_to = new_column_name}); part_columns.rename(part_column.name, new_column_name); } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index e4f7f67ee02..e8ed0888fa1 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -1752,7 +1752,7 @@ ReplicatedMergeTreeMergePredicate ReplicatedMergeTreeQueue::getMergePredicate(zk } -MutationCommands ReplicatedMergeTreeQueue::getFirstAlterMutationCommandsForPart(const MergeTreeData::DataPartPtr & part) const +MutationCommands ReplicatedMergeTreeQueue::getAlterMutationCommandsForPart(const MergeTreeData::DataPartPtr & part) const { std::lock_guard lock(state_mutex); auto in_partition = mutations_by_partition.find(part->info.partition_id); @@ -1761,6 +1761,10 @@ MutationCommands ReplicatedMergeTreeQueue::getFirstAlterMutationCommandsForPart( Int64 part_mutation_version = part->info.getMutationVersion(); MutationCommands result; + /// Here we return mutation commands for part which has bigger mutation version than part mutation version. + /// Please note, we don't use getDataVersion(). It's because these alter commands are used for in-fly conversions + /// of part's metadata. It mean that even if we have mutation with version X and part with data version X+10, but + /// without mutation version part can still have wrong metadata and we have to apply this change on-fly if needed. for (auto [mutation_version, mutation_status] : in_partition->second) { if (mutation_version > part_mutation_version && mutation_status->entry->alter_version != -1) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h index 36f1ee07ad4..2c566668877 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h @@ -393,10 +393,10 @@ public: MutationCommands getMutationCommands(const MergeTreeData::DataPartPtr & part, Int64 desired_mutation_version) const; - /// Return mutation commands for part with smallest mutation version bigger - /// than data part version. Used when we apply alter commands on fly, + /// Return mutation commands for part which could be not applied to + /// it according to part mutation version. Used when we apply alter commands on fly, /// without actual data modification on disk. - MutationCommands getFirstAlterMutationCommandsForPart(const MergeTreeData::DataPartPtr & part) const; + MutationCommands getAlterMutationCommandsForPart(const MergeTreeData::DataPartPtr & part) const; /// Mark finished mutations as done. If the function needs to be called again at some later time /// (because some mutations are probably done but we are not sure yet), returns true. diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index e3d4ca072ca..a9664007614 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -2056,7 +2056,7 @@ void StorageMergeTree::attachRestoredParts(MutableDataPartsVector && parts) } -MutationCommands StorageMergeTree::getFirstAlterMutationCommandsForPart(const DataPartPtr & part) const +MutationCommands StorageMergeTree::getAlterMutationCommandsForPart(const DataPartPtr & part) const { std::lock_guard lock(currently_processing_in_background_mutex); diff --git a/src/Storages/StorageMergeTree.h b/src/Storages/StorageMergeTree.h index 1dff6323e4c..957d7804b96 100644 --- a/src/Storages/StorageMergeTree.h +++ b/src/Storages/StorageMergeTree.h @@ -265,7 +265,7 @@ private: protected: - MutationCommands getFirstAlterMutationCommandsForPart(const DataPartPtr & part) const override; + MutationCommands getAlterMutationCommandsForPart(const DataPartPtr & part) const override; }; } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 7a4b97f6e49..58adc4be594 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -7892,9 +7892,9 @@ bool StorageReplicatedMergeTree::canUseAdaptiveGranularity() const } -MutationCommands StorageReplicatedMergeTree::getFirstAlterMutationCommandsForPart(const DataPartPtr & part) const +MutationCommands StorageReplicatedMergeTree::getAlterMutationCommandsForPart(const DataPartPtr & part) const { - return queue.getFirstAlterMutationCommandsForPart(part); + return queue.getAlterMutationCommandsForPart(part); } diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 042e6acf4e2..3dc5ab75891 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -830,7 +830,7 @@ private: void waitMutationToFinishOnReplicas( const Strings & replicas, const String & mutation_id) const; - MutationCommands getFirstAlterMutationCommandsForPart(const DataPartPtr & part) const override; + MutationCommands getAlterMutationCommandsForPart(const DataPartPtr & part) const override; void startBackgroundMovesIfNeeded() override; From fc6c62e2ff62ce754362f9a44731bed0598168a4 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 2 Feb 2023 17:41:36 +0100 Subject: [PATCH 104/717] Add comment --- src/Storages/MergeTree/AlterConversions.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Storages/MergeTree/AlterConversions.h b/src/Storages/MergeTree/AlterConversions.h index 6f5d5157138..47f964e85fe 100644 --- a/src/Storages/MergeTree/AlterConversions.h +++ b/src/Storages/MergeTree/AlterConversions.h @@ -17,9 +17,13 @@ struct AlterConversions /// Rename map new_name -> old_name std::unordered_map rename_map; + /// Column was renamed (lookup by value in rename_map) bool columnHasNewName(const std::string & old_name) const; + /// Get new name for column (lookup by value in rename_map) std::string getColumnNewName(const std::string & old_name) const; + /// Is this name is new name of column (lookup by key in rename_map) bool isColumnRenamed(const std::string & new_name) const { return rename_map.count(new_name) > 0; } + /// Get column old name before rename (lookup by key in rename_map) std::string getColumnOldName(const std::string & new_name) const { return rename_map.at(new_name); } }; From ef546833860f32440859789e2103ad05a58e0ca4 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Thu, 2 Feb 2023 19:25:14 +0100 Subject: [PATCH 105/717] Use cluster state data to check concurrent backup/restore Implementation: * BackupWorker checks the if any backup/restore which has a path in zookeeper has status not completed, if yes, new backup/restore is stopped. * For not on cluster only active backup / restore is checked. * Removed restore_uuid from RestoreSettings, as it is no longer used. --- src/Backups/BackupCoordinationRemote.cpp | 7 +- src/Backups/BackupCoordinationRemote.h | 2 + src/Backups/BackupCoordinationStage.h | 4 + src/Backups/BackupsWorker.cpp | 133 +++++++++++++++++------ src/Backups/BackupsWorker.h | 4 +- src/Backups/RestoreSettings.cpp | 3 +- src/Backups/RestoreSettings.h | 5 - 7 files changed, 110 insertions(+), 48 deletions(-) diff --git a/src/Backups/BackupCoordinationRemote.cpp b/src/Backups/BackupCoordinationRemote.cpp index 18789802769..ad164768333 100644 --- a/src/Backups/BackupCoordinationRemote.cpp +++ b/src/Backups/BackupCoordinationRemote.cpp @@ -160,9 +160,6 @@ namespace { return fmt::format("{:03}", counter); /// Outputs 001, 002, 003, ... } - - /// We try to store data to zookeeper several times due to possible version conflicts. - constexpr size_t NUM_ATTEMPTS = 10; } BackupCoordinationRemote::BackupCoordinationRemote( @@ -468,7 +465,7 @@ void BackupCoordinationRemote::updateFileInfo(const FileInfo & file_info) auto zk = getZooKeeper(); String size_and_checksum = serializeSizeAndChecksum(std::pair{file_info.size, file_info.checksum}); String full_path = zookeeper_path + "/file_infos/" + size_and_checksum; - for (size_t attempt = 0; attempt < NUM_ATTEMPTS; ++attempt) + for (size_t attempt = 0; attempt < MAX_ZOOKEEPER_ATTEMPTS; ++attempt) { Coordination::Stat stat; auto new_info = deserializeFileInfo(zk->get(full_path, &stat)); @@ -476,7 +473,7 @@ void BackupCoordinationRemote::updateFileInfo(const FileInfo & file_info) auto code = zk->trySet(full_path, serializeFileInfo(new_info), stat.version); if (code == Coordination::Error::ZOK) return; - bool is_last_attempt = (attempt == NUM_ATTEMPTS - 1); + bool is_last_attempt = (attempt == MAX_ZOOKEEPER_ATTEMPTS - 1); if ((code != Coordination::Error::ZBADVERSION) || is_last_attempt) throw zkutil::KeeperException(code, full_path); } diff --git a/src/Backups/BackupCoordinationRemote.h b/src/Backups/BackupCoordinationRemote.h index 711fadb539e..b1a7a12e109 100644 --- a/src/Backups/BackupCoordinationRemote.h +++ b/src/Backups/BackupCoordinationRemote.h @@ -5,6 +5,8 @@ #include #include +/// We try to store data to zookeeper several times due to possible version conflicts. +constexpr size_t MAX_ZOOKEEPER_ATTEMPTS = 10; namespace DB { diff --git a/src/Backups/BackupCoordinationStage.h b/src/Backups/BackupCoordinationStage.h index 091c1f11463..2c02b651851 100644 --- a/src/Backups/BackupCoordinationStage.h +++ b/src/Backups/BackupCoordinationStage.h @@ -8,6 +8,10 @@ namespace DB namespace BackupCoordinationStage { + /// This stage is set after concurrency check so ensure we dont start other backup/restores + /// when concurrent backup/restores are not allowed + constexpr const char * SCHEDULED_TO_START = "scheduled to start"; + /// Finding all tables and databases which we're going to put to the backup and collecting their metadata. constexpr const char * GATHERING_METADATA = "gathering metadata"; diff --git a/src/Backups/BackupsWorker.cpp b/src/Backups/BackupsWorker.cpp index 5348f4ec81a..743f3329f5c 100644 --- a/src/Backups/BackupsWorker.cpp +++ b/src/Backups/BackupsWorker.cpp @@ -173,13 +173,6 @@ OperationID BackupsWorker::startMakingBackup(const ASTPtr & query, const Context String backup_name_for_logging = backup_info.toStringForLogging(); try { - if (!allow_concurrent_backups && hasConcurrentBackups(backup_settings)) - { - /// addInfo is called here to record the failed backup details - addInfo(backup_id, backup_name_for_logging, backup_settings.internal, BackupStatus::BACKUP_FAILED); - throw Exception(ErrorCodes::CONCURRENT_ACCESS_NOT_SUPPORTED, "Concurrent backups not supported, turn on setting 'allow_concurrent_backups'"); - } - addInfo(backup_id, backup_name_for_logging, backup_settings.internal, BackupStatus::CREATING_BACKUP); /// Prepare context to use. @@ -259,6 +252,7 @@ void BackupsWorker::doBackup( } bool on_cluster = !backup_query->cluster.empty(); + assert(mutable_context || (!on_cluster && !called_async)); /// Checks access rights if this is not ON CLUSTER query. @@ -284,6 +278,9 @@ void BackupsWorker::doBackup( if (!backup_coordination) backup_coordination = makeBackupCoordination(backup_settings.coordination_zk_path, context, backup_settings.internal); + if (!allow_concurrent_backups && !backup_settings.internal && hasConcurrentBackups(backup_id, context, on_cluster)) + throw Exception(ErrorCodes::CONCURRENT_ACCESS_NOT_SUPPORTED, "Concurrent backups not supported, turn on setting 'allow_concurrent_backups'"); + /// Opens a backup for writing. BackupFactory::CreateParams backup_create_params; backup_create_params.open_mode = IBackup::OpenMode::WRITE; @@ -384,9 +381,6 @@ OperationID BackupsWorker::startRestoring(const ASTPtr & query, ContextMutablePt auto restore_query = std::static_pointer_cast(query->clone()); auto restore_settings = RestoreSettings::fromRestoreQuery(*restore_query); - if (!restore_settings.restore_uuid) - restore_settings.restore_uuid = UUIDHelpers::generateV4(); - /// `restore_id` will be used as a key to the `infos` map, so it should be unique. OperationID restore_id; if (restore_settings.internal) @@ -394,7 +388,7 @@ OperationID BackupsWorker::startRestoring(const ASTPtr & query, ContextMutablePt else if (!restore_settings.id.empty()) restore_id = restore_settings.id; else - restore_id = toString(*restore_settings.restore_uuid); + restore_id = toString(UUIDHelpers::generateV4()); std::shared_ptr restore_coordination; if (restore_settings.internal) @@ -410,13 +404,6 @@ OperationID BackupsWorker::startRestoring(const ASTPtr & query, ContextMutablePt auto backup_info = BackupInfo::fromAST(*restore_query->backup_name); String backup_name_for_logging = backup_info.toStringForLogging(); - if (!allow_concurrent_restores && hasConcurrentRestores(restore_settings)) - { - /// addInfo is called here to record the failed restore details - addInfo(restore_id, backup_name_for_logging, restore_settings.internal, BackupStatus::RESTORING); - throw Exception(ErrorCodes::CONCURRENT_ACCESS_NOT_SUPPORTED, "Concurrent restores not supported, turn on setting 'allow_concurrent_restores'"); - } - addInfo(restore_id, backup_name_for_logging, restore_settings.internal, BackupStatus::RESTORING); /// Prepare context to use. @@ -496,7 +483,6 @@ void BackupsWorker::doRestore( backup_open_params.context = context; backup_open_params.backup_info = backup_info; backup_open_params.base_backup_info = restore_settings.base_backup_info; - backup_open_params.backup_uuid = restore_settings.restore_uuid; backup_open_params.password = restore_settings.password; BackupPtr backup = BackupFactory::instance().createBackup(backup_open_params); @@ -532,12 +518,15 @@ void BackupsWorker::doRestore( if (on_cluster && restore_settings.coordination_zk_path.empty()) { String root_zk_path = context->getConfigRef().getString("backups.zookeeper_path", "/clickhouse/backups"); - restore_settings.coordination_zk_path = root_zk_path + "/restore-" + toString(UUIDHelpers::generateV4()); + restore_settings.coordination_zk_path = root_zk_path + "/restore-" + toString(restore_id); } if (!restore_coordination) restore_coordination = makeRestoreCoordination(restore_settings.coordination_zk_path, context, restore_settings.internal); + if (!allow_concurrent_restores && !restore_settings.internal && hasConcurrentRestores(restore_id, context, on_cluster)) + throw Exception(ErrorCodes::CONCURRENT_ACCESS_NOT_SUPPORTED, "Concurrent restores not supported, turn on setting 'allow_concurrent_restores'"); + /// Do RESTORE. if (on_cluster) { @@ -744,32 +733,108 @@ std::vector BackupsWorker::getAllActiveRestoreInfos() const return res_infos; } -bool BackupsWorker::hasConcurrentBackups(const BackupSettings & backup_settings) const +bool BackupsWorker::hasConcurrentBackups(const OperationID & backup_id, const ContextPtr & context, bool on_cluster) const { - /// Check if there are no concurrent backups - if (num_active_backups) + if (on_cluster) { - /// If its an internal backup and we currently have 1 active backup, it could be the original query, validate using backup_uuid - if (!(num_active_backups == 1 && backup_settings.internal && getAllActiveBackupInfos().at(0).id == toString(*backup_settings.backup_uuid))) + String common_backup_path = context->getConfigRef().getString("backups.zookeeper_path", "/clickhouse/backups") ; + auto zookeeper = context->getGlobalContext()->getZooKeeper(); + std::string backup_stage_path = common_backup_path + "/backup-" + toString(backup_id) +"/stage"; + + if (!zookeeper->exists(common_backup_path)) + zookeeper->createAncestors(common_backup_path); + + for (size_t attempt = 0; attempt < MAX_ZOOKEEPER_ATTEMPTS; ++attempt) { - return true; + Coordination::Stat stat; + zookeeper->get(common_backup_path, &stat); + Strings existing_backup_paths = zookeeper->getChildren(common_backup_path); + + for (const auto & existing_backup_path : existing_backup_paths) + { + if (startsWith(existing_backup_path, "restore-")) + continue; + + String existing_backup_id = existing_backup_path; + existing_backup_id.erase(0, String("backup-").size()); + + if (existing_backup_id == toString(backup_id)) + continue; + + const auto status = zookeeper->get(common_backup_path + "/" + existing_backup_path + "/stage"); + if (status != Stage::COMPLETED) + return true; + } + + zookeeper->createIfNotExists(backup_stage_path, ""); + auto code = zookeeper->trySet(backup_stage_path, Stage::SCHEDULED_TO_START, stat.version); + if (code == Coordination::Error::ZOK) + break; + bool is_last_attempt = (attempt == MAX_ZOOKEEPER_ATTEMPTS - 1); + if ((code != Coordination::Error::ZBADVERSION) || is_last_attempt) + throw zkutil::KeeperException(code, backup_stage_path); } + return false; + } + else + { + if (num_active_backups == 1) + return false; + else + return true; } - return false; } -bool BackupsWorker::hasConcurrentRestores(const RestoreSettings & restore_settings) const +bool BackupsWorker::hasConcurrentRestores(const OperationID & restore_id, const ContextPtr & context, bool on_cluster) const { - /// Check if there are no concurrent restores - if (num_active_restores) + if (on_cluster) { - /// If its an internal restore and we currently have 1 active restore, it could be the original query, validate using iz - if (!(num_active_restores == 1 && restore_settings.internal && getAllActiveRestoreInfos().at(0).id == toString(*restore_settings.restore_uuid))) + String common_restore_path = context->getConfigRef().getString("backups.zookeeper_path", "/clickhouse/backups") ; + auto zookeeper = context->getGlobalContext()->getZooKeeper(); + std::string path = common_restore_path + "/restore-" + toString(restore_id) +"/stage"; + + if (!zookeeper->exists(common_restore_path)) + zookeeper->createAncestors(common_restore_path); + + for (size_t attempt = 0; attempt < MAX_ZOOKEEPER_ATTEMPTS; ++attempt) { - return true; + Coordination::Stat stat; + zookeeper->get(common_restore_path, &stat); + Strings existing_restore_paths = zookeeper->getChildren(common_restore_path); + for (const auto & existing_restore_path : existing_restore_paths) + { + if (startsWith(existing_restore_path, "backup-")) + continue; + + String existing_restore_id = existing_restore_path; + existing_restore_id.erase(0, String("restore-").size()); + + if (existing_restore_id == toString(restore_id)) + continue; + + + const auto status = zookeeper->get(common_restore_path + "/" + existing_restore_path + "/stage"); + if (status != Stage::COMPLETED) + return true; + } + + zookeeper->createIfNotExists(path, ""); + auto code = zookeeper->trySet(path, Stage::SCHEDULED_TO_START, stat.version); + if (code == Coordination::Error::ZOK) + break; + bool is_last_attempt = (attempt == MAX_ZOOKEEPER_ATTEMPTS - 1); + if ((code != Coordination::Error::ZBADVERSION) || is_last_attempt) + throw zkutil::KeeperException(code, path); } + return false; + } + else + { + if (num_active_restores == 1) + return false; + else + return true; } - return false; } void BackupsWorker::shutdown() diff --git a/src/Backups/BackupsWorker.h b/src/Backups/BackupsWorker.h index 462a9033251..994e5f1f8b7 100644 --- a/src/Backups/BackupsWorker.h +++ b/src/Backups/BackupsWorker.h @@ -113,8 +113,8 @@ private: void setNumFilesAndSize(const OperationID & id, size_t num_files, size_t num_processed_files, UInt64 processed_files_size, UInt64 uncompressed_size, UInt64 compressed_size); std::vector getAllActiveBackupInfos() const; std::vector getAllActiveRestoreInfos() const; - bool hasConcurrentBackups(const BackupSettings & backup_settings) const; - bool hasConcurrentRestores(const RestoreSettings & restore_settings) const; + bool hasConcurrentBackups(const OperationID & backup_id, const ContextPtr & context, bool on_cluster) const; + bool hasConcurrentRestores(const OperationID & restore_id, const ContextPtr & context, bool on_cluster) const; ThreadPool backups_thread_pool; ThreadPool restores_thread_pool; diff --git a/src/Backups/RestoreSettings.cpp b/src/Backups/RestoreSettings.cpp index 5e06764a247..bbcefb819cf 100644 --- a/src/Backups/RestoreSettings.cpp +++ b/src/Backups/RestoreSettings.cpp @@ -163,8 +163,7 @@ namespace M(RestoreUDFCreationMode, create_function) \ M(Bool, internal) \ M(String, host_id) \ - M(String, coordination_zk_path) \ - M(OptionalUUID, restore_uuid) + M(String, coordination_zk_path) RestoreSettings RestoreSettings::fromRestoreQuery(const ASTBackupQuery & query) diff --git a/src/Backups/RestoreSettings.h b/src/Backups/RestoreSettings.h index 3bfe9f869ea..713adbe8029 100644 --- a/src/Backups/RestoreSettings.h +++ b/src/Backups/RestoreSettings.h @@ -122,11 +122,6 @@ struct RestoreSettings /// Path in Zookeeper used to coordinate restoring process while executing by RESTORE ON CLUSTER. String coordination_zk_path; - /// Internal, should not be specified by user. - /// UUID of the restore. If it's not set it will be generated randomly. - /// This is used to validate internal restores when allow_concurrent_restores is turned off - std::optional restore_uuid; - static RestoreSettings fromRestoreQuery(const ASTBackupQuery & query); void copySettingsToQuery(ASTBackupQuery & query) const; }; From 46ed75747256e38f87678cca5190938cf2097901 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Thu, 2 Feb 2023 20:07:47 +0000 Subject: [PATCH 106/717] Fix review comments --- src/Interpreters/ActionsDAG.h | 2 +- .../QueryPlan/Optimizations/removeRedundantDistinct.cpp | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index fa394c5bde6..a1d57db71d5 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -348,7 +348,7 @@ public: const ContextPtr & context); /// Return original node (input) for alias in output if exists - const Node* getOriginalNodeForOutputAlias(const String & output_name); + const Node * getOriginalNodeForOutputAlias(const String & output_name); private: NodeRawConstPtrs getParents(const Node * target) const; diff --git a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp index 6d14a8c3d9c..aa235e9e8af 100644 --- a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp +++ b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp @@ -49,7 +49,7 @@ namespace { const IQueryPlanStep * current_step = node->step.get(); - /// don't try to remove DISTINCT after union or join + /// don't try to remove DISTINCT after union/join/intersect/except if (typeid_cast(current_step) || typeid_cast(current_step) || typeid_cast(current_step)) break; @@ -129,8 +129,7 @@ size_t tryRemoveRedundantDistinct(QueryPlan::Node * parent_node, QueryPlan::Node for (const auto * node : parent_node->children) { /// check if it is distinct node - const DistinctStep * distinct_step = typeid_cast(node->step.get()); - if (!distinct_step) + if (typeid_cast(node->step.get())) continue; if (canRemoveDistinct(node)) From b8b0f1c3d3a2df7d0d23e4f67d6f6afe3c9dc5ca Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 2 Feb 2023 21:11:19 +0100 Subject: [PATCH 107/717] Make RENAME COLUMN barrier --- .../MergeTree/MergeTreeMarksLoader.cpp | 16 +++++++++- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 25 ++++++++++++++- src/Storages/StorageMergeTree.cpp | 31 +++++++++++++++++-- .../02543_alter_rename_modify_stuck.sh | 2 +- 4 files changed, 68 insertions(+), 6 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeMarksLoader.cpp b/src/Storages/MergeTree/MergeTreeMarksLoader.cpp index 3fc7ff54c35..1d85ac1bd34 100644 --- a/src/Storages/MergeTree/MergeTreeMarksLoader.cpp +++ b/src/Storages/MergeTree/MergeTreeMarksLoader.cpp @@ -102,6 +102,15 @@ MarkCache::MappedPtr MergeTreeMarksLoader::loadMarksImpl() auto res = std::make_shared(marks_count * columns_in_mark); + if (file_size == 0 && marks_count != 0) + { + throw Exception( + ErrorCodes::CORRUPTED_DATA, + "Empty marks file '{}': {}, must be: {}", + std::string(fs::path(data_part_storage->getFullPath()) / mrk_path), + file_size, expected_uncompressed_size); + } + if (!index_granularity_info.mark_type.compressed && expected_uncompressed_size != file_size) throw Exception( ErrorCodes::CORRUPTED_DATA, @@ -138,7 +147,12 @@ MarkCache::MappedPtr MergeTreeMarksLoader::loadMarksImpl() } if (i * mark_size != expected_uncompressed_size) - throw Exception(ErrorCodes::CANNOT_READ_ALL_DATA, "Cannot read all marks from file {}", mrk_path); + { + throw Exception( + ErrorCodes::CANNOT_READ_ALL_DATA, + "Cannot read all marks from file {}, marks expected {} (bytes size {}), marks read {} (bytes size {})", + mrk_path, marks_count, expected_uncompressed_size, i, reader->count()); + } } res->protect(); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index e8ed0888fa1..2c957132918 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -1813,7 +1813,30 @@ MutationCommands ReplicatedMergeTreeQueue::getMutationCommands( MutationCommands commands; for (auto it = begin; it != end; ++it) - commands.insert(commands.end(), it->second->entry->commands.begin(), it->second->entry->commands.end()); + { + bool rename_command = false; + if (it->second->entry->isAlterMutation()) + { + const auto & single_mutation_commands = it->second->entry->commands; + for (const auto & command : single_mutation_commands) + { + if (command.type == MutationCommand::Type::RENAME_COLUMN) + { + rename_command = true; + break; + } + } + } + + if (rename_command) + { + if (commands.empty()) + commands.insert(commands.end(), it->second->entry->commands.begin(), it->second->entry->commands.end()); + break; + } + else + commands.insert(commands.end(), it->second->entry->commands.begin(), it->second->entry->commands.end()); + } return commands; } diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index a9664007614..73585a630cb 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -1114,9 +1114,34 @@ MergeMutateSelectedEntryPtr StorageMergeTree::selectPartsToMutate( if (current_ast_elements + commands_size >= max_ast_elements) break; - current_ast_elements += commands_size; - commands->insert(commands->end(), it->second.commands.begin(), it->second.commands.end()); - last_mutation_to_apply = it; + bool rename_command = false; + const auto & single_mutation_commands = it->second.commands; + for (const auto & command : single_mutation_commands) + { + if (command.type == MutationCommand::Type::RENAME_COLUMN) + { + rename_command = true; + break; + } + } + + if (rename_command) + { + if (commands->empty()) + { + current_ast_elements += commands_size; + commands->insert(commands->end(), it->second.commands.begin(), it->second.commands.end()); + last_mutation_to_apply = it; + } + break; + } + else + { + current_ast_elements += commands_size; + commands->insert(commands->end(), it->second.commands.begin(), it->second.commands.end()); + last_mutation_to_apply = it; + } + } assert(commands->empty() == (last_mutation_to_apply == mutations_end_it)); diff --git a/tests/queries/0_stateless/02543_alter_rename_modify_stuck.sh b/tests/queries/0_stateless/02543_alter_rename_modify_stuck.sh index 0e9b39d6dae..adaf1846552 100755 --- a/tests/queries/0_stateless/02543_alter_rename_modify_stuck.sh +++ b/tests/queries/0_stateless/02543_alter_rename_modify_stuck.sh @@ -32,7 +32,7 @@ while [[ $counter -lt $retries ]]; do done -$CLICKHOUSE_CLIENT --query="ALTER TABLE table_to_rename UPDATE v2 = 77 WHERE 1 = 1" & +$CLICKHOUSE_CLIENT --query="ALTER TABLE table_to_rename UPDATE v2 = 77 WHERE 1 = 1 SETTINGS mutations_sync = 2" & counter=0 retries=60 From 74298f111e3f5922698c66c66c6823967710471f Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 2 Feb 2023 17:25:04 -0300 Subject: [PATCH 108/717] Implement AutoFinalQueryOnPass to enable it on Analyzer as well --- src/Analyzer/Passes/AutoFinalOnQueryPass.cpp | 64 ++++++++ src/Analyzer/Passes/AutoFinalOnQueryPass.h | 21 +++ src/Analyzer/QueryTreePassManager.cpp | 4 + .../02420_final_setting_analyzer.reference | 138 ++++++++++++++++++ .../02420_final_setting_analyzer.sql | 107 ++++++++++++++ 5 files changed, 334 insertions(+) create mode 100644 src/Analyzer/Passes/AutoFinalOnQueryPass.cpp create mode 100644 src/Analyzer/Passes/AutoFinalOnQueryPass.h create mode 100644 tests/queries/0_stateless/02420_final_setting_analyzer.reference create mode 100644 tests/queries/0_stateless/02420_final_setting_analyzer.sql diff --git a/src/Analyzer/Passes/AutoFinalOnQueryPass.cpp b/src/Analyzer/Passes/AutoFinalOnQueryPass.cpp new file mode 100644 index 00000000000..97a770a4a6c --- /dev/null +++ b/src/Analyzer/Passes/AutoFinalOnQueryPass.cpp @@ -0,0 +1,64 @@ +#include "AutoFinalOnQueryPass.h" + +#include +#include +#include +#include + +namespace DB +{ + + +namespace +{ + class AutoFinalOnQueryPassVisitor : public InDepthQueryTreeVisitorWithContext + { + public: + using Base = InDepthQueryTreeVisitorWithContext; + using Base::Base; + + void visitImpl(QueryTreeNodePtr & node) + { + if (auto * table_node = node->as()) + { + if (autoFinalOnQuery(*table_node, table_node->getStorage(), getContext())) + { + auto modifier = TableExpressionModifiers(true, std::nullopt, std::nullopt); + table_node->setTableExpressionModifiers(modifier); + } + } + } + + private: + bool autoFinalOnQuery(TableNode & table_node, StoragePtr storage, ContextPtr context) + { + // query.tables() is required because not all queries have tables in it, it could be a function. + bool is_auto_final_setting_on = context->getSettingsRef().final; + bool is_final_supported = storage && storage->supportsFinal() && !storage->isRemote(); + bool is_query_already_final = table_node.hasTableExpressionModifiers() ? table_node.getTableExpressionModifiers().has_value() : false; + + return is_auto_final_setting_on && !is_query_already_final && is_final_supported; + } + + }; + +} + +String AutoFinalOnQueryPass::getName() +{ + return "AutoFinalOnQueryPass"; +} + +String AutoFinalOnQueryPass::getDescription() +{ + return "Automatically applies final modifier to queries if it is supported and if user level final setting is set."; +} + +void AutoFinalOnQueryPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context) +{ + auto visitor = AutoFinalOnQueryPassVisitor(std::move(context)); + + visitor.visit(query_tree_node); +} + +} diff --git a/src/Analyzer/Passes/AutoFinalOnQueryPass.h b/src/Analyzer/Passes/AutoFinalOnQueryPass.h new file mode 100644 index 00000000000..eacbe0f8235 --- /dev/null +++ b/src/Analyzer/Passes/AutoFinalOnQueryPass.h @@ -0,0 +1,21 @@ +#pragma once + +#include +#include +#include + +namespace DB +{ + + +class AutoFinalOnQueryPass final : public IQueryTreePass +{ +public: + String getName() override; + + String getDescription() override; + + void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override; +}; + +} diff --git a/src/Analyzer/QueryTreePassManager.cpp b/src/Analyzer/QueryTreePassManager.cpp index 0280831a255..62bf5b7de5b 100644 --- a/src/Analyzer/QueryTreePassManager.cpp +++ b/src/Analyzer/QueryTreePassManager.cpp @@ -34,6 +34,7 @@ #include #include #include +#include namespace DB { @@ -239,6 +240,9 @@ void addQueryTreePasses(QueryTreePassManager & manager) manager.addPass(std::make_unique()); manager.addPass(std::make_unique()); + + manager.addPass(std::make_unique()); + } } diff --git a/tests/queries/0_stateless/02420_final_setting_analyzer.reference b/tests/queries/0_stateless/02420_final_setting_analyzer.reference new file mode 100644 index 00000000000..dfc64fa39a1 --- /dev/null +++ b/tests/queries/0_stateless/02420_final_setting_analyzer.reference @@ -0,0 +1,138 @@ +-- { echoOn } +set allow_experimental_analyzer=1; +SYSTEM STOP MERGES tbl; +-- simple test case +create table if not exists replacing_mt (x String) engine=ReplacingMergeTree() ORDER BY x; +insert into replacing_mt values ('abc'); +insert into replacing_mt values ('abc'); +-- expected output is 2 because final is turned off +select count() from replacing_mt; +2 +set final = 1; +-- expected output is 1 because final is turned on +select count() from replacing_mt; +1 +-- JOIN test cases +create table if not exists lhs (x String) engine=ReplacingMergeTree() ORDER BY x; +create table if not exists rhs (x String) engine=ReplacingMergeTree() ORDER BY x; +insert into lhs values ('abc'); +insert into lhs values ('abc'); +insert into rhs values ('abc'); +insert into rhs values ('abc'); +set final = 0; +-- expected output is 4 because select_final == 0 +select count() from lhs inner join rhs on lhs.x = rhs.x; +4 +set final = 1; +-- expected output is 1 because final == 1 +select count() from lhs inner join rhs on lhs.x = rhs.x; +1 +-- regular non final table +set final = 1; +create table if not exists regular_mt_table (x String) engine=MergeTree() ORDER BY x; +insert into regular_mt_table values ('abc'); +insert into regular_mt_table values ('abc'); +-- expected output is 2, it should silently ignore final modifier +select count() from regular_mt_table; +2 +-- view test +create materialized VIEW mv_regular_mt_table TO regular_mt_table AS SELECT * FROM regular_mt_table; +create view nv_regular_mt_table AS SELECT * FROM mv_regular_mt_table; +set final=1; +select count() from nv_regular_mt_table; +2 +-- join on mix of tables that support / do not support select final with explain +create table if not exists left_table (id UInt64, val_left String) engine=ReplacingMergeTree() ORDER BY id; +create table if not exists middle_table (id UInt64, val_middle String) engine=MergeTree() ORDER BY id; +create table if not exists right_table (id UInt64, val_right String) engine=ReplacingMergeTree() ORDER BY id; +insert into left_table values (1,'a'); +insert into left_table values (1,'b'); +insert into left_table values (1,'c'); +insert into middle_table values (1,'a'); +insert into middle_table values (1,'b'); +insert into right_table values (1,'a'); +insert into right_table values (1,'b'); +insert into right_table values (1,'c'); +-- expected output +-- 1 c a c +-- 1 c b c +select left_table.id,val_left, val_middle, val_right from left_table + inner join middle_table on left_table.id = middle_table.id + inner join right_table on middle_table.id = right_table.id +ORDER BY left_table.id, val_left, val_middle, val_right; +1 c a c +1 c b c +explain syntax select left_table.id,val_left, val_middle, val_right from left_table + inner join middle_table on left_table.id = middle_table.id + inner join right_table on middle_table.id = right_table.id + ORDER BY left_table.id, val_left, val_middle, val_right; +SELECT + `--left_table.id` AS `left_table.id`, + val_left, + val_middle, + val_right +FROM +( + SELECT + val_left, + id AS `--left_table.id`, + val_middle, + middle_table.id AS `--middle_table.id` + FROM left_table + FINAL + ALL INNER JOIN + ( + SELECT + val_middle, + id + FROM middle_table + FINAL + ) AS middle_table ON `--left_table.id` = `--middle_table.id` +) AS `--.s` +ALL INNER JOIN +( + SELECT + val_right, + id + FROM right_table + FINAL +) AS right_table ON `--middle_table.id` = id +ORDER BY + `--left_table.id` ASC, + val_left ASC, + val_middle ASC, + val_right ASC +-- extra: same with subquery +select left_table.id,val_left, val_middle, val_right from left_table + inner join middle_table on left_table.id = middle_table.id + inner join (SELECT * FROM right_table WHERE id = 1) r on middle_table.id = r.id +ORDER BY left_table.id, val_left, val_middle, val_right; +1 c a c +1 c b c +-- no distributed tests because it is not currently supported: +-- JOIN with remote storages is unsupported. + +-- Quite exotic with Merge engine +DROP TABLE IF EXISTS table_to_merge_a; +DROP TABLE IF EXISTS table_to_merge_b; +DROP TABLE IF EXISTS table_to_merge_c; +DROP TABLE IF EXISTS merge_table; +create table if not exists table_to_merge_a (id UInt64, val String) engine=ReplacingMergeTree() ORDER BY id; +create table if not exists table_to_merge_b (id UInt64, val String) engine=MergeTree() ORDER BY id; +create table if not exists table_to_merge_c (id UInt64, val String) engine=ReplacingMergeTree() ORDER BY id; +CREATE TABLE merge_table Engine=Merge(currentDatabase(), '^(table_to_merge_[a-z])$') AS table_to_merge_a; +insert into table_to_merge_a values (1,'a'); +insert into table_to_merge_a values (1,'b'); +insert into table_to_merge_a values (1,'c'); +insert into table_to_merge_b values (2,'a'); +insert into table_to_merge_b values (2,'b'); +insert into table_to_merge_c values (3,'a'); +insert into table_to_merge_c values (3,'b'); +insert into table_to_merge_c values (3,'c'); +-- expected output: +-- 1 c, 2 a, 2 b, 3 c +SELECT * FROM merge_table ORDER BY id, val; +1 c +2 a +2 b +3 c diff --git a/tests/queries/0_stateless/02420_final_setting_analyzer.sql b/tests/queries/0_stateless/02420_final_setting_analyzer.sql new file mode 100644 index 00000000000..5937e536239 --- /dev/null +++ b/tests/queries/0_stateless/02420_final_setting_analyzer.sql @@ -0,0 +1,107 @@ +-- { echoOn } +set allow_experimental_analyzer=1; +SYSTEM STOP MERGES tbl; + +-- simple test case +create table if not exists replacing_mt (x String) engine=ReplacingMergeTree() ORDER BY x; + +insert into replacing_mt values ('abc'); +insert into replacing_mt values ('abc'); + +-- expected output is 2 because final is turned off +select count() from replacing_mt; + +set final = 1; +-- expected output is 1 because final is turned on +select count() from replacing_mt; + +-- JOIN test cases +create table if not exists lhs (x String) engine=ReplacingMergeTree() ORDER BY x; +create table if not exists rhs (x String) engine=ReplacingMergeTree() ORDER BY x; + +insert into lhs values ('abc'); +insert into lhs values ('abc'); + +insert into rhs values ('abc'); +insert into rhs values ('abc'); + +set final = 0; +-- expected output is 4 because select_final == 0 +select count() from lhs inner join rhs on lhs.x = rhs.x; + +set final = 1; +-- expected output is 1 because final == 1 +select count() from lhs inner join rhs on lhs.x = rhs.x; + +-- regular non final table +set final = 1; +create table if not exists regular_mt_table (x String) engine=MergeTree() ORDER BY x; +insert into regular_mt_table values ('abc'); +insert into regular_mt_table values ('abc'); +-- expected output is 2, it should silently ignore final modifier +select count() from regular_mt_table; + +-- view test +create materialized VIEW mv_regular_mt_table TO regular_mt_table AS SELECT * FROM regular_mt_table; +create view nv_regular_mt_table AS SELECT * FROM mv_regular_mt_table; + +set final=1; +select count() from nv_regular_mt_table; + +-- join on mix of tables that support / do not support select final with explain +create table if not exists left_table (id UInt64, val_left String) engine=ReplacingMergeTree() ORDER BY id; +create table if not exists middle_table (id UInt64, val_middle String) engine=MergeTree() ORDER BY id; +create table if not exists right_table (id UInt64, val_right String) engine=ReplacingMergeTree() ORDER BY id; +insert into left_table values (1,'a'); +insert into left_table values (1,'b'); +insert into left_table values (1,'c'); +insert into middle_table values (1,'a'); +insert into middle_table values (1,'b'); +insert into right_table values (1,'a'); +insert into right_table values (1,'b'); +insert into right_table values (1,'c'); +-- expected output +-- 1 c a c +-- 1 c b c +select left_table.id,val_left, val_middle, val_right from left_table + inner join middle_table on left_table.id = middle_table.id + inner join right_table on middle_table.id = right_table.id +ORDER BY left_table.id, val_left, val_middle, val_right; + +explain syntax select left_table.id,val_left, val_middle, val_right from left_table + inner join middle_table on left_table.id = middle_table.id + inner join right_table on middle_table.id = right_table.id + ORDER BY left_table.id, val_left, val_middle, val_right; + +-- extra: same with subquery +select left_table.id,val_left, val_middle, val_right from left_table + inner join middle_table on left_table.id = middle_table.id + inner join (SELECT * FROM right_table WHERE id = 1) r on middle_table.id = r.id +ORDER BY left_table.id, val_left, val_middle, val_right; + +-- no distributed tests because it is not currently supported: +-- JOIN with remote storages is unsupported. + +-- Quite exotic with Merge engine +DROP TABLE IF EXISTS table_to_merge_a; +DROP TABLE IF EXISTS table_to_merge_b; +DROP TABLE IF EXISTS table_to_merge_c; +DROP TABLE IF EXISTS merge_table; + +create table if not exists table_to_merge_a (id UInt64, val String) engine=ReplacingMergeTree() ORDER BY id; +create table if not exists table_to_merge_b (id UInt64, val String) engine=MergeTree() ORDER BY id; +create table if not exists table_to_merge_c (id UInt64, val String) engine=ReplacingMergeTree() ORDER BY id; +CREATE TABLE merge_table Engine=Merge(currentDatabase(), '^(table_to_merge_[a-z])$') AS table_to_merge_a; + +insert into table_to_merge_a values (1,'a'); +insert into table_to_merge_a values (1,'b'); +insert into table_to_merge_a values (1,'c'); +insert into table_to_merge_b values (2,'a'); +insert into table_to_merge_b values (2,'b'); +insert into table_to_merge_c values (3,'a'); +insert into table_to_merge_c values (3,'b'); +insert into table_to_merge_c values (3,'c'); + +-- expected output: +-- 1 c, 2 a, 2 b, 3 c +SELECT * FROM merge_table ORDER BY id, val; From 2de89f62e2ce90ec24ba98312fa152c33516eed4 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 2 Feb 2023 17:30:09 -0300 Subject: [PATCH 109/717] make function static --- src/Analyzer/Passes/AutoFinalOnQueryPass.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Analyzer/Passes/AutoFinalOnQueryPass.cpp b/src/Analyzer/Passes/AutoFinalOnQueryPass.cpp index 97a770a4a6c..613a7b36261 100644 --- a/src/Analyzer/Passes/AutoFinalOnQueryPass.cpp +++ b/src/Analyzer/Passes/AutoFinalOnQueryPass.cpp @@ -8,7 +8,6 @@ namespace DB { - namespace { class AutoFinalOnQueryPassVisitor : public InDepthQueryTreeVisitorWithContext @@ -30,7 +29,7 @@ namespace } private: - bool autoFinalOnQuery(TableNode & table_node, StoragePtr storage, ContextPtr context) + static bool autoFinalOnQuery(TableNode & table_node, StoragePtr storage, ContextPtr context) { // query.tables() is required because not all queries have tables in it, it could be a function. bool is_auto_final_setting_on = context->getSettingsRef().final; From 75e3faed851adf9939dcdb5a1a0f4758d4a5538b Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 2 Feb 2023 17:32:07 -0300 Subject: [PATCH 110/717] remove unnecessary comment --- src/Analyzer/Passes/AutoFinalOnQueryPass.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Analyzer/Passes/AutoFinalOnQueryPass.cpp b/src/Analyzer/Passes/AutoFinalOnQueryPass.cpp index 613a7b36261..10efebe0731 100644 --- a/src/Analyzer/Passes/AutoFinalOnQueryPass.cpp +++ b/src/Analyzer/Passes/AutoFinalOnQueryPass.cpp @@ -31,7 +31,6 @@ namespace private: static bool autoFinalOnQuery(TableNode & table_node, StoragePtr storage, ContextPtr context) { - // query.tables() is required because not all queries have tables in it, it could be a function. bool is_auto_final_setting_on = context->getSettingsRef().final; bool is_final_supported = storage && storage->supportsFinal() && !storage->isRemote(); bool is_query_already_final = table_node.hasTableExpressionModifiers() ? table_node.getTableExpressionModifiers().has_value() : false; From 15ddcdc4a6606f62b4333333144231300243260c Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 3 Feb 2023 12:34:47 +0100 Subject: [PATCH 111/717] Fix clang tidy --- src/Storages/StorageMergeTree.cpp | 1 - tests/queries/0_stateless/02538_alter_rename_sequence.sql | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 73585a630cb..e36bb31551d 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -1129,7 +1129,6 @@ MergeMutateSelectedEntryPtr StorageMergeTree::selectPartsToMutate( { if (commands->empty()) { - current_ast_elements += commands_size; commands->insert(commands->end(), it->second.commands.begin(), it->second.commands.end()); last_mutation_to_apply = it; } diff --git a/tests/queries/0_stateless/02538_alter_rename_sequence.sql b/tests/queries/0_stateless/02538_alter_rename_sequence.sql index 0eb839ebe59..d7df27dc702 100644 --- a/tests/queries/0_stateless/02538_alter_rename_sequence.sql +++ b/tests/queries/0_stateless/02538_alter_rename_sequence.sql @@ -5,7 +5,7 @@ CREATE TABLE wrong_metadata( column2 UInt64, column3 UInt64 ) -ENGINE ReplicatedMergeTree('/test/tables/wrong_metadata', '1') +ENGINE ReplicatedMergeTree('/test/{database}/tables/wrong_metadata', '1') ORDER BY tuple(); INSERT INTO wrong_metadata VALUES (1, 2, 3); @@ -34,7 +34,7 @@ CREATE TABLE wrong_metadata_wide( column2 UInt64, column3 UInt64 ) -ENGINE ReplicatedMergeTree('/test/tables/wrong_metadata_wide', '1') +ENGINE ReplicatedMergeTree('/test/{database}/tables/wrong_metadata_wide', '1') ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0; From 0f2b9612e3a183b4a8cabd0792fe758e51ac4f1c Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Fri, 3 Feb 2023 10:03:39 -0300 Subject: [PATCH 112/717] revert old architecture visitor until I get it working if needed --- src/Interpreters/AutoFinalOnQueryVisitor.cpp | 37 -------------------- src/Interpreters/AutoFinalOnQueryVisitor.h | 31 ---------------- src/Interpreters/InterpreterSelectQuery.cpp | 13 +++---- 3 files changed, 4 insertions(+), 77 deletions(-) delete mode 100644 src/Interpreters/AutoFinalOnQueryVisitor.cpp delete mode 100644 src/Interpreters/AutoFinalOnQueryVisitor.h diff --git a/src/Interpreters/AutoFinalOnQueryVisitor.cpp b/src/Interpreters/AutoFinalOnQueryVisitor.cpp deleted file mode 100644 index f2b3eb147c1..00000000000 --- a/src/Interpreters/AutoFinalOnQueryVisitor.cpp +++ /dev/null @@ -1,37 +0,0 @@ -#include "AutoFinalOnQueryVisitor.h" -#include -#include - -namespace DB -{ - -void AutoFinalOnQuery::visit(ASTPtr & query, Data & data) -{ - if (auto * select_query = query->as()) - visit(*select_query, data.storage, data.context); -} - -void AutoFinalOnQuery::visit(ASTSelectQuery & query, StoragePtr storage, ContextPtr context) -{ - if (autoFinalOnQuery(query, storage, context)) - { - query.setFinal(); - } -} - -bool AutoFinalOnQuery::needChildVisit(ASTPtr &, const ASTPtr &) -{ - return true; -} - -bool AutoFinalOnQuery::autoFinalOnQuery(ASTSelectQuery & query, StoragePtr storage, ContextPtr context) -{ - // query.tables() is required because not all queries have tables in it, it could be a function. - bool is_auto_final_setting_on = context->getSettingsRef().final; - bool is_final_supported = storage && storage->supportsFinal() && !storage->isRemote() && query.tables(); - bool is_query_already_final = query.final(); - - return is_auto_final_setting_on && !is_query_already_final && is_final_supported; -} - -} diff --git a/src/Interpreters/AutoFinalOnQueryVisitor.h b/src/Interpreters/AutoFinalOnQueryVisitor.h deleted file mode 100644 index 850fb9fc0e4..00000000000 --- a/src/Interpreters/AutoFinalOnQueryVisitor.h +++ /dev/null @@ -1,31 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -namespace DB -{ - -class AutoFinalOnQuery -{ -public: - struct Data - { - StoragePtr storage; - ContextPtr context; - }; - - static bool needChildVisit(ASTPtr &, const ASTPtr &); - static void visit(ASTPtr & query, Data & data); - -private: - static void visit(ASTSelectQuery & select, StoragePtr storage, ContextPtr context); - static bool autoFinalOnQuery(ASTSelectQuery & select_query, StoragePtr storage, ContextPtr context); - -}; - -using AutoFinalOnQueryVisitor = InDepthNodeVisitor; - -} diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 8114b7a5375..ae4ccc34f78 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -38,7 +38,6 @@ #include #include #include -#include #include #include @@ -504,14 +503,10 @@ InterpreterSelectQuery::InterpreterSelectQuery( query_info.additional_filter_ast = parseAdditionalFilterConditionForTable( settings.additional_table_filters, joined_tables.tablesWithColumns().front().table, *context); - AutoFinalOnQuery::Data abc{storage, context}; - - AutoFinalOnQueryVisitor(abc).visit(query_ptr); - -// if (autoFinalOnQuery(query)) -// { -// query.setFinal(); -// } + if (autoFinalOnQuery(query)) + { + query.setFinal(); + } auto analyze = [&] (bool try_move_to_prewhere) { From 82e50fc211b3abe08e8e2562bcf7144ea2779730 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Fri, 3 Feb 2023 15:12:20 +0000 Subject: [PATCH 113/717] Fix review comments --- src/Interpreters/ActionsDAG.cpp | 28 ---------------- src/Interpreters/ActionsDAG.h | 3 -- .../Optimizations/removeRedundantDistinct.cpp | 32 +++++++++++++++++-- 3 files changed, 30 insertions(+), 33 deletions(-) diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index 8bc430ee825..5f1398fed39 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -2260,32 +2260,4 @@ ActionsDAGPtr ActionsDAG::buildFilterActionsDAG( return result_dag; } -const ActionsDAG::Node * ActionsDAG::getOriginalNodeForOutputAlias(const String & output_name) -{ - /// find alias in output - const Node * output_alias = nullptr; - for (const auto * node : outputs) - { - if (node->result_name == output_name && node->type == ActionsDAG::ActionType::ALIAS) - { - output_alias = node; - break; - } - } - if (!output_alias) - return nullptr; - - /// find original(non alias) node it refers to - const Node * node = output_alias; - while (node && node->type == ActionsDAG::ActionType::ALIAS) - { - chassert(!node->children.empty()); - node = node->children.front(); - } - if (node->type != ActionsDAG::ActionType::INPUT) - return nullptr; - - return node; -} - } diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index a1d57db71d5..40bc76fe057 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -347,9 +347,6 @@ public: const std::unordered_map & node_name_to_input_node_column, const ContextPtr & context); - /// Return original node (input) for alias in output if exists - const Node * getOriginalNodeForOutputAlias(const String & output_name); - private: NodeRawConstPtrs getParents(const Node * target) const; diff --git a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp index aa235e9e8af..1b012d81f82 100644 --- a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp +++ b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp @@ -37,6 +37,34 @@ namespace return non_const_columns; } + const ActionsDAG::Node * getOriginalNodeForOutputAlias(const ActionsDAGPtr & actions, const String & output_name) + { + /// find alias in output + const ActionsDAG::Node * output_alias = nullptr; + for (const auto * node : actions->getOutputs()) + { + if (node->result_name == output_name && node->type == ActionsDAG::ActionType::ALIAS) + { + output_alias = node; + break; + } + } + if (!output_alias) + return nullptr; + + /// find original(non alias) node it refers to + const ActionsDAG::Node * node = output_alias; + while (node && node->type == ActionsDAG::ActionType::ALIAS) + { + chassert(!node->children.empty()); + node = node->children.front(); + } + if (node->type != ActionsDAG::ActionType::INPUT) + return nullptr; + + return node; + } + bool canRemoveDistinct(const QueryPlan::Node * distinct_node) { const DistinctStep * distinct_step = typeid_cast(distinct_node->step.get()); @@ -98,7 +126,7 @@ namespace /// compare columns of two DISTINCTs for (const auto & column : distinct_columns) { - const auto * alias_node = path_actions->getOriginalNodeForOutputAlias(String(column)); + const auto * alias_node = getOriginalNodeForOutputAlias(path_actions, String(column)); if (!alias_node) return false; @@ -129,7 +157,7 @@ size_t tryRemoveRedundantDistinct(QueryPlan::Node * parent_node, QueryPlan::Node for (const auto * node : parent_node->children) { /// check if it is distinct node - if (typeid_cast(node->step.get())) + if (typeid_cast(node->step.get()) == nullptr) continue; if (canRemoveDistinct(node)) From 6f47c36389dc0c761c880ea221e91d165032ebcd Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Fri, 3 Feb 2023 14:31:41 -0300 Subject: [PATCH 114/717] fix test --- tests/queries/0_stateless/02420_final_setting_analyzer.reference | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/queries/0_stateless/02420_final_setting_analyzer.reference b/tests/queries/0_stateless/02420_final_setting_analyzer.reference index dfc64fa39a1..2f8bc48fc65 100644 --- a/tests/queries/0_stateless/02420_final_setting_analyzer.reference +++ b/tests/queries/0_stateless/02420_final_setting_analyzer.reference @@ -86,7 +86,6 @@ FROM val_middle, id FROM middle_table - FINAL ) AS middle_table ON `--left_table.id` = `--middle_table.id` ) AS `--.s` ALL INNER JOIN From 7a0d0f0c5324cddce2131a2af6f6e5535db65f30 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 3 Feb 2023 19:14:49 +0100 Subject: [PATCH 115/717] Some code which doesn't work --- src/Storages/MergeTree/MergeTreeData.cpp | 42 +++++-- src/Storages/MergeTree/MergeTreeData.h | 2 +- src/Storages/MergeTree/MutateTask.cpp | 116 ++++++++++-------- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 28 +++-- .../MergeTree/ReplicatedMergeTreeQueue.h | 2 +- src/Storages/StorageMergeTree.cpp | 6 +- src/Storages/StorageMergeTree.h | 2 +- src/Storages/StorageReplicatedMergeTree.cpp | 2 +- src/Storages/StorageReplicatedMergeTree.h | 2 +- .../01278_alter_rename_combination.reference | 4 +- .../01278_alter_rename_combination.sql | 2 +- 11 files changed, 128 insertions(+), 80 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index c8b1020bd82..27b9881bcf3 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -7522,16 +7522,44 @@ bool MergeTreeData::canUsePolymorphicParts(const MergeTreeSettings & settings, S AlterConversions MergeTreeData::getAlterConversionsForPart(const MergeTreeDataPartPtr part) const { - MutationCommands commands = getAlterMutationCommandsForPart(part); + std::map commands_map = getAlterMutationCommandsForPart(part); + auto part_columns = part->getColumnsDescription(); AlterConversions result{}; - for (const auto & command : commands) + auto & rename_map = result.rename_map; + for (const auto & [version, commands] : commands_map) { - /// Currently we need explicit conversions only for RENAME alter - /// all other conversions can be deduced from diff between part columns - /// and columns in storage. - if (command.type == MutationCommand::Type::RENAME_COLUMN) - result.rename_map[command.rename_to] = command.column_name; + for (const auto & command : commands) + { + /// Currently we need explicit conversions only for RENAME alter + /// all other conversions can be deduced from diff between part columns + /// and columns in storage. + if (command.type == MutationCommand::Type::RENAME_COLUMN) + { + if (!part_columns.has(command.column_name)) + continue; + + part_columns.rename(command.column_name, command.rename_to); + + if (auto it = rename_map.find(command.column_name); it != rename_map.end()) + { + auto rename_source = it->second; + rename_map.erase(it); + + rename_map[command.rename_to] = rename_source; + } + else + rename_map[command.rename_to] = command.column_name; + } + } + } + + for (auto it = rename_map.begin(); it != rename_map.end();) + { + if (it->first == it->second) + it = rename_map.erase(it); + else + ++it; } return result; diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 0dfeb44524e..a60f4721661 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -1303,7 +1303,7 @@ protected: /// Used to receive AlterConversions for part and apply them on fly. This /// method has different implementations for replicated and non replicated /// MergeTree because they store mutations in different way. - virtual MutationCommands getAlterMutationCommandsForPart(const DataPartPtr & part) const = 0; + virtual std::map getAlterMutationCommandsForPart(const DataPartPtr & part) const = 0; /// Moves part to specified space, used in ALTER ... MOVE ... queries bool movePartsToSpace(const DataPartsVector & parts, SpacePtr space); diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index 57036d4ba30..85dc9f7f0d6 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -97,40 +97,33 @@ static void splitMutationCommands( else mutated_columns.emplace(command.column_name); } - - if (command.type == MutationCommand::Type::RENAME_COLUMN) - { - for_interpreter.push_back( - { - .type = MutationCommand::Type::READ_COLUMN, - .column_name = command.rename_to, - }); - part_columns.rename(command.column_name, command.rename_to); - } } } auto alter_conversions = part->storage.getAlterConversionsForPart(part); /// If it's compact part, then we don't need to actually remove files /// from disk we just don't read dropped columns - for (const auto & column : part->getColumns()) + + for (const auto & [rename_to, rename_from] : alter_conversions.rename_map) + { + if (part_columns.has(rename_from)) + { + for_interpreter.push_back( + { + .type = MutationCommand::Type::READ_COLUMN, + .column_name = rename_to, + }); + + part_columns.rename(rename_from, rename_to); + } + } + + for (const auto & column : part_columns) { if (!mutated_columns.contains(column.name)) { - if (alter_conversions.columnHasNewName(column.name)) - { - for_interpreter.push_back( - { - .type = MutationCommand::Type::READ_COLUMN, - .column_name = alter_conversions.getColumnNewName(column.name), - .data_type = column.type - }); - } - else - { - for_interpreter.emplace_back( - MutationCommand{.type = MutationCommand::Type::READ_COLUMN, .column_name = column.name, .data_type = column.type}); - } + for_interpreter.emplace_back( + MutationCommand{.type = MutationCommand::Type::READ_COLUMN, .column_name = column.name, .data_type = column.type}); } } } @@ -157,22 +150,18 @@ static void splitMutationCommands( { if (command.type == MutationCommand::Type::READ_COLUMN) for_interpreter.push_back(command); - else if (command.type == MutationCommand::Type::RENAME_COLUMN) - part_columns.rename(command.column_name, command.rename_to); for_file_renames.push_back(command); } } auto alter_conversions = part->storage.getAlterConversionsForPart(part); - for (const auto & part_column : part_columns) + for (const auto & [rename_to, rename_from] : alter_conversions.rename_map) { - if (alter_conversions.columnHasNewName(part_column.name)) + if (part_columns.has(rename_from)) { - auto new_column_name = alter_conversions.getColumnNewName(part_column.name); - for_file_renames.push_back({.type = MutationCommand::Type::RENAME_COLUMN, .column_name = part_column.name, .rename_to = new_column_name}); - - part_columns.rename(part_column.name, new_column_name); + for_file_renames.push_back({.type = MutationCommand::Type::RENAME_COLUMN, .column_name = rename_from, .rename_to = rename_to}); + part_columns.rename(rename_from, rename_to); } } } @@ -585,6 +574,13 @@ static NameToNameVector collectFilesForRenames( /// Collect counts for shared streams of different columns. As an example, Nested columns have shared stream with array sizes. auto stream_counts = getStreamCounts(source_part, source_part->getColumns().getNames()); NameToNameVector rename_vector; + NameSet collected_names; + + auto add_rename = [&rename_vector, &collected_names] (const std::string & file_rename_from, const std::string & file_rename_to) + { + if (collected_names.emplace(file_rename_from).second) + rename_vector.emplace_back(file_rename_from, file_rename_to); + }; /// Remove old data for (const auto & command : commands_for_removes) @@ -593,19 +589,19 @@ static NameToNameVector collectFilesForRenames( { if (source_part->checksums.has(INDEX_FILE_PREFIX + command.column_name + ".idx2")) { - rename_vector.emplace_back(INDEX_FILE_PREFIX + command.column_name + ".idx2", ""); - rename_vector.emplace_back(INDEX_FILE_PREFIX + command.column_name + mrk_extension, ""); + add_rename(INDEX_FILE_PREFIX + command.column_name + ".idx2", ""); + add_rename(INDEX_FILE_PREFIX + command.column_name + mrk_extension, ""); } else if (source_part->checksums.has(INDEX_FILE_PREFIX + command.column_name + ".idx")) { - rename_vector.emplace_back(INDEX_FILE_PREFIX + command.column_name + ".idx", ""); - rename_vector.emplace_back(INDEX_FILE_PREFIX + command.column_name + mrk_extension, ""); + add_rename(INDEX_FILE_PREFIX + command.column_name + ".idx", ""); + add_rename(INDEX_FILE_PREFIX + command.column_name + mrk_extension, ""); } } else if (command.type == MutationCommand::Type::DROP_PROJECTION) { if (source_part->checksums.has(command.column_name + ".proj")) - rename_vector.emplace_back(command.column_name + ".proj", ""); + add_rename(command.column_name + ".proj", ""); } else if (command.type == MutationCommand::Type::DROP_COLUMN) { @@ -615,8 +611,8 @@ static NameToNameVector collectFilesForRenames( /// Delete files if they are no longer shared with another column. if (--stream_counts[stream_name] == 0) { - rename_vector.emplace_back(stream_name + ".bin", ""); - rename_vector.emplace_back(stream_name + mrk_extension, ""); + add_rename(stream_name + ".bin", ""); + add_rename(stream_name + mrk_extension, ""); } }; @@ -628,6 +624,7 @@ static NameToNameVector collectFilesForRenames( String escaped_name_from = escapeForFileName(command.column_name); String escaped_name_to = escapeForFileName(command.rename_to); + ISerialization::StreamCallback callback = [&](const ISerialization::SubstreamPath & substream_path) { String stream_from = ISerialization::getFileNameForStream(command.column_name, substream_path); @@ -635,8 +632,8 @@ static NameToNameVector collectFilesForRenames( if (stream_from != stream_to) { - rename_vector.emplace_back(stream_from + ".bin", stream_to + ".bin"); - rename_vector.emplace_back(stream_from + mrk_extension, stream_to + mrk_extension); + add_rename(stream_from + ".bin", stream_to + ".bin"); + add_rename(stream_from + mrk_extension, stream_to + mrk_extension); } }; @@ -656,8 +653,8 @@ static NameToNameVector collectFilesForRenames( { if (!new_streams.contains(old_stream) && --stream_counts[old_stream] == 0) { - rename_vector.emplace_back(old_stream + ".bin", ""); - rename_vector.emplace_back(old_stream + mrk_extension, ""); + add_rename(old_stream + ".bin", ""); + add_rename(old_stream + mrk_extension, ""); } } } @@ -1358,13 +1355,27 @@ private: ctx->new_data_part->storeVersionMetadata(); NameSet hardlinked_files; + + /// NOTE: Renames must be done in order + for (const auto & [rename_from, rename_to] : ctx->files_to_rename) + { + if (rename_to.empty()) /// It's DROP COLUMN + { + /// pass + } + else + { + ctx->new_data_part->getDataPartStorage().createHardLinkFrom( + ctx->source_part->getDataPartStorage(), rename_from, rename_to); + hardlinked_files.insert(rename_from); + } + } /// Create hardlinks for unchanged files for (auto it = ctx->source_part->getDataPartStorage().iterate(); it->isValid(); it->next()) { if (ctx->files_to_skip.contains(it->name())) continue; - String destination; String file_name = it->name(); auto rename_it = std::find_if(ctx->files_to_rename.begin(), ctx->files_to_rename.end(), [&file_name](const auto & rename_pair) @@ -1374,20 +1385,17 @@ private: if (rename_it != ctx->files_to_rename.end()) { - if (rename_it->second.empty()) - continue; - destination = rename_it->second; - } - else - { - destination = it->name(); + /// RENAMEs and DROPs already processed + continue; } + String destination = it->name(); + if (it->isFile()) { ctx->new_data_part->getDataPartStorage().createHardLinkFrom( - ctx->source_part->getDataPartStorage(), it->name(), destination); - hardlinked_files.insert(it->name()); + ctx->source_part->getDataPartStorage(), file_name, destination); + hardlinked_files.insert(file_name); } else if (!endsWith(it->name(), ".tmp_proj")) // ignore projection tmp merge dir { diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 2c957132918..df4dea978a1 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -11,6 +11,7 @@ #include #include +#include namespace DB { @@ -1752,23 +1753,34 @@ ReplicatedMergeTreeMergePredicate ReplicatedMergeTreeQueue::getMergePredicate(zk } -MutationCommands ReplicatedMergeTreeQueue::getAlterMutationCommandsForPart(const MergeTreeData::DataPartPtr & part) const +std::map ReplicatedMergeTreeQueue::getAlterMutationCommandsForPart(const MergeTreeData::DataPartPtr & part) const { - std::lock_guard lock(state_mutex); + std::unique_lock lock(state_mutex); auto in_partition = mutations_by_partition.find(part->info.partition_id); if (in_partition == mutations_by_partition.end()) - return MutationCommands{}; + return {}; - Int64 part_mutation_version = part->info.getMutationVersion(); - MutationCommands result; + Int64 part_data_version = part->info.getDataVersion(); + std::map result; /// Here we return mutation commands for part which has bigger mutation version than part mutation version. /// Please note, we don't use getDataVersion(). It's because these alter commands are used for in-fly conversions /// of part's metadata. It mean that even if we have mutation with version X and part with data version X+10, but /// without mutation version part can still have wrong metadata and we have to apply this change on-fly if needed. - for (auto [mutation_version, mutation_status] : in_partition->second) + + for (auto [mutation_version, mutation_status] : in_partition->second | std::views::reverse) { - if (mutation_version > part_mutation_version && mutation_status->entry->alter_version != -1) - result.insert(result.end(), mutation_status->entry->commands.begin(), mutation_status->entry->commands.end()); + if (mutation_status->entry->alter_version != -1) + { + if (mutation_version > part_data_version) + { + result[mutation_version] = mutation_status->entry->commands; + } + else + { + result[mutation_version] = mutation_status->entry->commands; + break; + } + } } return result; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h index 2c566668877..150fb49b333 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h @@ -396,7 +396,7 @@ public: /// Return mutation commands for part which could be not applied to /// it according to part mutation version. Used when we apply alter commands on fly, /// without actual data modification on disk. - MutationCommands getAlterMutationCommandsForPart(const MergeTreeData::DataPartPtr & part) const; + std::map getAlterMutationCommandsForPart(const MergeTreeData::DataPartPtr & part) const; /// Mark finished mutations as done. If the function needs to be called again at some later time /// (because some mutations are probably done but we are not sure yet), returns true. diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index e36bb31551d..b0a45f9cfba 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -2080,17 +2080,17 @@ void StorageMergeTree::attachRestoredParts(MutableDataPartsVector && parts) } -MutationCommands StorageMergeTree::getAlterMutationCommandsForPart(const DataPartPtr & part) const +std::map StorageMergeTree::getAlterMutationCommandsForPart(const DataPartPtr & part) const { std::lock_guard lock(currently_processing_in_background_mutex); Int64 part_mutation_version = part->info.getMutationVersion(); - MutationCommands result; + std::map result; for (const auto & current_mutation_by_version : current_mutations_by_version) { if (static_cast(current_mutation_by_version.first) > part_mutation_version) - result.insert(result.end(), current_mutation_by_version.second.commands.begin(), current_mutation_by_version.second.commands.end()); + result[current_mutation_by_version.first] = current_mutation_by_version.second.commands; } return result; } diff --git a/src/Storages/StorageMergeTree.h b/src/Storages/StorageMergeTree.h index 957d7804b96..9385542dcac 100644 --- a/src/Storages/StorageMergeTree.h +++ b/src/Storages/StorageMergeTree.h @@ -265,7 +265,7 @@ private: protected: - MutationCommands getAlterMutationCommandsForPart(const DataPartPtr & part) const override; + std::map getAlterMutationCommandsForPart(const DataPartPtr & part) const override; }; } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 58adc4be594..ef2fd46b705 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -7892,7 +7892,7 @@ bool StorageReplicatedMergeTree::canUseAdaptiveGranularity() const } -MutationCommands StorageReplicatedMergeTree::getAlterMutationCommandsForPart(const DataPartPtr & part) const +std::map StorageReplicatedMergeTree::getAlterMutationCommandsForPart(const DataPartPtr & part) const { return queue.getAlterMutationCommandsForPart(part); } diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 3dc5ab75891..a4ac60052bc 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -830,7 +830,7 @@ private: void waitMutationToFinishOnReplicas( const Strings & replicas, const String & mutation_id) const; - MutationCommands getAlterMutationCommandsForPart(const DataPartPtr & part) const override; + std::map getAlterMutationCommandsForPart(const DataPartPtr & part) const override; void startBackgroundMovesIfNeeded() override; diff --git a/tests/queries/0_stateless/01278_alter_rename_combination.reference b/tests/queries/0_stateless/01278_alter_rename_combination.reference index cc912e9b265..e70c2d2e6f8 100644 --- a/tests/queries/0_stateless/01278_alter_rename_combination.reference +++ b/tests/queries/0_stateless/01278_alter_rename_combination.reference @@ -1,7 +1,7 @@ -CREATE TABLE default.rename_table\n(\n `key` Int32,\n `old_value1` Int32,\n `value1` Int32\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.rename_table\n(\n `key` Int32,\n `old_value1` Int32,\n `value1` Int32\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS min_bytes_for_wide_part = 0, index_granularity = 8192 key old_value1 value1 1 2 3 -CREATE TABLE default.rename_table\n(\n `k` Int32,\n `v1` Int32,\n `v2` Int32\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.rename_table\n(\n `k` Int32,\n `v1` Int32,\n `v2` Int32\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS min_bytes_for_wide_part = 0, index_granularity = 8192 k v1 v2 1 2 3 4 5 6 diff --git a/tests/queries/0_stateless/01278_alter_rename_combination.sql b/tests/queries/0_stateless/01278_alter_rename_combination.sql index fa73362622c..51322f5d86f 100644 --- a/tests/queries/0_stateless/01278_alter_rename_combination.sql +++ b/tests/queries/0_stateless/01278_alter_rename_combination.sql @@ -1,6 +1,6 @@ DROP TABLE IF EXISTS rename_table; -CREATE TABLE rename_table (key Int32, value1 Int32, value2 Int32) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE rename_table (key Int32, value1 Int32, value2 Int32) ENGINE = MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part=0; INSERT INTO rename_table VALUES (1, 2, 3); From f91c10d09d56de8d23faf326c5c300de1ee20c2a Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 3 Feb 2023 20:49:07 +0100 Subject: [PATCH 116/717] Initial support for version --- src/Storages/MergeTree/DataPartsExchange.cpp | 3 +- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 26 ++++++++++---- src/Storages/MergeTree/IMergeTreeDataPart.h | 7 +++- src/Storages/MergeTree/MergeTask.cpp | 2 +- src/Storages/MergeTree/MergeTreeData.cpp | 3 +- .../MergeTree/MergeTreeDataPartInMemory.cpp | 4 +-- .../MergeTree/MergeTreeDataWriter.cpp | 4 +-- .../MergeTree/MergeTreeWriteAheadLog.cpp | 2 +- .../MergeTree/MergedBlockOutputStream.cpp | 10 +++++- .../MergedColumnOnlyOutputStream.cpp | 2 +- src/Storages/MergeTree/MutateTask.cpp | 2 +- .../ReplicatedMergeTreeAttachThread.cpp | 4 ++- src/Storages/StorageInMemoryMetadata.cpp | 7 ++++ src/Storages/StorageInMemoryMetadata.h | 7 ++++ src/Storages/StorageReplicatedMergeTree.cpp | 35 +++++++++++-------- src/Storages/StorageReplicatedMergeTree.h | 9 +++-- 16 files changed, 89 insertions(+), 38 deletions(-) diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index cbbdb911974..1864ce37ddf 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -692,7 +692,8 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToMemory( auto block = block_in.read(); throttler->add(block.bytes()); - new_data_part->setColumns(block.getNamesAndTypesList(), {}); + //// TODO Read them from file + new_data_part->setColumns(block.getNamesAndTypesList(), {}, metadata_snapshot->getMetadataVersion()); if (!is_projection) { diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 5de13020a1d..3424ef5bf0a 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -416,10 +416,11 @@ std::pair IMergeTreeDataPart::getMinMaxTime() const } -void IMergeTreeDataPart::setColumns(const NamesAndTypesList & new_columns, const SerializationInfoByName & new_infos) +void IMergeTreeDataPart::setColumns(const NamesAndTypesList & new_columns, const SerializationInfoByName & new_infos, int32_t metadata_version_) { columns = new_columns; serialization_infos = new_infos; + metadata_version = metadata_version_; column_name_to_position.clear(); column_name_to_position.reserve(new_columns.size()); @@ -798,6 +799,9 @@ NameSet IMergeTreeDataPart::getFileNamesWithoutChecksums() const if (getDataPartStorage().exists(TXN_VERSION_METADATA_FILE_NAME)) result.emplace(TXN_VERSION_METADATA_FILE_NAME); + if (getDataPartStorage().exists(METADATA_VERSION_FILE_NAME)) + result.emplace(METADATA_VERSION_FILE_NAME); + return result; } @@ -1288,8 +1292,7 @@ void IMergeTreeDataPart::loadColumns(bool require) metadata_snapshot = metadata_snapshot->projections.get(name).metadata; NamesAndTypesList loaded_columns; - bool exists = metadata_manager->exists("columns.txt"); - if (!exists) + if (!metadata_manager->exists("columns.txt")) { /// We can get list of columns only from columns.txt in compact parts. if (require || part_type == Type::Compact) @@ -1322,14 +1325,25 @@ void IMergeTreeDataPart::loadColumns(bool require) }; SerializationInfoByName infos(loaded_columns, settings); - exists = metadata_manager->exists(SERIALIZATION_FILE_NAME); - if (exists) + if (metadata_manager->exists(SERIALIZATION_FILE_NAME)) { auto in = metadata_manager->read(SERIALIZATION_FILE_NAME); infos.readJSON(*in); } - setColumns(loaded_columns, infos); + int32_t loaded_metadata_version; + if (metadata_manager->exists(METADATA_VERSION_FILE_NAME)) + { + auto in = metadata_manager->read(METADATA_VERSION_FILE_NAME); + readIntText(loaded_metadata_version, *in); + } + else + { + loaded_metadata_version = metadata_snapshot->getMetadataVersion(); + } + + ///TODO read metadata here + setColumns(loaded_columns, infos, loaded_metadata_version); } /// Project part / part with project parts / compact part doesn't support LWD. diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index 9d0252bd625..df83d138322 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -139,7 +139,8 @@ public: String getTypeName() const { return getType().toString(); } - void setColumns(const NamesAndTypesList & new_columns, const SerializationInfoByName & new_infos); + void setColumns(const NamesAndTypesList & new_columns, const SerializationInfoByName & new_infos, int32_t metadata_version_); + int32_t getMetadataVersion() const { return metadata_version; } const NamesAndTypesList & getColumns() const { return columns; } const ColumnsDescription & getColumnsDescription() const { return columns_description; } @@ -310,6 +311,8 @@ public: mutable VersionMetadata version; + int32_t metadata_version; + /// For data in RAM ('index') UInt64 getIndexSizeInBytes() const; UInt64 getIndexSizeInAllocatedBytes() const; @@ -383,6 +386,8 @@ public: static inline constexpr auto TXN_VERSION_METADATA_FILE_NAME = "txn_version.txt"; + static inline constexpr auto METADATA_VERSION_FILE_NAME = "metadata_version.txt"; + /// One of part files which is used to check how many references (I'd like /// to say hardlinks, but it will confuse even more) we have for the part /// for zero copy replication. Sadly it's very complex. diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index 5874c257ad0..97bb9099c25 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -205,7 +205,7 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() infos.add(part->getSerializationInfos()); } - global_ctx->new_data_part->setColumns(global_ctx->storage_columns, infos); + global_ctx->new_data_part->setColumns(global_ctx->storage_columns, infos, global_ctx->metadata_snapshot->getMetadataVersion()); const auto & local_part_min_ttl = global_ctx->new_data_part->ttl_infos.part_min_ttl; if (local_part_min_ttl && local_part_min_ttl <= global_ctx->time_of_merge) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 27b9881bcf3..b35d58688d5 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -7548,6 +7548,7 @@ AlterConversions MergeTreeData::getAlterConversionsForPart(const MergeTreeDataPa rename_map[command.rename_to] = rename_source; } + else rename_map[command.rename_to] = command.column_name; } @@ -7935,7 +7936,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::createEmptyPart( if (settings->assign_part_uuids) new_data_part->uuid = UUIDHelpers::generateV4(); - new_data_part->setColumns(columns, {}); + new_data_part->setColumns(columns, {}, metadata_snapshot->getMetadataVersion()); new_data_part->rows_count = block.rows(); new_data_part->partition = partition; diff --git a/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp b/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp index 20049976acf..5b1054d0a0e 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp @@ -73,7 +73,7 @@ MutableDataPartStoragePtr MergeTreeDataPartInMemory::flushToDisk(const String & new_data_part_storage->beginTransaction(); new_data_part->uuid = uuid; - new_data_part->setColumns(columns, {}); + new_data_part->setColumns(columns, {}, metadata_snapshot->getMetadataVersion()); new_data_part->partition.value = partition.value; new_data_part->minmax_idx = minmax_idx; @@ -104,7 +104,7 @@ MutableDataPartStoragePtr MergeTreeDataPartInMemory::flushToDisk(const String & .build(); new_projection_part->is_temp = false; // clean up will be done on parent part - new_projection_part->setColumns(projection->getColumns(), {}); + new_projection_part->setColumns(projection->getColumns(), {}, metadata_snapshot->getMetadataVersion()); auto new_projection_part_storage = new_projection_part->getDataPartStoragePtr(); if (new_projection_part_storage->exists()) diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index 93b0abeca35..ba344dc70aa 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -465,7 +465,7 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeTempPartImpl( SerializationInfoByName infos(columns, settings); infos.add(block); - new_data_part->setColumns(columns, infos); + new_data_part->setColumns(columns, infos, metadata_snapshot->getMetadataVersion()); new_data_part->rows_count = block.rows(); new_data_part->partition = std::move(partition); new_data_part->minmax_idx = std::move(minmax_idx); @@ -587,7 +587,7 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeProjectionPartImpl( SerializationInfoByName infos(columns, settings); infos.add(block); - new_data_part->setColumns(columns, infos); + new_data_part->setColumns(columns, infos, metadata_snapshot->getMetadataVersion()); if (new_data_part->isStoredOnDisk()) { diff --git a/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp b/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp index fabf2acdad3..87d9bb0f168 100644 --- a/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp +++ b/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp @@ -229,7 +229,7 @@ MergeTreeData::MutableDataPartsVector MergeTreeWriteAheadLog::restore( part->minmax_idx->update(block, storage.getMinMaxColumnsNames(metadata_snapshot->getPartitionKey())); part->partition.create(metadata_snapshot, block, 0, context); - part->setColumns(block.getNamesAndTypesList(), {}); + part->setColumns(block.getNamesAndTypesList(), {}, metadata_snapshot->getMetadataVersion()); if (metadata_snapshot->hasSortingKey()) metadata_snapshot->getSortingKey().expression->execute(block); diff --git a/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/src/Storages/MergeTree/MergedBlockOutputStream.cpp index ced43ae25b0..53d4a32fc0e 100644 --- a/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -175,7 +175,7 @@ MergedBlockOutputStream::Finalizer MergedBlockOutputStream::finalizePartAsync( serialization_infos.replaceData(new_serialization_infos); files_to_remove_after_sync = removeEmptyColumnsFromPart(new_part, part_columns, serialization_infos, checksums); - new_part->setColumns(part_columns, serialization_infos); + new_part->setColumns(part_columns, serialization_infos, metadata_snapshot->getMetadataVersion()); } auto finalizer = std::make_unique(*writer, new_part, files_to_remove_after_sync, sync); @@ -289,6 +289,14 @@ MergedBlockOutputStream::WrittenFiles MergedBlockOutputStream::finalizePartOnDis written_files.emplace_back(std::move(out)); } + { + /// Write a file with a description of columns. + auto out = new_part->getDataPartStorage().writeFile(IMergeTreeDataPart::METADATA_VERSION_FILE_NAME, 4096, write_settings); + DB::writeIntText(new_part->getMetadataVersion(), *out); + out->preFinalize(); + written_files.emplace_back(std::move(out)); + } + if (default_codec != nullptr) { auto out = new_part->getDataPartStorage().writeFile(IMergeTreeDataPart::DEFAULT_COMPRESSION_CODEC_FILE_NAME, 4096, write_settings); diff --git a/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp b/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp index 03829f1daf9..c70c5187b8b 100644 --- a/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp +++ b/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp @@ -85,7 +85,7 @@ MergedColumnOnlyOutputStream::fillChecksums( all_checksums.files.erase(removed_file); } - new_part->setColumns(columns, serialization_infos); + new_part->setColumns(columns, serialization_infos, metadata_snapshot->getMetadataVersion()); return checksums; } diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index 85dc9f7f0d6..4c1ca86f5cb 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -1700,7 +1700,7 @@ bool MutateTask::prepare() ctx->source_part, ctx->updated_header, ctx->storage_columns, ctx->source_part->getSerializationInfos(), ctx->commands_for_part); - ctx->new_data_part->setColumns(new_columns, new_infos); + ctx->new_data_part->setColumns(new_columns, new_infos, ctx->metadata_snapshot->getMetadataVersion()); ctx->new_data_part->partition.assign(ctx->source_part->partition); /// Don't change granularity type while mutating subset of columns diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp index 557123ddae2..271edb0399f 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp @@ -149,7 +149,9 @@ void ReplicatedMergeTreeAttachThread::runImpl() const bool replica_metadata_version_exists = zookeeper->tryGet(replica_path + "/metadata_version", replica_metadata_version); if (replica_metadata_version_exists) { - storage.metadata_version = parse(replica_metadata_version); + StorageInMemoryMetadata metadata_with_new_version; + metadata_with_new_version.setMetadataVersion(parse(replica_metadata_version)); + storage.setInMemoryMetadata(metadata_with_new_version); } else { diff --git a/src/Storages/StorageInMemoryMetadata.cpp b/src/Storages/StorageInMemoryMetadata.cpp index f6550c6cd5d..5250c6f5330 100644 --- a/src/Storages/StorageInMemoryMetadata.cpp +++ b/src/Storages/StorageInMemoryMetadata.cpp @@ -41,6 +41,7 @@ StorageInMemoryMetadata::StorageInMemoryMetadata(const StorageInMemoryMetadata & , settings_changes(other.settings_changes ? other.settings_changes->clone() : nullptr) , select(other.select) , comment(other.comment) + , metadata_version(other.metadata_version) { } @@ -69,6 +70,7 @@ StorageInMemoryMetadata & StorageInMemoryMetadata::operator=(const StorageInMemo settings_changes.reset(); select = other.select; comment = other.comment; + metadata_version = other.metadata_version; return *this; } @@ -122,6 +124,11 @@ void StorageInMemoryMetadata::setSelectQuery(const SelectQueryDescription & sele select = select_; } +void StorageInMemoryMetadata::setMetadataVersion(int32_t metadata_version_) +{ + metadata_version = metadata_version_; +} + const ColumnsDescription & StorageInMemoryMetadata::getColumns() const { return columns; diff --git a/src/Storages/StorageInMemoryMetadata.h b/src/Storages/StorageInMemoryMetadata.h index eadce581334..2dd6c32e3c6 100644 --- a/src/Storages/StorageInMemoryMetadata.h +++ b/src/Storages/StorageInMemoryMetadata.h @@ -50,6 +50,8 @@ struct StorageInMemoryMetadata String comment; + int32_t metadata_version; + StorageInMemoryMetadata() = default; StorageInMemoryMetadata(const StorageInMemoryMetadata & other); @@ -90,6 +92,8 @@ struct StorageInMemoryMetadata /// Set SELECT query for (Materialized)View void setSelectQuery(const SelectQueryDescription & select_); + void setMetadataVersion(int32_t metadata_version_); + /// Returns combined set of columns const ColumnsDescription & getColumns() const; @@ -218,6 +222,9 @@ struct StorageInMemoryMetadata const SelectQueryDescription & getSelectQuery() const; bool hasSelectQuery() const; + int32_t getMetadataVersion() const { return metadata_version; } + bool hasMetadataVersion() const { return metadata_version != -1; } + /// Check that all the requested names are in the table and have the correct types. void check(const NamesAndTypesList & columns) const; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index ef2fd46b705..3083fbd2c46 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -452,7 +452,10 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( Coordination::Stat metadata_stat; current_zookeeper->get(zookeeper_path + "/metadata", &metadata_stat); - metadata_version = metadata_stat.version; + + StorageInMemoryMetadata storage_metadata; + storage_metadata.setMetadataVersion(metadata_stat.version); + setInMemoryMetadata(storage_metadata); } catch (Coordination::Exception & e) { @@ -772,7 +775,7 @@ bool StorageReplicatedMergeTree::createTableIfNotExists(const StorageMetadataPtr zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest(replica_path + "/columns", metadata_snapshot->getColumns().toString(), zkutil::CreateMode::Persistent)); - ops.emplace_back(zkutil::makeCreateRequest(replica_path + "/metadata_version", std::to_string(metadata_version), + ops.emplace_back(zkutil::makeCreateRequest(replica_path + "/metadata_version", toString(metadata_snapshot->getMetadataVersion()), zkutil::CreateMode::Persistent)); /// The following 3 nodes were added in version 1.1.xxx, so we create them here, not in createNewZooKeeperNodes() @@ -845,7 +848,7 @@ void StorageReplicatedMergeTree::createReplica(const StorageMetadataPtr & metada zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest(replica_path + "/columns", metadata_snapshot->getColumns().toString(), zkutil::CreateMode::Persistent)); - ops.emplace_back(zkutil::makeCreateRequest(replica_path + "/metadata_version", std::to_string(metadata_version), + ops.emplace_back(zkutil::makeCreateRequest(replica_path + "/metadata_version", toString(metadata_snapshot->getMetadataVersion()), zkutil::CreateMode::Persistent)); /// The following 3 nodes were added in version 1.1.xxx, so we create them here, not in createNewZooKeeperNodes() @@ -1140,16 +1143,19 @@ void StorageReplicatedMergeTree::checkTableStructure(const String & zookeeper_pr } void StorageReplicatedMergeTree::setTableStructure(const StorageID & table_id, const ContextPtr & local_context, - ColumnsDescription new_columns, const ReplicatedMergeTreeTableMetadata::Diff & metadata_diff) + ColumnsDescription new_columns, const ReplicatedMergeTreeTableMetadata::Diff & metadata_diff, int32_t new_metadata_version) { StorageInMemoryMetadata old_metadata = getInMemoryMetadata(); + StorageInMemoryMetadata new_metadata = metadata_diff.getNewMetadata(new_columns, local_context, old_metadata); + new_metadata.setMetadataVersion(new_metadata_version); /// Even if the primary/sorting/partition keys didn't change we must reinitialize it /// because primary/partition key column types might have changed. checkTTLExpressions(new_metadata, old_metadata); setProperties(new_metadata, old_metadata); + DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(local_context, table_id, new_metadata); } @@ -2760,8 +2766,9 @@ void StorageReplicatedMergeTree::cloneMetadataIfNeeded(const String & source_rep return; } + auto metadata_snapshot = getInMemoryMetadataPtr(); Int32 source_metadata_version = parse(source_metadata_version_str); - if (metadata_version == source_metadata_version) + if (metadata_snapshot->getMetadataVersion() == source_metadata_version) return; /// Our metadata it not up to date with source replica metadata. @@ -2779,7 +2786,7 @@ void StorageReplicatedMergeTree::cloneMetadataIfNeeded(const String & source_rep /// if all such entries were cleaned up from the log and source_queue. LOG_WARNING(log, "Metadata version ({}) on replica is not up to date with metadata ({}) on source replica {}", - metadata_version, source_metadata_version, source_replica); + metadata_snapshot->getMetadataVersion(), source_metadata_version, source_replica); String source_metadata; String source_columns; @@ -4926,14 +4933,15 @@ bool StorageReplicatedMergeTree::optimize( bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMergeTree::LogEntry & entry) { - if (entry.alter_version < metadata_version) + auto current_metadata = getInMemoryMetadataPtr(); + if (entry.alter_version < current_metadata->getMetadataVersion()) { /// TODO Can we replace it with LOGICAL_ERROR? /// As for now, it may rarely happen due to reordering of ALTER_METADATA entries in the queue of /// non-initial replica and also may happen after stale replica recovery. LOG_WARNING(log, "Attempt to update metadata of version {} " "to older version {} when processing log entry {}: {}", - metadata_version, entry.alter_version, entry.znode_name, entry.toString()); + current_metadata->getMetadataVersion(), entry.alter_version, entry.znode_name, entry.toString()); return true; } @@ -4981,10 +4989,9 @@ bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMer LOG_INFO(log, "Metadata changed in ZooKeeper. Applying changes locally."); auto metadata_diff = ReplicatedMergeTreeTableMetadata(*this, getInMemoryMetadataPtr()).checkAndFindDiff(metadata_from_entry, getInMemoryMetadataPtr()->getColumns(), getContext()); - setTableStructure(table_id, alter_context, std::move(columns_from_entry), metadata_diff); - metadata_version = entry.alter_version; + setTableStructure(table_id, alter_context, std::move(columns_from_entry), metadata_diff, entry.alter_version); - LOG_INFO(log, "Applied changes to the metadata of the table. Current metadata version: {}", metadata_version); + LOG_INFO(log, "Applied changes to the metadata of the table. Current metadata version: {}", current_metadata->getMetadataVersion()); } { @@ -4996,7 +5003,7 @@ bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMer /// This transaction may not happen, but it's OK, because on the next retry we will eventually create/update this node /// TODO Maybe do in in one transaction for Replicated database? - zookeeper->createOrUpdate(fs::path(replica_path) / "metadata_version", std::to_string(metadata_version), zkutil::CreateMode::Persistent); + zookeeper->createOrUpdate(fs::path(replica_path) / "metadata_version", std::to_string(current_metadata->getMetadataVersion()), zkutil::CreateMode::Persistent); return true; } @@ -5120,7 +5127,7 @@ void StorageReplicatedMergeTree::alter( size_t mutation_path_idx = std::numeric_limits::max(); String new_metadata_str = future_metadata_in_zk.toString(); - ops.emplace_back(zkutil::makeSetRequest(fs::path(zookeeper_path) / "metadata", new_metadata_str, metadata_version)); + ops.emplace_back(zkutil::makeSetRequest(fs::path(zookeeper_path) / "metadata", new_metadata_str, current_metadata->getMetadataVersion())); String new_columns_str = future_metadata.columns.toString(); ops.emplace_back(zkutil::makeSetRequest(fs::path(zookeeper_path) / "columns", new_columns_str, -1)); @@ -5136,7 +5143,7 @@ void StorageReplicatedMergeTree::alter( /// We can be sure, that in case of successful commit in zookeeper our /// version will increments by 1. Because we update with version check. - int new_metadata_version = metadata_version + 1; + int new_metadata_version = current_metadata->getMetadataVersion() + 1; alter_entry->type = LogEntry::ALTER_METADATA; alter_entry->source_replica = replica_name; diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index a4ac60052bc..e0143b9aac6 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -214,8 +214,6 @@ public: /// It's used if not set in engine's arguments while creating a replicated table. static String getDefaultReplicaName(const ContextPtr & context_); - int getMetadataVersion() const { return metadata_version; } - /// Modify a CREATE TABLE query to make a variant which must be written to a backup. void adjustCreateQueryForBackup(ASTPtr & create_query) const override; @@ -425,7 +423,6 @@ private: std::atomic shutdown_called {false}; std::atomic flush_called {false}; - int metadata_version = 0; /// Threads. /// A task that keeps track of the updates in the logs of all replicas and loads them into the queue. @@ -502,8 +499,10 @@ private: /// A part of ALTER: apply metadata changes only (data parts are altered separately). /// Must be called under IStorage::lockForAlter() lock. - void setTableStructure(const StorageID & table_id, const ContextPtr & local_context, - ColumnsDescription new_columns, const ReplicatedMergeTreeTableMetadata::Diff & metadata_diff); + void setTableStructure( + const StorageID & table_id, const ContextPtr & local_context, + ColumnsDescription new_columns, const ReplicatedMergeTreeTableMetadata::Diff & metadata_diff, + int32_t new_metadata_version); /** Check that the set of parts corresponds to that in ZK (/replicas/me/parts/). * If any parts described in ZK are not locally, throw an exception. From 9416401406b4978677d3d77a41cb80e07213f66c Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 3 Feb 2023 20:52:10 +0100 Subject: [PATCH 117/717] Fix --- src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index df4dea978a1..d6e3f53c757 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -1760,7 +1760,7 @@ std::map ReplicatedMergeTreeQueue::getAlterMutationCo if (in_partition == mutations_by_partition.end()) return {}; - Int64 part_data_version = part->info.getDataVersion(); + Int64 part_metadata_version = part->getMetadataVersion(); std::map result; /// Here we return mutation commands for part which has bigger mutation version than part mutation version. /// Please note, we don't use getDataVersion(). It's because these alter commands are used for in-fly conversions @@ -1771,13 +1771,12 @@ std::map ReplicatedMergeTreeQueue::getAlterMutationCo { if (mutation_status->entry->alter_version != -1) { - if (mutation_version > part_data_version) + if (mutation_status->entry->alter_version > part_metadata_version) { result[mutation_version] = mutation_status->entry->commands; } else { - result[mutation_version] = mutation_status->entry->commands; break; } } From 51b6154d68a6fe7da0ab06f955f8fba8415c09a4 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 3 Feb 2023 20:56:09 +0100 Subject: [PATCH 118/717] Fix stupid bug --- src/Storages/StorageReplicatedMergeTree.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 3083fbd2c46..9559c61e1a5 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -453,7 +453,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( Coordination::Stat metadata_stat; current_zookeeper->get(zookeeper_path + "/metadata", &metadata_stat); - StorageInMemoryMetadata storage_metadata; + StorageInMemoryMetadata storage_metadata(*metadata_snapshot); storage_metadata.setMetadataVersion(metadata_stat.version); setInMemoryMetadata(storage_metadata); } From fd4ae0990ba3bbb00c3353c911d0c8df4ea7abe8 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 3 Feb 2023 21:07:34 +0100 Subject: [PATCH 119/717] Tiny fix --- src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp | 1 + src/Storages/MergeTree/IMergeTreeDataPart.cpp | 8 ++++++++ src/Storages/MergeTree/IMergeTreeDataPart.h | 2 ++ src/Storages/MergeTree/MergeTreeData.cpp | 6 ------ 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp b/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp index 175df9b6e28..7a9f927ae0f 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp +++ b/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp @@ -683,6 +683,7 @@ void DataPartStorageOnDiskBase::clearDirectory( request.emplace_back(fs::path(dir) / "default_compression_codec.txt", true); request.emplace_back(fs::path(dir) / "delete-on-destroy.txt", true); request.emplace_back(fs::path(dir) / "txn_version.txt", true); + request.emplace_back(fs::path(dir) / "metadata_version.txt", true); disk->removeSharedFiles(request, !can_remove_shared_data, names_not_to_remove); disk->removeDirectory(dir); diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 3424ef5bf0a..5fb529ddf64 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -661,6 +661,7 @@ void IMergeTreeDataPart::appendFilesOfColumnsChecksumsIndexes(Strings & files, b appendFilesOfPartitionAndMinMaxIndex(files); appendFilesOfTTLInfos(files); appendFilesOfDefaultCompressionCodec(files); + appendFilesOfMetadataVersion(files); } if (!parent_part && include_projection) @@ -980,6 +981,11 @@ void IMergeTreeDataPart::appendFilesOfDefaultCompressionCodec(Strings & files) files.push_back(DEFAULT_COMPRESSION_CODEC_FILE_NAME); } +void IMergeTreeDataPart::appendFilesOfMetadataVersion(Strings & files) +{ + files.push_back(METADATA_VERSION_FILE_NAME); +} + CompressionCodecPtr IMergeTreeDataPart::detectDefaultCompressionCodec() const { /// In memory parts doesn't have any compression @@ -1346,6 +1352,8 @@ void IMergeTreeDataPart::loadColumns(bool require) setColumns(loaded_columns, infos, loaded_metadata_version); } + + /// Project part / part with project parts / compact part doesn't support LWD. bool IMergeTreeDataPart::supportLightweightDeleteMutate() const { diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index df83d138322..d706045c3fe 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -589,6 +589,8 @@ private: static void appendFilesOfDefaultCompressionCodec(Strings & files); + static void appendFilesOfMetadataVersion(Strings & files); + /// Found column without specific compression and return codec /// for this column with default parameters. CompressionCodecPtr detectDefaultCompressionCodec() const; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index b35d58688d5..e569513914e 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -7536,11 +7536,6 @@ AlterConversions MergeTreeData::getAlterConversionsForPart(const MergeTreeDataPa /// and columns in storage. if (command.type == MutationCommand::Type::RENAME_COLUMN) { - if (!part_columns.has(command.column_name)) - continue; - - part_columns.rename(command.column_name, command.rename_to); - if (auto it = rename_map.find(command.column_name); it != rename_map.end()) { auto rename_source = it->second; @@ -7548,7 +7543,6 @@ AlterConversions MergeTreeData::getAlterConversionsForPart(const MergeTreeDataPa rename_map[command.rename_to] = rename_source; } - else rename_map[command.rename_to] = command.column_name; } From 5183cc6ee50f3e5880e7e1ed6843128034bfb6f2 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Sat, 4 Feb 2023 16:14:53 +0000 Subject: [PATCH 120/717] fix --- .../useDataParallelAggregation.cpp | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp b/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp index 681b8f78c3a..f39e3a511c6 100644 --- a/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp +++ b/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp @@ -136,11 +136,11 @@ bool allOutputsDependsOnlyOnAllowedNodes( /// No need to explicitly check that each function is deterministic, because it is a guaranteed property of partition key expression (checked on table creation). /// So it is left only to check that each output node depends only on the allowed set of nodes (`irreducible_nodes`). bool allOutputsDependsOnlyOnAllowedNodes( - const ActionsDAGPtr & partition_actions, const NodeSet & irreducible_nodes, const MatchedTrees::Matches & matches) + const ActionsDAG & partition_actions, const NodeSet & irreducible_nodes, const MatchedTrees::Matches & matches) { NodeMap visited; bool res = true; - for (const auto & node : partition_actions->getOutputs()) + for (const auto & node : partition_actions.getOutputs()) if (node->type != ActionsDAG::ActionType::INPUT) res &= allOutputsDependsOnlyOnAllowedNodes(irreducible_nodes, matches, node, visited); return res; @@ -151,27 +151,26 @@ bool allOutputsDependsOnlyOnAllowedNodes( /// 2. To find col1, ..., coln we apply removeInjectiveFunctionsFromResultsRecursively to group by key actions. /// 3. We match partition key actions with group by key actions to find col1', ..., coln' in partition key actions. /// 4. We check that partition key is indeed a deterministic function of col1', ..., coln'. -bool isPartitionKeySuitsGroupByKey(const ReadFromMergeTree & reading, ActionsDAGPtr group_by_actions, const AggregatingStep & aggregating) +bool isPartitionKeySuitsGroupByKey( + const ReadFromMergeTree & reading, const ActionsDAGPtr & group_by_actions, const AggregatingStep & aggregating) { if (aggregating.isGroupingSets()) return false; - /// We are interested only in calculations required to obtain group by keys. - group_by_actions->removeUnusedActions(aggregating.getParams().keys); if (group_by_actions->hasArrayJoin() || group_by_actions->hasStatefulFunctions() || group_by_actions->hasNonDeterministic()) return false; const auto & gb_key_required_columns = group_by_actions->getRequiredColumnsNames(); - const auto partition_actions = reading.getStorageMetadata()->getPartitionKey().expression->getActionsDAG().clone(); + const auto & partition_actions = reading.getStorageMetadata()->getPartitionKey().expression->getActionsDAG(); /// Check that PK columns is a subset of GBK columns. - for (const auto & col : partition_actions->getRequiredColumnsNames()) + for (const auto & col : partition_actions.getRequiredColumnsNames()) if (std::ranges::find(gb_key_required_columns, col) == gb_key_required_columns.end()) return false; const auto irreducibe_nodes = removeInjectiveFunctionsFromResultsRecursively(group_by_actions); - const auto matches = matchTrees(*group_by_actions, *partition_actions); + const auto matches = matchTrees(*group_by_actions, partition_actions); return allOutputsDependsOnlyOnAllowedNodes(partition_actions, irreducibe_nodes, matches); } @@ -209,7 +208,7 @@ size_t tryAggregatePartitionsIndependently(QueryPlan::Node * node, QueryPlan::No return 0; if (!reading->willOutputEachPartitionThroughSeparatePort() - && isPartitionKeySuitsGroupByKey(*reading, expression_step->getExpression()->clone(), *aggregating_step)) + && isPartitionKeySuitsGroupByKey(*reading, expression_step->getExpression(), *aggregating_step)) { if (reading->requestOutputEachPartitionThroughSeparatePort()) aggregating_step->skipMerging(); From 750933d26550a9e7ebdadb74e66ee2cebc82a901 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Sat, 4 Feb 2023 21:09:44 +0000 Subject: [PATCH 121/717] fix --- src/Processors/QueryPlan/ReadFromMergeTree.cpp | 2 +- tests/queries/0_stateless/02521_aggregation_by_partitions.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index ce7854371a6..12be0471114 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -640,7 +640,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder( bool need_preliminary_merge = (parts_with_ranges.size() > settings.read_in_order_two_level_merge_threshold); std::vector splitted_parts_and_ranges; - splitted_parts_and_ranges.reserve(requested_num_streams); + splitted_parts_and_ranges.reserve(num_streams); const auto read_type = input_order_info->direction == 1 ? ReadFromMergeTree::ReadType::InOrder diff --git a/tests/queries/0_stateless/02521_aggregation_by_partitions.sql b/tests/queries/0_stateless/02521_aggregation_by_partitions.sql index 0bb4445a05a..aaa1b8b9b2d 100644 --- a/tests/queries/0_stateless/02521_aggregation_by_partitions.sql +++ b/tests/queries/0_stateless/02521_aggregation_by_partitions.sql @@ -5,7 +5,7 @@ set allow_aggregate_partitions_independently = 1; set force_aggregate_partitions_independently = 1; set allow_experimental_projection_optimization = 0; -create table t1(a UInt32) engine=MergeTree order by tuple() partition by a % 4; +create table t1(a UInt32) engine=MergeTree order by tuple() partition by a % 4 settings index_granularity = 8192, index_granularity_bytes = 10485760; system stop merges t1; From 1f1be6195ea1b7702e71d517f70789e5d4b26810 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Sat, 4 Feb 2023 21:54:48 +0000 Subject: [PATCH 122/717] fix --- src/Processors/QueryPlan/ReadFromMergeTree.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 12be0471114..dd249e4a242 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -1440,8 +1440,19 @@ Pipe ReadFromMergeTree::spreadMarkRanges( column_names_to_read.erase(std::unique(column_names_to_read.begin(), column_names_to_read.end()), column_names_to_read.end()); } + /// Construct a proper coordinator + if (input_order_info && is_parallel_reading_from_replicas && context->getClientInfo().interface == ClientInfo::Interface::LOCAL) + { + assert(context->parallel_reading_coordinator); + auto mode = input_order_info->direction == 1 ? CoordinationMode::WithOrder : CoordinationMode::ReverseOrder; + context->parallel_reading_coordinator->setMode(mode); + } + if (final) { + if (is_parallel_reading_from_replicas) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Final modifier is not supported with parallel replicas"); + if (output_each_partition_through_separate_port) throw Exception(ErrorCodes::LOGICAL_ERROR, "Optimisation isn't supposed to be used for queries with final"); From e51e385017178260af81e40e933aae9238f241b5 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 6 Feb 2023 14:16:17 +0100 Subject: [PATCH 123/717] fix style --- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 5fb529ddf64..8f5f353b777 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -1353,7 +1353,6 @@ void IMergeTreeDataPart::loadColumns(bool require) } - /// Project part / part with project parts / compact part doesn't support LWD. bool IMergeTreeDataPart::supportLightweightDeleteMutate() const { From a02e6f3d5b0f83203b8063ea7ed51d5a24d51b05 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 6 Feb 2023 14:57:21 +0100 Subject: [PATCH 124/717] fix bug ordinary merge tree --- src/Storages/StorageMergeTree.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 37c2ca8b746..a6f26654a80 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -2111,13 +2111,16 @@ std::map StorageMergeTree::getAlterMutationCommandsFo { std::lock_guard lock(currently_processing_in_background_mutex); - Int64 part_mutation_version = part->info.getMutationVersion(); + Int64 part_data_version = part->info.getDataVersion(); std::map result; - for (const auto & current_mutation_by_version : current_mutations_by_version) + if (!current_mutations_by_version.empty()) { - if (static_cast(current_mutation_by_version.first) > part_mutation_version) - result[current_mutation_by_version.first] = current_mutation_by_version.second.commands; + const auto & [latest_mutation_id, latest_commands] = *current_mutations_by_version.rbegin(); + if (part_data_version < static_cast(latest_mutation_id)) + { + result[latest_mutation_id] = latest_commands.commands; + } } return result; } From bd4a3ce2d52d0656fa7e502281023a997c952318 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 6 Feb 2023 15:05:13 +0100 Subject: [PATCH 125/717] Update src/Storages/MergeTree/DataPartsExchange.cpp --- src/Storages/MergeTree/DataPartsExchange.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index 1864ce37ddf..86bf86c916e 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -692,7 +692,6 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToMemory( auto block = block_in.read(); throttler->add(block.bytes()); - //// TODO Read them from file new_data_part->setColumns(block.getNamesAndTypesList(), {}, metadata_snapshot->getMetadataVersion()); if (!is_projection) From 30a7198411b42f07287ca99b32984ace43de4a40 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 6 Feb 2023 15:06:28 +0100 Subject: [PATCH 126/717] Update src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp --- src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index fb05cc1bddd..6be4cadc03e 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -1769,7 +1769,7 @@ std::map ReplicatedMergeTreeQueue::getAlterMutationCo /// of part's metadata. It mean that even if we have mutation with version X and part with data version X+10, but /// without mutation version part can still have wrong metadata and we have to apply this change on-fly if needed. - for (auto [mutation_version, mutation_status] : in_partition->second | std::views::reverse) + for (const auto & [mutation_version, mutation_status] : in_partition->second | std::views::reverse) { if (mutation_status->entry->alter_version != -1) { From 2a0484ca41316a11d6706bdb435262980a996405 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 6 Feb 2023 18:04:42 +0100 Subject: [PATCH 127/717] Add commnents, fix bugs --- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 6 ++ src/Storages/MergeTree/IMergeTreeDataPart.h | 10 ++ src/Storages/MergeTree/MergeTreeData.cpp | 18 ++-- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 8 +- .../02555_davengers_rename_chain.reference | 26 ++++++ .../02555_davengers_rename_chain.sql | 91 +++++++++++++++++++ 6 files changed, 145 insertions(+), 14 deletions(-) create mode 100644 tests/queries/0_stateless/02555_davengers_rename_chain.reference create mode 100644 tests/queries/0_stateless/02555_davengers_rename_chain.sql diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 8f5f353b777..cfd298952fa 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -976,6 +976,12 @@ void IMergeTreeDataPart::removeVersionMetadata() getDataPartStorage().removeFileIfExists("txn_version.txt"); } + +void IMergeTreeDataPart::removeMetadataVersion() +{ + getDataPartStorage().removeFileIfExists(METADATA_VERSION_FILE_NAME); +} + void IMergeTreeDataPart::appendFilesOfDefaultCompressionCodec(Strings & files) { files.push_back(DEFAULT_COMPRESSION_CODEC_FILE_NAME); diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index d706045c3fe..e31c83a0278 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -139,7 +139,10 @@ public: String getTypeName() const { return getType().toString(); } + /// We could have separate method like setMetadata, but it's much more convenient to set it up with columns void setColumns(const NamesAndTypesList & new_columns, const SerializationInfoByName & new_infos, int32_t metadata_version_); + + /// Version of metadata for part (columns, pk and so on) int32_t getMetadataVersion() const { return metadata_version; } const NamesAndTypesList & getColumns() const { return columns; } @@ -311,6 +314,7 @@ public: mutable VersionMetadata version; + /// Version of part metadata (columns, pk and so on). Managed properly only for replicated merge tree. int32_t metadata_version; /// For data in RAM ('index') @@ -384,8 +388,10 @@ public: /// (number of rows, number of rows with default values, etc). static inline constexpr auto SERIALIZATION_FILE_NAME = "serialization.json"; + /// Version used for transactions. static inline constexpr auto TXN_VERSION_METADATA_FILE_NAME = "txn_version.txt"; + static inline constexpr auto METADATA_VERSION_FILE_NAME = "metadata_version.txt"; /// One of part files which is used to check how many references (I'd like @@ -450,7 +456,11 @@ public: void writeDeleteOnDestroyMarker(); void removeDeleteOnDestroyMarker(); + /// It may look like a stupid joke. but these two methods are absolutely unrelated. + /// This one is about removing file with metadata about part version (for transactions) void removeVersionMetadata(); + /// This one is about removing file with version of part's metadata (columns, pk and so on) + void removeMetadataVersion(); mutable std::atomic removal_state = DataPartRemovalState::NOT_ATTEMPTED; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index be53a126c18..d96a903e6bd 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -4374,6 +4374,11 @@ MergeTreeData::DataPartPtr MergeTreeData::getPartIfExistsUnlocked(const MergeTre static void loadPartAndFixMetadataImpl(MergeTreeData::MutableDataPartPtr part) { + /// Remove metadata version file and take it from table. + /// Currently we cannot attach parts with different schema, so + /// we can assume that it's equal to table's current schema. + part->removeMetadataVersion(); + part->loadColumnsChecksumsIndexes(false, true); part->modification_time = part->getDataPartStorage().getLastModified().epochTime(); part->removeDeleteOnDestroyMarker(); @@ -7565,9 +7570,9 @@ AlterConversions MergeTreeData::getAlterConversionsForPart(const MergeTreeDataPa { std::map commands_map = getAlterMutationCommandsForPart(part); - auto part_columns = part->getColumnsDescription(); AlterConversions result{}; auto & rename_map = result.rename_map; + /// Squash "log of renames" into single map for (const auto & [version, commands] : commands_map) { for (const auto & command : commands) @@ -7577,15 +7582,7 @@ AlterConversions MergeTreeData::getAlterConversionsForPart(const MergeTreeDataPa /// and columns in storage. if (command.type == MutationCommand::Type::RENAME_COLUMN) { - if (auto it = rename_map.find(command.column_name); it != rename_map.end()) - { - auto rename_source = it->second; - rename_map.erase(it); - - rename_map[command.rename_to] = rename_source; - } - else - rename_map[command.rename_to] = command.column_name; + rename_map[command.rename_to] = command.column_name; } } } @@ -7596,6 +7593,7 @@ AlterConversions MergeTreeData::getAlterConversionsForPart(const MergeTreeDataPa it = rename_map.erase(it); else ++it; + } return result; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 6be4cadc03e..341007e79e5 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -1764,21 +1764,21 @@ std::map ReplicatedMergeTreeQueue::getAlterMutationCo Int64 part_metadata_version = part->getMetadataVersion(); std::map result; - /// Here we return mutation commands for part which has bigger mutation version than part mutation version. + /// Here we return mutation commands for part which has bigger alter version than part metadata version. /// Please note, we don't use getDataVersion(). It's because these alter commands are used for in-fly conversions - /// of part's metadata. It mean that even if we have mutation with version X and part with data version X+10, but - /// without mutation version part can still have wrong metadata and we have to apply this change on-fly if needed. - + /// of part's metadata. for (const auto & [mutation_version, mutation_status] : in_partition->second | std::views::reverse) { if (mutation_status->entry->alter_version != -1) { + /// we take commands with bigger metadata version if (mutation_status->entry->alter_version > part_metadata_version) { result[mutation_version] = mutation_status->entry->commands; } else { + /// entries are ordered, we processing them in reverse order so we can break break; } } diff --git a/tests/queries/0_stateless/02555_davengers_rename_chain.reference b/tests/queries/0_stateless/02555_davengers_rename_chain.reference new file mode 100644 index 00000000000..a9fc4b395e2 --- /dev/null +++ b/tests/queries/0_stateless/02555_davengers_rename_chain.reference @@ -0,0 +1,26 @@ +{"a1":"1","b1":"2","c":"3"} +~~~~~~~ +{"a1":"1","b1":"2","c":"3"} +{"a1":"4","b1":"5","c":"6"} +~~~~~~~ +{"a1":"1","b1":"2","c":"3"} +{"a1":"4","b1":"5","c":"6"} +{"a1":"7","b1":"8","c":"9"} +~~~~~~~ +{"b":"1","a":"2","c":"3"} +{"b":"4","a":"5","c":"6"} +{"b":"7","a":"8","c":"9"} +~~~~~~~ +{"a1":"1","b1":"2","c":"3"} +~~~~~~~ +{"a1":"1","b1":"2","c":"3"} +{"a1":"4","b1":"5","c":"6"} +~~~~~~~ +{"a1":"1","b1":"2","c":"3"} +{"a1":"4","b1":"5","c":"6"} +{"a1":"7","b1":"8","c":"9"} +~~~~~~~ +{"b":"1","a":"2","c":"3"} +{"b":"4","a":"5","c":"6"} +{"b":"7","a":"8","c":"9"} +~~~~~~~ diff --git a/tests/queries/0_stateless/02555_davengers_rename_chain.sql b/tests/queries/0_stateless/02555_davengers_rename_chain.sql new file mode 100644 index 00000000000..eae345d0472 --- /dev/null +++ b/tests/queries/0_stateless/02555_davengers_rename_chain.sql @@ -0,0 +1,91 @@ +DROP TABLE IF EXISTS wrong_metadata; + +CREATE TABLE wrong_metadata( + a UInt64, + b UInt64, + c UInt64 +) +ENGINE ReplicatedMergeTree('/test/{database}/tables/wrong_metadata', '1') +ORDER BY tuple() +SETTINGS min_bytes_for_wide_part = 0; + +INSERT INTO wrong_metadata VALUES (1, 2, 3); + +SYSTEM STOP MERGES wrong_metadata; + +ALTER TABLE wrong_metadata RENAME COLUMN a TO a1, RENAME COLUMN b to b1 SETTINGS replication_alter_partitions_sync = 0; + +SELECT sleep(1) FORMAT Null; + +SELECT * FROM wrong_metadata ORDER BY a1 FORMAT JSONEachRow; +SELECT '~~~~~~~'; + +INSERT INTO wrong_metadata VALUES (4, 5, 6); + +SELECT * FROM wrong_metadata ORDER BY a1 FORMAT JSONEachRow; +SELECT '~~~~~~~'; + +ALTER TABLE wrong_metadata RENAME COLUMN a1 TO b, RENAME COLUMN b1 to a SETTINGS replication_alter_partitions_sync = 0; + +SELECT sleep(1) FORMAT Null; +SELECT sleep(1) FORMAT Null; + +INSERT INTO wrong_metadata VALUES (7, 8, 9); + +SELECT * FROM wrong_metadata ORDER by a1 FORMAT JSONEachRow; +SELECT '~~~~~~~'; + +SYSTEM START MERGES wrong_metadata; + +SYSTEM SYNC REPLICA wrong_metadata; + +SELECT * FROM wrong_metadata order by a FORMAT JSONEachRow; +SELECT '~~~~~~~'; + +DROP TABLE IF EXISTS wrong_metadata; + +DROP TABLE IF EXISTS wrong_metadata_compact; + +CREATE TABLE wrong_metadata_compact( + a UInt64, + b UInt64, + c UInt64 +) +ENGINE ReplicatedMergeTree('/test/{database}/tables/wrong_metadata_compact', '1') +ORDER BY tuple() +SETTINGS min_bytes_for_wide_part = 10000000; + +INSERT INTO wrong_metadata_compact VALUES (1, 2, 3); + +SYSTEM STOP MERGES wrong_metadata_compact; + +ALTER TABLE wrong_metadata_compact RENAME COLUMN a TO a1, RENAME COLUMN b to b1 SETTINGS replication_alter_partitions_sync = 0; + +SELECT sleep(1) FORMAT Null; + +SELECT * FROM wrong_metadata_compact ORDER BY a1 FORMAT JSONEachRow; +SELECT '~~~~~~~'; + +INSERT INTO wrong_metadata_compact VALUES (4, 5, 6); + +SELECT * FROM wrong_metadata_compact ORDER BY a1 FORMAT JSONEachRow; +SELECT '~~~~~~~'; + +ALTER TABLE wrong_metadata_compact RENAME COLUMN a1 TO b, RENAME COLUMN b1 to a SETTINGS replication_alter_partitions_sync = 0; + +SELECT sleep(1) FORMAT Null; +SELECT sleep(1) FORMAT Null; + +INSERT INTO wrong_metadata_compact VALUES (7, 8, 9); + +SELECT * FROM wrong_metadata_compact ORDER by a1 FORMAT JSONEachRow; +SELECT '~~~~~~~'; + +SYSTEM START MERGES wrong_metadata_compact; + +SYSTEM SYNC REPLICA wrong_metadata_compact; + +SELECT * FROM wrong_metadata_compact order by a FORMAT JSONEachRow; +SELECT '~~~~~~~'; + +DROP TABLE IF EXISTS wrong_metadata_compact; From aac5da77e88f63a26153e76408fe305549b7c243 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 6 Feb 2023 21:32:40 +0100 Subject: [PATCH 128/717] fix stupid bug --- src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp index 271edb0399f..2082a7c000a 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp @@ -149,7 +149,7 @@ void ReplicatedMergeTreeAttachThread::runImpl() const bool replica_metadata_version_exists = zookeeper->tryGet(replica_path + "/metadata_version", replica_metadata_version); if (replica_metadata_version_exists) { - StorageInMemoryMetadata metadata_with_new_version; + StorageInMemoryMetadata metadata_with_new_version(*metadata_snapshot); metadata_with_new_version.setMetadataVersion(parse(replica_metadata_version)); storage.setInMemoryMetadata(metadata_with_new_version); } From dd3a98e88dc3055ab17e08000e4fd4c2aeb91b2e Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 6 Feb 2023 22:30:31 +0100 Subject: [PATCH 129/717] Fixes --- src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp | 1 + src/Storages/StorageInMemoryMetadata.h | 2 +- src/Storages/StorageReplicatedMergeTree.cpp | 1 + tests/queries/0_stateless/02361_fsync_profile_events.sh | 4 ++-- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp index 2082a7c000a..941d0850f67 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp @@ -150,6 +150,7 @@ void ReplicatedMergeTreeAttachThread::runImpl() if (replica_metadata_version_exists) { StorageInMemoryMetadata metadata_with_new_version(*metadata_snapshot); + metadata_with_new_version.setMetadataVersion(parse(replica_metadata_version)); storage.setInMemoryMetadata(metadata_with_new_version); } diff --git a/src/Storages/StorageInMemoryMetadata.h b/src/Storages/StorageInMemoryMetadata.h index 2dd6c32e3c6..dfb5e9ddb54 100644 --- a/src/Storages/StorageInMemoryMetadata.h +++ b/src/Storages/StorageInMemoryMetadata.h @@ -50,7 +50,7 @@ struct StorageInMemoryMetadata String comment; - int32_t metadata_version; + int32_t metadata_version = 0; StorageInMemoryMetadata() = default; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 55d857c4c28..8e6e809a54a 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -5018,6 +5018,7 @@ bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMer auto metadata_diff = ReplicatedMergeTreeTableMetadata(*this, getInMemoryMetadataPtr()).checkAndFindDiff(metadata_from_entry, getInMemoryMetadataPtr()->getColumns(), getContext()); setTableStructure(table_id, alter_context, std::move(columns_from_entry), metadata_diff, entry.alter_version); + current_metadata = getInMemoryMetadataPtr(); LOG_INFO(log, "Applied changes to the metadata of the table. Current metadata version: {}", current_metadata->getMetadataVersion()); } diff --git a/tests/queries/0_stateless/02361_fsync_profile_events.sh b/tests/queries/0_stateless/02361_fsync_profile_events.sh index 44a1bd58d36..5b603133f6c 100755 --- a/tests/queries/0_stateless/02361_fsync_profile_events.sh +++ b/tests/queries/0_stateless/02361_fsync_profile_events.sh @@ -44,8 +44,8 @@ for i in {1..100}; do ")" # Non retriable errors - if [[ $FileSync -ne 7 ]]; then - echo "FileSync: $FileSync != 11" >&2 + if [[ $FileSync -ne 8 ]]; then + echo "FileSync: $FileSync != 8" >&2 exit 2 fi # Check that all files was synced From 14f8eff29030f72c46971746f1d7b4c9302bde0a Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 7 Feb 2023 11:48:23 +0000 Subject: [PATCH 130/717] Fixes: checking if steps between DISTINCTs can generate new rows --- .../Optimizations/removeRedundantDistinct.cpp | 23 ++++++- .../02500_remove_redundant_distinct.reference | 69 +++++++++++++++++++ .../02500_remove_redundant_distinct.sh | 29 ++++++++ 3 files changed, 118 insertions(+), 3 deletions(-) diff --git a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp index 1b012d81f82..576525bf9a3 100644 --- a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp +++ b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp @@ -1,7 +1,9 @@ #include #include +#include #include #include +#include #include #include #include @@ -77,15 +79,30 @@ namespace { const IQueryPlanStep * current_step = node->step.get(); - /// don't try to remove DISTINCT after union/join/intersect/except - if (typeid_cast(current_step) || typeid_cast(current_step) - || typeid_cast(current_step)) + /// don't try to remove DISTINCT after step with many inputs, like union/join/intersect/except + if (current_step->getInputStreams().size() > 1) + break; + + /// do not remove DISTINCT if there are steps which can generate new rows + if (typeid_cast(current_step) || typeid_cast(current_step)) break; if (const auto * const expr = typeid_cast(current_step); expr) + { + /// arrayJoin() can generate new rows + if (expr->getExpression()->hasArrayJoin()) + break; + dag_stack.push_back(expr->getExpression()); + } if (const auto * const filter = typeid_cast(current_step); filter) + { + /// arrayJoin() can generate new rows + if (filter->getExpression()->hasArrayJoin()) + break; + dag_stack.push_back(filter->getExpression()); + } node = node->children.front(); inner_distinct_step = typeid_cast(node->step.get()); diff --git a/tests/queries/0_stateless/02500_remove_redundant_distinct.reference b/tests/queries/0_stateless/02500_remove_redundant_distinct.reference index 30da5d536f9..ffee4a54ee6 100644 --- a/tests/queries/0_stateless/02500_remove_redundant_distinct.reference +++ b/tests/queries/0_stateless/02500_remove_redundant_distinct.reference @@ -139,3 +139,72 @@ Expression ((Project names + (Projection + (Change column names to column identi 2 0 0 2 1 2 2 2 4 +-- ARRAY JOIN: do _not_ remove outer DISTINCT because new rows are generated between inner and outer DISTINCTs +-- query +SELECT DISTINCT * +FROM +( + SELECT DISTINCT * + FROM VALUES('Hello', 'World', 'Goodbye') +) AS words +ARRAY JOIN [0, 1] AS arr +-- explain +Expression (Project names) + Distinct (DISTINCT) + Distinct (Preliminary DISTINCT) + Expression (Projection) + ArrayJoin (ARRAY JOIN) + Expression ((ARRAY JOIN actions + (Change column names to column identifiers + Project names))) + Distinct (DISTINCT) + Distinct (Preliminary DISTINCT) + Expression ((Projection + Change column names to column identifiers)) + ReadFromStorage (Values) +-- execute +Hello +World +Goodbye +-- WITH FILL: do _not_ remove outer DISTINCT because new rows are generated between inner and outer DISTINCTs +-- query +SELECT DISTINCT * +FROM +( + SELECT DISTINCT * + FROM values('id UInt8', 0, 2) + ORDER BY id ASC WITH FILL +) +-- explain +Expression (Project names) + Distinct (DISTINCT) + Distinct (Preliminary DISTINCT) + Expression ((Projection + (Change column names to column identifiers + Project names))) + Filling + Distinct (DISTINCT) + Sorting (Sorting for ORDER BY) + Expression (Before ORDER BY) + Distinct (Preliminary DISTINCT) + Expression ((Projection + Change column names to column identifiers)) + ReadFromStorage (Values) +-- execute +0 +1 +2 +-- WHERE with arrayJoin(): do _not_ remove outer DISTINCT because new rows are generated between inner and outer DISTINCTs +-- query +SELECT DISTINCT * +FROM +( + SELECT DISTINCT ['Istanbul', 'Berlin', 'Bensheim'] AS cities +) +WHERE arrayJoin(cities) IN ['Berlin', 'Bensheim'] +-- explain +Expression (Project names) + Distinct (DISTINCT) + Distinct (Preliminary DISTINCT) + Expression (Projection) + Filter ((WHERE + (Change column names to column identifiers + Project names))) + Distinct (DISTINCT) + Distinct (Preliminary DISTINCT) + Expression ((Projection + Change column names to column identifiers)) + ReadFromStorage (SystemOne) +-- execute +['Istanbul','Berlin','Bensheim'] diff --git a/tests/queries/0_stateless/02500_remove_redundant_distinct.sh b/tests/queries/0_stateless/02500_remove_redundant_distinct.sh index 94d99242ead..4d9bbd301a1 100755 --- a/tests/queries/0_stateless/02500_remove_redundant_distinct.sh +++ b/tests/queries/0_stateless/02500_remove_redundant_distinct.sh @@ -83,3 +83,32 @@ FROM ) )" run_query "$query" + +echo "-- ARRAY JOIN: do _not_ remove outer DISTINCT because new rows are generated between inner and outer DISTINCTs" +query="SELECT DISTINCT * +FROM +( + SELECT DISTINCT * + FROM VALUES('Hello', 'World', 'Goodbye') +) AS words +ARRAY JOIN [0, 1] AS arr" +run_query "$query" + +echo "-- WITH FILL: do _not_ remove outer DISTINCT because new rows are generated between inner and outer DISTINCTs" +query="SELECT DISTINCT * +FROM +( + SELECT DISTINCT * + FROM values('id UInt8', 0, 2) + ORDER BY id ASC WITH FILL +)" +run_query "$query" + +echo "-- WHERE with arrayJoin(): do _not_ remove outer DISTINCT because new rows are generated between inner and outer DISTINCTs" +query="SELECT DISTINCT * +FROM +( + SELECT DISTINCT ['Istanbul', 'Berlin', 'Bensheim'] AS cities +) +WHERE arrayJoin(cities) IN ['Berlin', 'Bensheim']" +run_query "$query" From 1fd9baa3f3b2c67196317723031d04687b3db7c9 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 7 Feb 2023 12:49:51 +0100 Subject: [PATCH 131/717] Fix fetch: --- src/Storages/MergeTree/DataPartsExchange.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index 86bf86c916e..a8fbecbf523 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -768,7 +768,8 @@ void Fetcher::downloadBaseOrProjectionPartToDisk( if (file_name != "checksums.txt" && file_name != "columns.txt" && - file_name != IMergeTreeDataPart::DEFAULT_COMPRESSION_CODEC_FILE_NAME) + file_name != IMergeTreeDataPart::DEFAULT_COMPRESSION_CODEC_FILE_NAME && + file_name != IMergeTreeDataPart::METADATA_VERSION_FILE_NAME) checksums.addFile(file_name, file_size, expected_hash); } From 36f41da6e001734574f10be747e8f1762801d48d Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 7 Feb 2023 18:53:47 +0100 Subject: [PATCH 132/717] One more bugfix --- src/Storages/MergeTree/AlterConversions.cpp | 21 ++++ src/Storages/MergeTree/AlterConversions.h | 11 ++- src/Storages/MergeTree/MergeTreeData.cpp | 4 +- src/Storages/MergeTree/MutateTask.cpp | 95 ++++++++++++++++--- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 8 +- 5 files changed, 118 insertions(+), 21 deletions(-) diff --git a/src/Storages/MergeTree/AlterConversions.cpp b/src/Storages/MergeTree/AlterConversions.cpp index e2eea2e68f6..7a298b0f6ca 100644 --- a/src/Storages/MergeTree/AlterConversions.cpp +++ b/src/Storages/MergeTree/AlterConversions.cpp @@ -31,4 +31,25 @@ std::string AlterConversions::getColumnNewName(const std::string & old_name) con throw Exception(ErrorCodes::LOGICAL_ERROR, "Column {} was not renamed", old_name); } + +bool AlterConversions::isColumnRenamed(const std::string & new_name) const +{ + for (const auto & [name_to, name_from] : rename_map) + { + if (name_to == new_name) + return true; + } + return false; +} +/// Get column old name before rename (lookup by key in rename_map) +std::string AlterConversions::getColumnOldName(const std::string & new_name) const +{ + for (const auto & [name_to, name_from] : rename_map) + { + if (name_to == new_name) + return name_from; + } + throw Exception(ErrorCodes::LOGICAL_ERROR, "Column {} was not renamed", new_name); +} + } diff --git a/src/Storages/MergeTree/AlterConversions.h b/src/Storages/MergeTree/AlterConversions.h index 47f964e85fe..ada385d6100 100644 --- a/src/Storages/MergeTree/AlterConversions.h +++ b/src/Storages/MergeTree/AlterConversions.h @@ -14,17 +14,22 @@ namespace DB /// part->getColumns() and storage->getColumns(). struct AlterConversions { + struct RenamePair + { + std::string rename_to; + std::string rename_from; + }; /// Rename map new_name -> old_name - std::unordered_map rename_map; + std::vector rename_map; /// Column was renamed (lookup by value in rename_map) bool columnHasNewName(const std::string & old_name) const; /// Get new name for column (lookup by value in rename_map) std::string getColumnNewName(const std::string & old_name) const; /// Is this name is new name of column (lookup by key in rename_map) - bool isColumnRenamed(const std::string & new_name) const { return rename_map.count(new_name) > 0; } + bool isColumnRenamed(const std::string & new_name) const; /// Get column old name before rename (lookup by key in rename_map) - std::string getColumnOldName(const std::string & new_name) const { return rename_map.at(new_name); } + std::string getColumnOldName(const std::string & new_name) const; }; } diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index d96a903e6bd..05c75067f3f 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -7582,14 +7582,14 @@ AlterConversions MergeTreeData::getAlterConversionsForPart(const MergeTreeDataPa /// and columns in storage. if (command.type == MutationCommand::Type::RENAME_COLUMN) { - rename_map[command.rename_to] = command.column_name; + rename_map.emplace_back(AlterConversions::RenamePair{command.rename_to, command.column_name}); } } } for (auto it = rename_map.begin(); it != rename_map.end();) { - if (it->first == it->second) + if (it->rename_to == it->rename_from) it = rename_map.erase(it); else ++it; diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index 4c1ca86f5cb..16ddcaae89e 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -158,11 +158,7 @@ static void splitMutationCommands( auto alter_conversions = part->storage.getAlterConversionsForPart(part); for (const auto & [rename_to, rename_from] : alter_conversions.rename_map) { - if (part_columns.has(rename_from)) - { - for_file_renames.push_back({.type = MutationCommand::Type::RENAME_COLUMN, .column_name = rename_from, .rename_to = rename_to}); - part_columns.rename(rename_from, rename_to); - } + for_file_renames.push_back({.type = MutationCommand::Type::RENAME_COLUMN, .column_name = rename_from, .rename_to = rename_to}); } } } @@ -174,8 +170,13 @@ getColumnsForNewDataPart( const Block & updated_header, NamesAndTypesList storage_columns, const SerializationInfoByName & serialization_infos, + const MutationCommands & commands_for_interpreter, const MutationCommands & commands_for_removes) { + MutationCommands all_commands; + all_commands.insert(all_commands.end(), commands_for_interpreter.begin(), commands_for_interpreter.end()); + all_commands.insert(all_commands.end(), commands_for_removes.begin(), commands_for_removes.end()); + NameSet removed_columns; NameToNameMap renamed_columns_to_from; NameToNameMap renamed_columns_from_to; @@ -191,8 +192,37 @@ getColumnsForNewDataPart( storage_columns.emplace_back(column); } - /// All commands are validated in AlterCommand so we don't care about order - for (const auto & command : commands_for_removes) + NameToNameMap squashed_renames; + for (const auto & command : all_commands) + { + std::string result_name = command.rename_to; + + bool squashed = false; + for (const auto & [name_from, name_to] : squashed_renames) + { + if (name_to == command.column_name) + { + squashed = true; + squashed_renames[name_from] = result_name; + break; + } + } + if (!squashed) + squashed_renames[command.column_name] = result_name; + } + + MutationCommands squashed_commands; + for (const auto & command : all_commands) + { + if (squashed_renames.contains(command.column_name)) + { + squashed_commands.push_back(command); + squashed_commands.back().rename_to = squashed_renames[command.column_name]; + } + } + + + for (const auto & command : squashed_commands) { if (command.type == MutationCommand::UPDATE) { @@ -285,10 +315,10 @@ getColumnsForNewDataPart( /// should it's previous version should be dropped or removed if (renamed_columns_to_from.contains(it->name) && !was_renamed && !was_removed) throw Exception( - ErrorCodes::LOGICAL_ERROR, - "Incorrect mutation commands, trying to rename column {} to {}, " - "but part {} already has column {}", - renamed_columns_to_from[it->name], it->name, source_part->name, it->name); + ErrorCodes::LOGICAL_ERROR, + "Incorrect mutation commands, trying to rename column {} to {}, " + "but part {} already has column {}", + renamed_columns_to_from[it->name], it->name, source_part->name, it->name); /// Column was renamed and no other column renamed to it's name /// or column is dropped. @@ -582,8 +612,46 @@ static NameToNameVector collectFilesForRenames( rename_vector.emplace_back(file_rename_from, file_rename_to); }; - /// Remove old data + NameToNameMap squashed_renames; for (const auto & command : commands_for_removes) + { + + std::string result_name; + if (command.type == MutationCommand::Type::DROP_INDEX + || command.type == MutationCommand::Type::DROP_PROJECTION + || command.type == MutationCommand::Type::DROP_COLUMN + || command.type == MutationCommand::Type::READ_COLUMN) + result_name = ""; + + if (command.type == MutationCommand::RENAME_COLUMN) + result_name = command.rename_to; + + bool squashed = false; + for (const auto & [name_from, name_to] : squashed_renames) + { + if (name_to == command.column_name) + { + squashed = true; + squashed_renames[name_from] = result_name; + break; + } + } + if (!squashed) + squashed_renames[command.column_name] = result_name; + } + + MutationCommands squashed_commands; + for (const auto & command : commands_for_removes) + { + if (squashed_renames.contains(command.column_name)) + { + squashed_commands.push_back(command); + squashed_commands.back().rename_to = squashed_renames[command.column_name]; + } + } + + /// Remove old data + for (const auto & command : squashed_commands) { if (command.type == MutationCommand::Type::DROP_INDEX) { @@ -624,7 +692,6 @@ static NameToNameVector collectFilesForRenames( String escaped_name_from = escapeForFileName(command.column_name); String escaped_name_to = escapeForFileName(command.rename_to); - ISerialization::StreamCallback callback = [&](const ISerialization::SubstreamPath & substream_path) { String stream_from = ISerialization::getFileNameForStream(command.column_name, substream_path); @@ -1698,7 +1765,7 @@ bool MutateTask::prepare() auto [new_columns, new_infos] = MutationHelpers::getColumnsForNewDataPart( ctx->source_part, ctx->updated_header, ctx->storage_columns, - ctx->source_part->getSerializationInfos(), ctx->commands_for_part); + ctx->source_part->getSerializationInfos(), ctx->for_interpreter, ctx->for_file_renames); ctx->new_data_part->setColumns(new_columns, new_infos, ctx->metadata_snapshot->getMetadataVersion()); ctx->new_data_part->partition.assign(ctx->source_part->partition); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 341007e79e5..f491cdc8cd1 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -1769,10 +1769,14 @@ std::map ReplicatedMergeTreeQueue::getAlterMutationCo /// of part's metadata. for (const auto & [mutation_version, mutation_status] : in_partition->second | std::views::reverse) { - if (mutation_status->entry->alter_version != -1) + int32_t alter_version = mutation_status->entry->alter_version; + if (alter_version != -1) { + if (!alter_sequence.canExecuteDataAlter(alter_version, lock)) + continue; + /// we take commands with bigger metadata version - if (mutation_status->entry->alter_version > part_metadata_version) + if (alter_version > part_metadata_version) { result[mutation_version] = mutation_status->entry->commands; } From bbf044f4773547d40401838a7e8112f674ae96cc Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 7 Feb 2023 22:30:38 +0100 Subject: [PATCH 133/717] Fxi --- src/Storages/MergeTree/MutateTask.cpp | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index 16ddcaae89e..5738a48783d 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -114,6 +114,12 @@ static void splitMutationCommands( .column_name = rename_to, }); + for_file_renames.push_back( + { + .type = MutationCommand::Type::RENAME_COLUMN, + .column_name = rename_from, + .rename_to = rename_to + }); part_columns.rename(rename_from, rename_to); } } @@ -615,16 +621,7 @@ static NameToNameVector collectFilesForRenames( NameToNameMap squashed_renames; for (const auto & command : commands_for_removes) { - - std::string result_name; - if (command.type == MutationCommand::Type::DROP_INDEX - || command.type == MutationCommand::Type::DROP_PROJECTION - || command.type == MutationCommand::Type::DROP_COLUMN - || command.type == MutationCommand::Type::READ_COLUMN) - result_name = ""; - - if (command.type == MutationCommand::RENAME_COLUMN) - result_name = command.rename_to; + std::string result_name = command.rename_to; bool squashed = false; for (const auto & [name_from, name_to] : squashed_renames) From a28d10a81013c71d26c8c5d5d87791bc3533617b Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 8 Feb 2023 14:50:30 +0100 Subject: [PATCH 134/717] Fxi --- src/Storages/MergeTree/DataPartsExchange.cpp | 11 +++-- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 6 ++- src/Storages/MergeTree/MutateTask.cpp | 9 ++++- tests/integration/test_merge_tree_s3/test.py | 7 ++-- ...system_cache_on_write_operations.reference | 40 +++++++++---------- 5 files changed, 45 insertions(+), 28 deletions(-) diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index a8fbecbf523..35c627c3210 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -63,8 +63,9 @@ constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_DEFAULT_COMPRESSION = 4; constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_UUID = 5; constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_ZERO_COPY = 6; constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_PROJECTION = 7; +constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_METADATA_VERSION = 8; // Reserved for ALTER PRIMARY KEY -// constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_PRIMARY_KEY = 8; +// constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_PRIMARY_KEY = 9; std::string getEndpointId(const std::string & node_id) { @@ -120,7 +121,7 @@ void Service::processQuery(const HTMLForm & params, ReadBuffer & /*body*/, Write MergeTreePartInfo::fromPartName(part_name, data.format_version); /// We pretend to work as older server version, to be sure that client will correctly process our version - response.addCookie({"server_protocol_version", toString(std::min(client_protocol_version, REPLICATION_PROTOCOL_VERSION_WITH_PARTS_PROJECTION))}); + response.addCookie({"server_protocol_version", toString(std::min(client_protocol_version, REPLICATION_PROTOCOL_VERSION_WITH_METADATA_VERSION))}); LOG_TRACE(log, "Sending part {}", part_name); @@ -280,6 +281,10 @@ MergeTreeData::DataPart::Checksums Service::sendPartFromDisk( && name == IMergeTreeDataPart::DEFAULT_COMPRESSION_CODEC_FILE_NAME) continue; + if (client_protocol_version < REPLICATION_PROTOCOL_VERSION_WITH_METADATA_VERSION + && name == IMergeTreeDataPart::METADATA_VERSION_FILE_NAME) + continue; + files_to_replicate.insert(name); } @@ -407,7 +412,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchSelectedPart( { {"endpoint", getEndpointId(replica_path)}, {"part", part_name}, - {"client_protocol_version", toString(REPLICATION_PROTOCOL_VERSION_WITH_PARTS_PROJECTION)}, + {"client_protocol_version", toString(REPLICATION_PROTOCOL_VERSION_WITH_METADATA_VERSION)}, {"compress", "false"} }); diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index cfd298952fa..94aa2a72949 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -1352,9 +1352,13 @@ void IMergeTreeDataPart::loadColumns(bool require) else { loaded_metadata_version = metadata_snapshot->getMetadataVersion(); + + writeMetadata(METADATA_VERSION_FILE_NAME, {}, [loaded_metadata_version](auto & buffer) + { + writeIntText(loaded_metadata_version, buffer); + }); } - ///TODO read metadata here setColumns(loaded_columns, infos, loaded_metadata_version); } diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index 5738a48783d..340709bff2c 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -735,6 +735,7 @@ void finalizeMutatedPart( ExecuteTTLType execute_ttl_type, const CompressionCodecPtr & codec, ContextPtr context, + StorageMetadataPtr metadata_snapshot, bool sync) { std::vector> written_files; @@ -783,6 +784,12 @@ void finalizeMutatedPart( written_files.push_back(std::move(out_comp)); } + { + auto out_metadata = new_data_part->getDataPartStorage().writeFile(IMergeTreeDataPart::METADATA_VERSION_FILE_NAME, 4096, context->getWriteSettings()); + DB::writeText(metadata_snapshot->getMetadataVersion(), *out_metadata); + written_files.push_back(std::move(out_metadata)); + } + { /// Write a file with a description of columns. auto out_columns = new_data_part->getDataPartStorage().writeFile("columns.txt", 4096, context->getWriteSettings()); @@ -1555,7 +1562,7 @@ private: } } - MutationHelpers::finalizeMutatedPart(ctx->source_part, ctx->new_data_part, ctx->execute_ttl_type, ctx->compression_codec, ctx->context, ctx->need_sync); + MutationHelpers::finalizeMutatedPart(ctx->source_part, ctx->new_data_part, ctx->execute_ttl_type, ctx->compression_codec, ctx->context, ctx->metadata_snapshot, ctx->need_sync); } diff --git a/tests/integration/test_merge_tree_s3/test.py b/tests/integration/test_merge_tree_s3/test.py index f0f81100320..c8c97f4d40f 100644 --- a/tests/integration/test_merge_tree_s3/test.py +++ b/tests/integration/test_merge_tree_s3/test.py @@ -52,8 +52,10 @@ def cluster(): FILES_OVERHEAD = 1 FILES_OVERHEAD_PER_COLUMN = 2 # Data and mark files -FILES_OVERHEAD_PER_PART_WIDE = FILES_OVERHEAD_PER_COLUMN * 3 + 2 + 6 + 1 -FILES_OVERHEAD_PER_PART_COMPACT = 10 + 1 +FILES_OVERHEAD_DEFAULT_COMPRESSION_CODEC = 1 +FILES_OVERHEAD_METADATA_VERSION = 1 +FILES_OVERHEAD_PER_PART_WIDE = FILES_OVERHEAD_PER_COLUMN * 3 + 2 + 6 + FILES_OVERHEAD_DEFAULT_COMPRESSION_CODEC + FILES_OVERHEAD_METADATA_VERSION +FILES_OVERHEAD_PER_PART_COMPACT = 10 + FILES_OVERHEAD_DEFAULT_COMPRESSION_CODEC + FILES_OVERHEAD_METADATA_VERSION def create_table(node, table_name, **additional_settings): @@ -232,7 +234,6 @@ def test_insert_same_partition_and_merge(cluster, merge_vertical, node_name): def test_alter_table_columns(cluster, node_name): node = cluster.instances[node_name] create_table(node, "s3_test") - minio = cluster.minio_client node.query( "INSERT INTO s3_test VALUES {}".format(generate_values("2020-01-03", 4096)) diff --git a/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.reference b/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.reference index bbca9bbbfee..7fa38f6ff44 100644 --- a/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.reference +++ b/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.reference @@ -7,25 +7,25 @@ file_segment_range_begin: 0 file_segment_range_end: 745 size: 746 state: DOWNLOADED -7 -7 +8 +8 0 2 2 -7 +8 Row 1: ────── file_segment_range_begin: 0 file_segment_range_end: 1659 size: 1660 state: DOWNLOADED -7 -7 -7 -7 -21 -31 -38 +8 +8 +8 +8 +24 +27 +27 5010500 18816 Using storage policy: local_cache @@ -37,24 +37,24 @@ file_segment_range_begin: 0 file_segment_range_end: 745 size: 746 state: DOWNLOADED -7 -7 +8 +8 0 2 2 -7 +8 Row 1: ────── file_segment_range_begin: 0 file_segment_range_end: 1659 size: 1660 state: DOWNLOADED -7 -7 -7 -7 -21 -31 -38 +8 +8 +8 +8 +24 +27 +27 5010500 18816 From f81f0b351fc7882aae2ddf4e156c7fec235d9ad2 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 8 Feb 2023 13:58:55 +0000 Subject: [PATCH 135/717] Automatic style fix --- tests/integration/test_merge_tree_s3/test.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_merge_tree_s3/test.py b/tests/integration/test_merge_tree_s3/test.py index c8c97f4d40f..696c016f760 100644 --- a/tests/integration/test_merge_tree_s3/test.py +++ b/tests/integration/test_merge_tree_s3/test.py @@ -54,8 +54,16 @@ FILES_OVERHEAD = 1 FILES_OVERHEAD_PER_COLUMN = 2 # Data and mark files FILES_OVERHEAD_DEFAULT_COMPRESSION_CODEC = 1 FILES_OVERHEAD_METADATA_VERSION = 1 -FILES_OVERHEAD_PER_PART_WIDE = FILES_OVERHEAD_PER_COLUMN * 3 + 2 + 6 + FILES_OVERHEAD_DEFAULT_COMPRESSION_CODEC + FILES_OVERHEAD_METADATA_VERSION -FILES_OVERHEAD_PER_PART_COMPACT = 10 + FILES_OVERHEAD_DEFAULT_COMPRESSION_CODEC + FILES_OVERHEAD_METADATA_VERSION +FILES_OVERHEAD_PER_PART_WIDE = ( + FILES_OVERHEAD_PER_COLUMN * 3 + + 2 + + 6 + + FILES_OVERHEAD_DEFAULT_COMPRESSION_CODEC + + FILES_OVERHEAD_METADATA_VERSION +) +FILES_OVERHEAD_PER_PART_COMPACT = ( + 10 + FILES_OVERHEAD_DEFAULT_COMPRESSION_CODEC + FILES_OVERHEAD_METADATA_VERSION +) def create_table(node, table_name, **additional_settings): From 5f33aaf59b12d06d2f1dac3412ce8b58723426b0 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 8 Feb 2023 16:31:37 +0100 Subject: [PATCH 136/717] Fix three leftovers --- tests/integration/test_merge_tree_hdfs/test.py | 14 ++++++++++++-- .../test_replicated_merge_tree_s3/test.py | 14 ++++++++++++-- .../test.py | 14 ++++++++++++-- 3 files changed, 36 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_merge_tree_hdfs/test.py b/tests/integration/test_merge_tree_hdfs/test.py index 3950077e619..782237539fa 100644 --- a/tests/integration/test_merge_tree_hdfs/test.py +++ b/tests/integration/test_merge_tree_hdfs/test.py @@ -43,8 +43,18 @@ def create_table(cluster, table_name, additional_settings=None): FILES_OVERHEAD = 1 FILES_OVERHEAD_PER_COLUMN = 2 # Data and mark files -FILES_OVERHEAD_PER_PART_WIDE = FILES_OVERHEAD_PER_COLUMN * 3 + 2 + 6 + 1 -FILES_OVERHEAD_PER_PART_COMPACT = 10 + 1 +FILES_OVERHEAD_DEFAULT_COMPRESSION_CODEC = 1 +FILES_OVERHEAD_METADATA_VERSION = 1 +FILES_OVERHEAD_PER_PART_WIDE = ( + FILES_OVERHEAD_PER_COLUMN * 3 + + 2 + + 6 + + FILES_OVERHEAD_DEFAULT_COMPRESSION_CODEC + + FILES_OVERHEAD_METADATA_VERSION +) +FILES_OVERHEAD_PER_PART_COMPACT = ( + 10 + FILES_OVERHEAD_DEFAULT_COMPRESSION_CODEC + FILES_OVERHEAD_METADATA_VERSION +) @pytest.fixture(scope="module") diff --git a/tests/integration/test_replicated_merge_tree_s3/test.py b/tests/integration/test_replicated_merge_tree_s3/test.py index 0d978bb6967..b90e28dfdb2 100644 --- a/tests/integration/test_replicated_merge_tree_s3/test.py +++ b/tests/integration/test_replicated_merge_tree_s3/test.py @@ -44,8 +44,18 @@ def cluster(): FILES_OVERHEAD = 1 FILES_OVERHEAD_PER_COLUMN = 2 # Data and mark files -FILES_OVERHEAD_PER_PART_WIDE = FILES_OVERHEAD_PER_COLUMN * 3 + 2 + 6 + 1 -FILES_OVERHEAD_PER_PART_COMPACT = 10 + 1 +FILES_OVERHEAD_DEFAULT_COMPRESSION_CODEC = 1 +FILES_OVERHEAD_METADATA_VERSION = 1 +FILES_OVERHEAD_PER_PART_WIDE = ( + FILES_OVERHEAD_PER_COLUMN * 3 + + 2 + + 6 + + FILES_OVERHEAD_DEFAULT_COMPRESSION_CODEC + + FILES_OVERHEAD_METADATA_VERSION +) +FILES_OVERHEAD_PER_PART_COMPACT = ( + 10 + FILES_OVERHEAD_DEFAULT_COMPRESSION_CODEC + FILES_OVERHEAD_METADATA_VERSION +) def random_string(length): diff --git a/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py b/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py index 60a1b9b9746..1b80b80987d 100644 --- a/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py +++ b/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py @@ -45,8 +45,18 @@ def cluster(): FILES_OVERHEAD = 1 FILES_OVERHEAD_PER_COLUMN = 2 # Data and mark files -FILES_OVERHEAD_PER_PART_WIDE = FILES_OVERHEAD_PER_COLUMN * 3 + 2 + 6 + 1 -FILES_OVERHEAD_PER_PART_COMPACT = 10 + 1 +FILES_OVERHEAD_DEFAULT_COMPRESSION_CODEC = 1 +FILES_OVERHEAD_METADATA_VERSION = 1 +FILES_OVERHEAD_PER_PART_WIDE = ( + FILES_OVERHEAD_PER_COLUMN * 3 + + 2 + + 6 + + FILES_OVERHEAD_DEFAULT_COMPRESSION_CODEC + + FILES_OVERHEAD_METADATA_VERSION +) +FILES_OVERHEAD_PER_PART_COMPACT = ( + 10 + FILES_OVERHEAD_DEFAULT_COMPRESSION_CODEC + FILES_OVERHEAD_METADATA_VERSION +) def random_string(length): From 738311c7fa0f63bd983e84e23304d421f9afaa78 Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 8 Feb 2023 15:48:18 +0000 Subject: [PATCH 137/717] Use default of columnt type in insert_null_as_default if column DEFAULT values is not specified --- src/Interpreters/InterpreterInsertQuery.cpp | 2 +- src/Interpreters/inplaceBlockConversions.cpp | 18 +++++++++++++++--- .../02560_null_as_default.reference | 2 ++ .../0_stateless/02560_null_as_default.sql | 7 +++++++ 4 files changed, 25 insertions(+), 4 deletions(-) create mode 100644 tests/queries/0_stateless/02560_null_as_default.reference create mode 100644 tests/queries/0_stateless/02560_null_as_default.sql diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp index 35ce28ec0b6..7dcd43f6134 100644 --- a/src/Interpreters/InterpreterInsertQuery.cpp +++ b/src/Interpreters/InterpreterInsertQuery.cpp @@ -449,7 +449,7 @@ BlockIO InterpreterInsertQuery::execute() { /// Change query sample block columns to Nullable to allow inserting nullable columns, where NULL values will be substituted with /// default column values (in AddingDefaultsTransform), so all values will be cast correctly. - if (input_columns[col_idx].type->isNullable() && !query_columns[col_idx].type->isNullable() && output_columns.hasDefault(query_columns[col_idx].name)) + if (input_columns[col_idx].type->isNullable() && !query_columns[col_idx].type->isNullable() && output_columns.has(query_columns[col_idx].name)) query_sample_block.setColumn(col_idx, ColumnWithTypeAndName(makeNullable(query_columns[col_idx].column), makeNullable(query_columns[col_idx].type), query_columns[col_idx].name)); } } diff --git a/src/Interpreters/inplaceBlockConversions.cpp b/src/Interpreters/inplaceBlockConversions.cpp index a4791690f4e..8f1fb016c62 100644 --- a/src/Interpreters/inplaceBlockConversions.cpp +++ b/src/Interpreters/inplaceBlockConversions.cpp @@ -99,8 +99,10 @@ void addDefaultRequiredExpressionsRecursively( /// This column is required, but doesn't have default expression, so lets use "default default" auto column = columns.get(required_column_name); auto default_value = column.type->getDefault(); - auto default_ast = std::make_shared(default_value); - default_expr_list_accum->children.emplace_back(setAlias(default_ast, required_column_name)); + ASTPtr expr = std::make_shared(default_value); + if (is_column_in_query && convert_null_to_default) + expr = makeASTFunction("ifNull", std::make_shared(required_column_name), std::move(expr)); + default_expr_list_accum->children.emplace_back(setAlias(expr, required_column_name)); added_columns.emplace(required_column_name); } } @@ -173,6 +175,16 @@ void performRequiredConversions(Block & block, const NamesAndTypesList & require } } +bool needConvertAnyNullToDefault(const Block & header, const NamesAndTypesList & required_columns, const ColumnsDescription & columns) +{ + for (const auto & required_column : required_columns) + { + if (columns.has(required_column.name) && header.findByName(required_column.name)->type->isNullable() && !required_column.type->isNullable()) + return true; + } + return false; +} + ActionsDAGPtr evaluateMissingDefaults( const Block & header, const NamesAndTypesList & required_columns, @@ -181,7 +193,7 @@ ActionsDAGPtr evaluateMissingDefaults( bool save_unneeded_columns, bool null_as_default) { - if (!columns.hasDefaults()) + if (!columns.hasDefaults() && (!null_as_default || !needConvertAnyNullToDefault(header, required_columns, columns))) return nullptr; ASTPtr expr_list = defaultRequiredExpressions(header, required_columns, columns, null_as_default); diff --git a/tests/queries/0_stateless/02560_null_as_default.reference b/tests/queries/0_stateless/02560_null_as_default.reference new file mode 100644 index 00000000000..a8e497ca982 --- /dev/null +++ b/tests/queries/0_stateless/02560_null_as_default.reference @@ -0,0 +1,2 @@ +42 +0 diff --git a/tests/queries/0_stateless/02560_null_as_default.sql b/tests/queries/0_stateless/02560_null_as_default.sql new file mode 100644 index 00000000000..5fa0874fbf3 --- /dev/null +++ b/tests/queries/0_stateless/02560_null_as_default.sql @@ -0,0 +1,7 @@ +drop table if exists test; +create table test (x UInt64) engine=Memory(); +set insert_null_as_default=1; +insert into test select number % 2 ? NULL : 42 as x from numbers(2); +select * from test; +drop table test; + From 242d6d0cb8b928da0d93adc4541d864b33f9fd83 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 8 Feb 2023 17:25:55 +0100 Subject: [PATCH 138/717] better test --- .../02555_davengers_rename_chain.sh | 143 ++++++++++++++++++ .../02555_davengers_rename_chain.sql | 91 ----------- 2 files changed, 143 insertions(+), 91 deletions(-) create mode 100755 tests/queries/0_stateless/02555_davengers_rename_chain.sh delete mode 100644 tests/queries/0_stateless/02555_davengers_rename_chain.sql diff --git a/tests/queries/0_stateless/02555_davengers_rename_chain.sh b/tests/queries/0_stateless/02555_davengers_rename_chain.sh new file mode 100755 index 00000000000..71201537170 --- /dev/null +++ b/tests/queries/0_stateless/02555_davengers_rename_chain.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +# Tags: replica +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS wrong_metadata" + +$CLICKHOUSE_CLIENT -n --query="CREATE TABLE wrong_metadata( + a UInt64, + b UInt64, + c UInt64 +) +ENGINE ReplicatedMergeTree('/test/{database}/tables/wrong_metadata', '1') +ORDER BY tuple() +SETTINGS min_bytes_for_wide_part = 0" + +$CLICKHOUSE_CLIENT --query="INSERT INTO wrong_metadata VALUES (1, 2, 3)" + + +$CLICKHOUSE_CLIENT --query="SYSTEM STOP MERGES wrong_metadata" + + +$CLICKHOUSE_CLIENT --query="ALTER TABLE wrong_metadata RENAME COLUMN a TO a1, RENAME COLUMN b to b1 SETTINGS replication_alter_partitions_sync = 0" + +counter=0 retries=60 +I=0 +while [[ $counter -lt $retries ]]; do + I=$((I + 1)) + result=$($CLICKHOUSE_CLIENT --query "show create table wrong_metadata") + if [[ $result == *"a1 UInt64"* ]]; then + break; + fi + sleep 0.1 + ((++counter)) +done + + +$CLICKHOUSE_CLIENT --query="SELECT * FROM wrong_metadata ORDER BY a1 FORMAT JSONEachRow" + +$CLICKHOUSE_CLIENT --query="SELECT '~~~~~~~'" + +$CLICKHOUSE_CLIENT --query="INSERT INTO wrong_metadata VALUES (4, 5, 6)" + + +$CLICKHOUSE_CLIENT --query="SELECT * FROM wrong_metadata ORDER BY a1 FORMAT JSONEachRow" +$CLICKHOUSE_CLIENT --query="SELECT '~~~~~~~'" + + +$CLICKHOUSE_CLIENT --query="ALTER TABLE wrong_metadata RENAME COLUMN a1 TO b, RENAME COLUMN b1 to a SETTINGS replication_alter_partitions_sync = 0" + +counter=0 retries=60 +I=0 +while [[ $counter -lt $retries ]]; do + I=$((I + 1)) + result=$($CLICKHOUSE_CLIENT --query "show create table wrong_metadata") + if [[ $result == *"b UInt64"* ]]; then + break; + fi + sleep 0.1 + ((++counter)) +done + +$CLICKHOUSE_CLIENT --query="INSERT INTO wrong_metadata VALUES (7, 8, 9)" + +$CLICKHOUSE_CLIENT --query="SELECT * FROM wrong_metadata ORDER by a1 FORMAT JSONEachRow" +$CLICKHOUSE_CLIENT --query="SELECT '~~~~~~~'" + +$CLICKHOUSE_CLIENT --query="SYSTEM START MERGES wrong_metadata" + +$CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA wrong_metadata" + +$CLICKHOUSE_CLIENT --query="SELECT * FROM wrong_metadata order by a FORMAT JSONEachRow" + +$CLICKHOUSE_CLIENT --query="SELECT '~~~~~~~'" + + +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS wrong_metadata" + +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS wrong_metadata_compact" + +$CLICKHOUSE_CLIENT -n --query="CREATE TABLE wrong_metadata_compact( + a UInt64, + b UInt64, + c UInt64 +) +ENGINE ReplicatedMergeTree('/test/{database}/tables/wrong_metadata_compact', '1') +ORDER BY tuple() +SETTINGS min_bytes_for_wide_part = 10000000" + +$CLICKHOUSE_CLIENT --query="INSERT INTO wrong_metadata_compact VALUES (1, 2, 3)" + +$CLICKHOUSE_CLIENT --query="SYSTEM STOP MERGES wrong_metadata_compact" + +$CLICKHOUSE_CLIENT --query="ALTER TABLE wrong_metadata_compact RENAME COLUMN a TO a1, RENAME COLUMN b to b1 SETTINGS replication_alter_partitions_sync = 0" + +counter=0 retries=60 +I=0 +while [[ $counter -lt $retries ]]; do + I=$((I + 1)) + result=$($CLICKHOUSE_CLIENT --query "show create table wrong_metadata_compact") + if [[ $result == *"b1 UInt64"* ]]; then + break; + fi + sleep 0.1 + ((++counter)) +done + +$CLICKHOUSE_CLIENT --query="SELECT * FROM wrong_metadata_compact ORDER BY a1 FORMAT JSONEachRow" +$CLICKHOUSE_CLIENT --query="SELECT '~~~~~~~'" + +$CLICKHOUSE_CLIENT --query="INSERT INTO wrong_metadata_compact VALUES (4, 5, 6)" + +$CLICKHOUSE_CLIENT --query="SELECT * FROM wrong_metadata_compact ORDER BY a1 FORMAT JSONEachRow" +$CLICKHOUSE_CLIENT --query="SELECT '~~~~~~~'" + +$CLICKHOUSE_CLIENT --query="ALTER TABLE wrong_metadata_compact RENAME COLUMN a1 TO b, RENAME COLUMN b1 to a SETTINGS replication_alter_partitions_sync = 0" + +counter=0 retries=60 +I=0 +while [[ $counter -lt $retries ]]; do + I=$((I + 1)) + result=$($CLICKHOUSE_CLIENT --query "show create table wrong_metadata_compact") + if [[ $result == *"b UInt64"* ]]; then + break; + fi + sleep 0.1 + ((++counter)) +done + +$CLICKHOUSE_CLIENT --query="INSERT INTO wrong_metadata_compact VALUES (7, 8, 9)" + +$CLICKHOUSE_CLIENT --query="SELECT * FROM wrong_metadata_compact ORDER by a1 FORMAT JSONEachRow" +$CLICKHOUSE_CLIENT --query="SELECT '~~~~~~~'" + +$CLICKHOUSE_CLIENT --query="SYSTEM START MERGES wrong_metadata_compact" + +$CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA wrong_metadata_compact" + +$CLICKHOUSE_CLIENT --query="SELECT * FROM wrong_metadata_compact order by a FORMAT JSONEachRow" +$CLICKHOUSE_CLIENT --query="SELECT '~~~~~~~'" + +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS wrong_metadata_compact" diff --git a/tests/queries/0_stateless/02555_davengers_rename_chain.sql b/tests/queries/0_stateless/02555_davengers_rename_chain.sql deleted file mode 100644 index eae345d0472..00000000000 --- a/tests/queries/0_stateless/02555_davengers_rename_chain.sql +++ /dev/null @@ -1,91 +0,0 @@ -DROP TABLE IF EXISTS wrong_metadata; - -CREATE TABLE wrong_metadata( - a UInt64, - b UInt64, - c UInt64 -) -ENGINE ReplicatedMergeTree('/test/{database}/tables/wrong_metadata', '1') -ORDER BY tuple() -SETTINGS min_bytes_for_wide_part = 0; - -INSERT INTO wrong_metadata VALUES (1, 2, 3); - -SYSTEM STOP MERGES wrong_metadata; - -ALTER TABLE wrong_metadata RENAME COLUMN a TO a1, RENAME COLUMN b to b1 SETTINGS replication_alter_partitions_sync = 0; - -SELECT sleep(1) FORMAT Null; - -SELECT * FROM wrong_metadata ORDER BY a1 FORMAT JSONEachRow; -SELECT '~~~~~~~'; - -INSERT INTO wrong_metadata VALUES (4, 5, 6); - -SELECT * FROM wrong_metadata ORDER BY a1 FORMAT JSONEachRow; -SELECT '~~~~~~~'; - -ALTER TABLE wrong_metadata RENAME COLUMN a1 TO b, RENAME COLUMN b1 to a SETTINGS replication_alter_partitions_sync = 0; - -SELECT sleep(1) FORMAT Null; -SELECT sleep(1) FORMAT Null; - -INSERT INTO wrong_metadata VALUES (7, 8, 9); - -SELECT * FROM wrong_metadata ORDER by a1 FORMAT JSONEachRow; -SELECT '~~~~~~~'; - -SYSTEM START MERGES wrong_metadata; - -SYSTEM SYNC REPLICA wrong_metadata; - -SELECT * FROM wrong_metadata order by a FORMAT JSONEachRow; -SELECT '~~~~~~~'; - -DROP TABLE IF EXISTS wrong_metadata; - -DROP TABLE IF EXISTS wrong_metadata_compact; - -CREATE TABLE wrong_metadata_compact( - a UInt64, - b UInt64, - c UInt64 -) -ENGINE ReplicatedMergeTree('/test/{database}/tables/wrong_metadata_compact', '1') -ORDER BY tuple() -SETTINGS min_bytes_for_wide_part = 10000000; - -INSERT INTO wrong_metadata_compact VALUES (1, 2, 3); - -SYSTEM STOP MERGES wrong_metadata_compact; - -ALTER TABLE wrong_metadata_compact RENAME COLUMN a TO a1, RENAME COLUMN b to b1 SETTINGS replication_alter_partitions_sync = 0; - -SELECT sleep(1) FORMAT Null; - -SELECT * FROM wrong_metadata_compact ORDER BY a1 FORMAT JSONEachRow; -SELECT '~~~~~~~'; - -INSERT INTO wrong_metadata_compact VALUES (4, 5, 6); - -SELECT * FROM wrong_metadata_compact ORDER BY a1 FORMAT JSONEachRow; -SELECT '~~~~~~~'; - -ALTER TABLE wrong_metadata_compact RENAME COLUMN a1 TO b, RENAME COLUMN b1 to a SETTINGS replication_alter_partitions_sync = 0; - -SELECT sleep(1) FORMAT Null; -SELECT sleep(1) FORMAT Null; - -INSERT INTO wrong_metadata_compact VALUES (7, 8, 9); - -SELECT * FROM wrong_metadata_compact ORDER by a1 FORMAT JSONEachRow; -SELECT '~~~~~~~'; - -SYSTEM START MERGES wrong_metadata_compact; - -SYSTEM SYNC REPLICA wrong_metadata_compact; - -SELECT * FROM wrong_metadata_compact order by a FORMAT JSONEachRow; -SELECT '~~~~~~~'; - -DROP TABLE IF EXISTS wrong_metadata_compact; From c3e8dd8984634e57acfa76aebf9999895e44b490 Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 8 Feb 2023 19:14:28 +0000 Subject: [PATCH 139/717] Fix low cardinality case --- src/Columns/ColumnLowCardinality.cpp | 14 ++++++++++++++ src/Columns/ColumnLowCardinality.h | 3 +++ src/Columns/ColumnNullable.cpp | 18 ++++++++++++++++++ src/Columns/ColumnNullable.h | 1 + src/DataTypes/DataTypeNullable.cpp | 16 ++++++++++++++++ src/DataTypes/DataTypeNullable.h | 1 + src/DataTypes/IDataType.h | 7 ++++++- src/DataTypes/hasNullable.cpp | 2 +- src/Formats/EscapingRuleUtils.cpp | 2 +- src/Formats/JSONUtils.cpp | 2 +- src/Formats/SchemaInferenceUtils.cpp | 2 +- src/Interpreters/InterpreterInsertQuery.cpp | 4 ++-- .../Formats/Impl/CSVRowInputFormat.cpp | 2 +- .../Formats/Impl/MySQLDumpRowInputFormat.cpp | 2 +- .../Formats/Impl/TSKVRowInputFormat.cpp | 2 +- .../Impl/TabSeparatedRowInputFormat.cpp | 4 ++-- .../Formats/Impl/ValuesBlockInputFormat.cpp | 2 +- .../MergeTree/MergeTreeIndexFullText.cpp | 2 +- .../MergeTree/MergeTreeIndexInverted.cpp | 2 +- 19 files changed, 73 insertions(+), 15 deletions(-) diff --git a/src/Columns/ColumnLowCardinality.cpp b/src/Columns/ColumnLowCardinality.cpp index ecdaf240e5e..73e9eecb823 100644 --- a/src/Columns/ColumnLowCardinality.cpp +++ b/src/Columns/ColumnLowCardinality.cpp @@ -310,6 +310,13 @@ MutableColumnPtr ColumnLowCardinality::cloneResized(size_t size) const return ColumnLowCardinality::create(IColumn::mutate(std::move(unique_ptr)), getIndexes().cloneResized(size)); } +MutableColumnPtr ColumnLowCardinality::cloneNullable() const +{ + auto res = cloneFinalized(); + assert_cast(*res).nestedToNullable(); + return res; +} + int ColumnLowCardinality::compareAtImpl(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint, const Collator * collator) const { const auto & low_cardinality_column = assert_cast(rhs); @@ -830,4 +837,11 @@ void ColumnLowCardinality::Dictionary::compact(ColumnPtr & positions) shared = false; } +bool isColumnLowCardinalityNullable(const IColumn & column) +{ + if (const auto * lc_column = checkAndGetColumn(column)) + return lc_column->nestedIsNullable(); + return false; +} + } diff --git a/src/Columns/ColumnLowCardinality.h b/src/Columns/ColumnLowCardinality.h index e895bc6b54e..db0100b0aa9 100644 --- a/src/Columns/ColumnLowCardinality.h +++ b/src/Columns/ColumnLowCardinality.h @@ -219,6 +219,7 @@ public: bool nestedCanBeInsideNullable() const { return dictionary.getColumnUnique().getNestedColumn()->canBeInsideNullable(); } void nestedToNullable() { dictionary.getColumnUnique().nestedToNullable(); } void nestedRemoveNullable() { dictionary.getColumnUnique().nestedRemoveNullable(); } + MutableColumnPtr cloneNullable() const; const IColumnUnique & getDictionary() const { return dictionary.getColumnUnique(); } IColumnUnique & getDictionary() { return dictionary.getColumnUnique(); } @@ -360,5 +361,7 @@ private: void getPermutationImpl(IColumn::PermutationSortDirection direction, IColumn::PermutationSortStability stability, size_t limit, int nan_direction_hint, Permutation & res, const Collator * collator = nullptr) const; }; +bool isColumnLowCardinalityNullable(const IColumn & column); + } diff --git a/src/Columns/ColumnNullable.cpp b/src/Columns/ColumnNullable.cpp index 9398c66bef0..db99cee54fb 100644 --- a/src/Columns/ColumnNullable.cpp +++ b/src/Columns/ColumnNullable.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #if USE_EMBEDDED_COMPILER @@ -792,6 +793,23 @@ ColumnPtr makeNullable(const ColumnPtr & column) return ColumnNullable::create(column, ColumnUInt8::create(column->size(), 0)); } +ColumnPtr makeNullableOrLowCardinalityNullable(const ColumnPtr & column) +{ + if (isColumnNullable(*column)) + return column; + + if (isColumnLowCardinalityNullable(*column)) + return column; + + if (isColumnConst(*column)) + return ColumnConst::create(makeNullable(assert_cast(*column).getDataColumnPtr()), column->size()); + + if (column->lowCardinality()) + return assert_cast(*column).cloneNullable(); + + return ColumnNullable::create(column, ColumnUInt8::create(column->size(), 0)); +} + ColumnPtr makeNullableSafe(const ColumnPtr & column) { if (isColumnNullable(*column)) diff --git a/src/Columns/ColumnNullable.h b/src/Columns/ColumnNullable.h index 85bf095a9d1..7574cc0b501 100644 --- a/src/Columns/ColumnNullable.h +++ b/src/Columns/ColumnNullable.h @@ -220,5 +220,6 @@ private: ColumnPtr makeNullable(const ColumnPtr & column); ColumnPtr makeNullableSafe(const ColumnPtr & column); +ColumnPtr makeNullableOrLowCardinalityNullable(const ColumnPtr & column); } diff --git a/src/DataTypes/DataTypeNullable.cpp b/src/DataTypes/DataTypeNullable.cpp index 2dec5a41978..41a9a1de543 100644 --- a/src/DataTypes/DataTypeNullable.cpp +++ b/src/DataTypes/DataTypeNullable.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -99,4 +100,19 @@ DataTypePtr removeNullable(const DataTypePtr & type) return type; } +DataTypePtr makeNullableOrLowCardinalityNullable(const DataTypePtr & type) +{ + if (isNullableOrLowCardinalityNullable(type)) + return type; + + if (type->lowCardinality()) + { + const auto & dictionary_type = assert_cast(*type).getDictionaryType(); + return std::make_shared(makeNullable(dictionary_type)); + } + + return std::make_shared(type); +} + + } diff --git a/src/DataTypes/DataTypeNullable.h b/src/DataTypes/DataTypeNullable.h index 379119b364c..06d46fb15ed 100644 --- a/src/DataTypes/DataTypeNullable.h +++ b/src/DataTypes/DataTypeNullable.h @@ -53,5 +53,6 @@ private: DataTypePtr makeNullable(const DataTypePtr & type); DataTypePtr makeNullableSafe(const DataTypePtr & type); DataTypePtr removeNullable(const DataTypePtr & type); +DataTypePtr makeNullableOrLowCardinalityNullable(const DataTypePtr & type); } diff --git a/src/DataTypes/IDataType.h b/src/DataTypes/IDataType.h index bafe03dbc3a..e5bdbeca69e 100644 --- a/src/DataTypes/IDataType.h +++ b/src/DataTypes/IDataType.h @@ -392,7 +392,7 @@ struct WhichDataType constexpr bool isAggregateFunction() const { return idx == TypeIndex::AggregateFunction; } constexpr bool isSimple() const { return isInt() || isUInt() || isFloat() || isString(); } - constexpr bool isLowCarnality() const { return idx == TypeIndex::LowCardinality; } + constexpr bool isLowCardinality() const { return idx == TypeIndex::LowCardinality; } }; /// IDataType helpers (alternative for IDataType virtual methods with single point of truth) @@ -548,6 +548,11 @@ inline bool isAggregateFunction(const DataTypePtr & data_type) return which.isAggregateFunction(); } +inline bool isNullableOrLowCardinalityNullable(const DataTypePtr & data_type) +{ + return data_type->isNullable() || data_type->isLowCardinalityNullable(); +} + template constexpr bool IsDataTypeDecimal = false; template constexpr bool IsDataTypeNumber = false; template constexpr bool IsDataTypeDateOrDateTime = false; diff --git a/src/DataTypes/hasNullable.cpp b/src/DataTypes/hasNullable.cpp index 2c699806874..908b9880473 100644 --- a/src/DataTypes/hasNullable.cpp +++ b/src/DataTypes/hasNullable.cpp @@ -8,7 +8,7 @@ namespace DB bool hasNullable(const DataTypePtr & type) { - if (type->isNullable() || type->isLowCardinalityNullable()) + if (isNullableOrLowCardinalityNullable(type)) return true; if (const DataTypeArray * type_array = typeid_cast(type.get())) diff --git a/src/Formats/EscapingRuleUtils.cpp b/src/Formats/EscapingRuleUtils.cpp index 7c8c3460235..804f32e4b46 100644 --- a/src/Formats/EscapingRuleUtils.cpp +++ b/src/Formats/EscapingRuleUtils.cpp @@ -104,7 +104,7 @@ bool deserializeFieldByEscapingRule( const FormatSettings & format_settings) { bool read = true; - bool parse_as_nullable = format_settings.null_as_default && !type->isNullable() && !type->isLowCardinalityNullable(); + bool parse_as_nullable = format_settings.null_as_default && !isNullableOrLowCardinalityNullable(type); switch (escaping_rule) { case FormatSettings::EscapingRule::Escaped: diff --git a/src/Formats/JSONUtils.cpp b/src/Formats/JSONUtils.cpp index 148c51938fc..6580d98fe98 100644 --- a/src/Formats/JSONUtils.cpp +++ b/src/Formats/JSONUtils.cpp @@ -198,7 +198,7 @@ namespace JSONUtils { try { - bool as_nullable = format_settings.null_as_default && !type->isNullable() && !type->isLowCardinalityNullable(); + bool as_nullable = format_settings.null_as_default && !isNullableOrLowCardinalityNullable(type); if (yield_strings) { diff --git a/src/Formats/SchemaInferenceUtils.cpp b/src/Formats/SchemaInferenceUtils.cpp index b20a15a4613..7a242a9f81c 100644 --- a/src/Formats/SchemaInferenceUtils.cpp +++ b/src/Formats/SchemaInferenceUtils.cpp @@ -1084,7 +1084,7 @@ DataTypePtr makeNullableRecursively(DataTypePtr type) return key_type && value_type ? std::make_shared(removeNullable(key_type), value_type) : nullptr; } - if (which.isLowCarnality()) + if (which.isLowCardinality()) { const auto * lc_type = assert_cast(type.get()); auto nested_type = makeNullableRecursively(lc_type->getDictionaryType()); diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp index 7dcd43f6134..2569e6ddc33 100644 --- a/src/Interpreters/InterpreterInsertQuery.cpp +++ b/src/Interpreters/InterpreterInsertQuery.cpp @@ -449,8 +449,8 @@ BlockIO InterpreterInsertQuery::execute() { /// Change query sample block columns to Nullable to allow inserting nullable columns, where NULL values will be substituted with /// default column values (in AddingDefaultsTransform), so all values will be cast correctly. - if (input_columns[col_idx].type->isNullable() && !query_columns[col_idx].type->isNullable() && output_columns.has(query_columns[col_idx].name)) - query_sample_block.setColumn(col_idx, ColumnWithTypeAndName(makeNullable(query_columns[col_idx].column), makeNullable(query_columns[col_idx].type), query_columns[col_idx].name)); + if (isNullableOrLowCardinalityNullable(input_columns[col_idx].type) && !isNullableOrLowCardinalityNullable(query_columns[col_idx].type) && output_columns.has(query_columns[col_idx].name)) + query_sample_block.setColumn(col_idx, ColumnWithTypeAndName(makeNullableOrLowCardinalityNullable(query_columns[col_idx].column), makeNullableOrLowCardinalityNullable(query_columns[col_idx].type), query_columns[col_idx].name)); } } } diff --git a/src/Processors/Formats/Impl/CSVRowInputFormat.cpp b/src/Processors/Formats/Impl/CSVRowInputFormat.cpp index 18e5b4dfc13..ad8d88a2b65 100644 --- a/src/Processors/Formats/Impl/CSVRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/CSVRowInputFormat.cpp @@ -293,7 +293,7 @@ bool CSVFormatReader::readField( return false; } - if (format_settings.null_as_default && !type->isNullable() && !type->isLowCardinalityNullable()) + if (format_settings.null_as_default && !isNullableOrLowCardinalityNullable(type)) { /// If value is null but type is not nullable then use default value instead. return SerializationNullable::deserializeTextCSVImpl(column, *buf, format_settings, serialization); diff --git a/src/Processors/Formats/Impl/MySQLDumpRowInputFormat.cpp b/src/Processors/Formats/Impl/MySQLDumpRowInputFormat.cpp index bf55fe88469..90dd07bd5a8 100644 --- a/src/Processors/Formats/Impl/MySQLDumpRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/MySQLDumpRowInputFormat.cpp @@ -389,7 +389,7 @@ bool MySQLDumpRowInputFormat::readField(IColumn & column, size_t column_idx) { const auto & type = types[column_idx]; const auto & serialization = serializations[column_idx]; - if (format_settings.null_as_default && !type->isNullable() && !type->isLowCardinalityNullable()) + if (format_settings.null_as_default && !isNullableOrLowCardinalityNullable(type)) return SerializationNullable::deserializeTextQuotedImpl(column, *in, format_settings, serialization); serialization->deserializeTextQuoted(column, *in, format_settings); diff --git a/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp b/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp index bf6d0ab88d2..448b6e26387 100644 --- a/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp @@ -146,7 +146,7 @@ bool TSKVRowInputFormat::readRow(MutableColumns & columns, RowReadExtension & ex seen_columns[index] = read_columns[index] = true; const auto & type = getPort().getHeader().getByPosition(index).type; const auto & serialization = serializations[index]; - if (format_settings.null_as_default && !type->isNullable() && !type->isLowCardinalityNullable()) + if (format_settings.null_as_default && !isNullableOrLowCardinalityNullable(type)) read_columns[index] = SerializationNullable::deserializeTextEscapedImpl(*columns[index], *in, format_settings, serialization); else serialization->deserializeTextEscaped(*columns[index], *in, format_settings); diff --git a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp index 868639e66c2..23a0f3a0bc2 100644 --- a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp @@ -156,7 +156,7 @@ bool TabSeparatedFormatReader::readField(IColumn & column, const DataTypePtr & t return false; } - bool as_nullable = format_settings.null_as_default && !type->isNullable() && !type->isLowCardinalityNullable(); + bool as_nullable = format_settings.null_as_default && !isNullableOrLowCardinalityNullable(type); if (is_raw) { @@ -242,7 +242,7 @@ bool TabSeparatedFormatReader::parseRowEndWithDiagnosticInfo(WriteBuffer & out) void TabSeparatedFormatReader::checkNullValueForNonNullable(DataTypePtr type) { - bool can_be_parsed_as_null = type->isNullable() || type->isLowCardinalityNullable() || format_settings.null_as_default; + bool can_be_parsed_as_null = isNullableOrLowCardinalityNullable(type) || format_settings.null_as_default; // check null value for type is not nullable. don't cross buffer bound for simplicity, so maybe missing some case if (!can_be_parsed_as_null && !buf->eof()) diff --git a/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp b/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp index 9511b37ff15..3e505424bd9 100644 --- a/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp @@ -272,7 +272,7 @@ bool ValuesBlockInputFormat::tryReadValue(IColumn & column, size_t column_idx) { const auto & type = types[column_idx]; const auto & serialization = serializations[column_idx]; - if (format_settings.null_as_default && !type->isNullable() && !type->isLowCardinalityNullable()) + if (format_settings.null_as_default && !isNullableOrLowCardinalityNullable(type)) read = SerializationNullable::deserializeTextQuotedImpl(column, *buf, format_settings, serialization); else serialization->deserializeTextQuoted(column, *buf, format_settings); diff --git a/src/Storages/MergeTree/MergeTreeIndexFullText.cpp b/src/Storages/MergeTree/MergeTreeIndexFullText.cpp index 35ca484cff0..abb51811006 100644 --- a/src/Storages/MergeTree/MergeTreeIndexFullText.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexFullText.cpp @@ -737,7 +737,7 @@ void bloomFilterIndexValidator(const IndexDescription & index, bool /*attach*/) const auto & array_type = assert_cast(*index_data_type); data_type = WhichDataType(array_type.getNestedType()); } - else if (data_type.isLowCarnality()) + else if (data_type.isLowCardinality()) { const auto & low_cardinality = assert_cast(*index_data_type); data_type = WhichDataType(low_cardinality.getDictionaryType()); diff --git a/src/Storages/MergeTree/MergeTreeIndexInverted.cpp b/src/Storages/MergeTree/MergeTreeIndexInverted.cpp index 02222aa530c..c8f5d40bedd 100644 --- a/src/Storages/MergeTree/MergeTreeIndexInverted.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexInverted.cpp @@ -747,7 +747,7 @@ void invertedIndexValidator(const IndexDescription & index, bool /*attach*/) const auto & gin_type = assert_cast(*index_data_type); data_type = WhichDataType(gin_type.getNestedType()); } - else if (data_type.isLowCarnality()) + else if (data_type.isLowCardinality()) { const auto & low_cardinality = assert_cast(*index_data_type); data_type = WhichDataType(low_cardinality.getDictionaryType()); From 85dd3303746a693c61579901cca3c4994a4f2865 Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 8 Feb 2023 19:24:04 +0000 Subject: [PATCH 140/717] Add test, fix low cardinality again --- src/Interpreters/inplaceBlockConversions.cpp | 4 ++-- tests/queries/0_stateless/02560_null_as_default.reference | 2 ++ tests/queries/0_stateless/02560_null_as_default.sql | 5 +++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/inplaceBlockConversions.cpp b/src/Interpreters/inplaceBlockConversions.cpp index 8f1fb016c62..ed606fff825 100644 --- a/src/Interpreters/inplaceBlockConversions.cpp +++ b/src/Interpreters/inplaceBlockConversions.cpp @@ -44,7 +44,7 @@ void addDefaultRequiredExpressionsRecursively( bool convert_null_to_default = false; if (is_column_in_query) - convert_null_to_default = null_as_default && block.findByName(required_column_name)->type->isNullable() && !required_column_type->isNullable(); + convert_null_to_default = null_as_default && isNullableOrLowCardinalityNullable(block.findByName(required_column_name)->type) && !isNullableOrLowCardinalityNullable(required_column_type); if ((is_column_in_query && !convert_null_to_default) || added_columns.contains(required_column_name)) return; @@ -179,7 +179,7 @@ bool needConvertAnyNullToDefault(const Block & header, const NamesAndTypesList & { for (const auto & required_column : required_columns) { - if (columns.has(required_column.name) && header.findByName(required_column.name)->type->isNullable() && !required_column.type->isNullable()) + if (columns.has(required_column.name) && isNullableOrLowCardinalityNullable(header.findByName(required_column.name)->type) && !isNullableOrLowCardinalityNullable(required_column.type)) return true; } return false; diff --git a/tests/queries/0_stateless/02560_null_as_default.reference b/tests/queries/0_stateless/02560_null_as_default.reference index a8e497ca982..517cbfb1e27 100644 --- a/tests/queries/0_stateless/02560_null_as_default.reference +++ b/tests/queries/0_stateless/02560_null_as_default.reference @@ -1,2 +1,4 @@ 42 0 +World +Hello diff --git a/tests/queries/0_stateless/02560_null_as_default.sql b/tests/queries/0_stateless/02560_null_as_default.sql index 5fa0874fbf3..0a7da55a210 100644 --- a/tests/queries/0_stateless/02560_null_as_default.sql +++ b/tests/queries/0_stateless/02560_null_as_default.sql @@ -5,3 +5,8 @@ insert into test select number % 2 ? NULL : 42 as x from numbers(2); select * from test; drop table test; +create table test (x LowCardinality(String) default 'Hello') engine=Memory(); +insert into test select (number % 2 ? NULL : 'World')::LowCardinality(Nullable(String)) from numbers(2); +select * from test; +drop table test; + From 00686ae03b377482a2ae1ee335d4e61693a22880 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 8 Feb 2023 22:34:09 +0100 Subject: [PATCH 141/717] Fix tests again --- tests/integration/test_partition/test.py | 2 ++ tests/integration/test_s3_zero_copy_ttl/test.py | 4 ++-- .../02241_filesystem_cache_on_write_operations.reference | 8 ++++---- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_partition/test.py b/tests/integration/test_partition/test.py index ae4393fc6f6..b59cc21f39f 100644 --- a/tests/integration/test_partition/test.py +++ b/tests/integration/test_partition/test.py @@ -105,6 +105,8 @@ def partition_complex_assert_checksums(): "c4ca4238a0b923820dcc509a6f75849b\tshadow/1/data/test/partition_complex/19700102_2_2_0/count.txt\n" "c4ca4238a0b923820dcc509a6f75849b\tshadow/1/data/test/partition_complex/19700201_1_1_0/count.txt\n" "cfcb770c3ecd0990dcceb1bde129e6c6\tshadow/1/data/test/partition_complex/19700102_2_2_0/p.bin\n" + "cfcd208495d565ef66e7dff9f98764da\tshadow/1/data/test/partition_complex/19700102_2_2_0/metadata_version.txt\n" + "cfcd208495d565ef66e7dff9f98764da\tshadow/1/data/test/partition_complex/19700201_1_1_0/metadata_version.txt\n" "e2af3bef1fd129aea73a890ede1e7a30\tshadow/1/data/test/partition_complex/19700201_1_1_0/k.bin\n" "f2312862cc01adf34a93151377be2ddf\tshadow/1/data/test/partition_complex/19700201_1_1_0/minmax_p.idx\n" ) diff --git a/tests/integration/test_s3_zero_copy_ttl/test.py b/tests/integration/test_s3_zero_copy_ttl/test.py index 9a782aacef6..7dcf3734653 100644 --- a/tests/integration/test_s3_zero_copy_ttl/test.py +++ b/tests/integration/test_s3_zero_copy_ttl/test.py @@ -86,9 +86,9 @@ def test_ttl_move_and_s3(started_cluster): print(f"Total objects: {counter}") - if counter == 300: + if counter == 330: break print(f"Attempts remaining: {attempt}") - assert counter == 300 + assert counter == 330 diff --git a/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.reference b/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.reference index 7fa38f6ff44..f3fac9b32d3 100644 --- a/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.reference +++ b/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.reference @@ -24,8 +24,8 @@ state: DOWNLOADED 8 8 24 -27 -27 +35 +43 5010500 18816 Using storage policy: local_cache @@ -54,7 +54,7 @@ state: DOWNLOADED 8 8 24 -27 -27 +35 +43 5010500 18816 From 0a823854137271ae19c3b498f7b1cfa2923bb220 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 9 Feb 2023 12:23:18 +0100 Subject: [PATCH 142/717] Blind fix for trash test --- tests/integration/test_merge_tree_s3_failover/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_merge_tree_s3_failover/test.py b/tests/integration/test_merge_tree_s3_failover/test.py index d4c691fdb55..9194a1b68f6 100644 --- a/tests/integration/test_merge_tree_s3_failover/test.py +++ b/tests/integration/test_merge_tree_s3_failover/test.py @@ -89,7 +89,7 @@ def drop_table(cluster): # S3 request will be failed for an appropriate part file write. -FILES_PER_PART_BASE = 5 # partition.dat, default_compression_codec.txt, count.txt, columns.txt, checksums.txt +FILES_PER_PART_BASE = 6 # partition.dat, metadata_version.txt, default_compression_codec.txt, count.txt, columns.txt, checksums.txt FILES_PER_PART_WIDE = ( FILES_PER_PART_BASE + 1 + 1 + 3 * 2 ) # Primary index, MinMax, Mark and data file for column(s) From df23731061d89c76b3f66da42cde36e2915380d6 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Thu, 9 Feb 2023 11:37:15 +0000 Subject: [PATCH 143/717] Removing DISTINCT on top of aggregation if possible --- src/Processors/QueryPlan/DistinctStep.h | 3 +- .../Optimizations/removeRedundantDistinct.cpp | 187 ++++++++++++++---- src/Processors/QueryPlan/RollupStep.h | 2 + .../02500_remove_redundant_distinct.reference | 61 ++++++ .../02500_remove_redundant_distinct.sh | 36 ++++ 5 files changed, 250 insertions(+), 39 deletions(-) diff --git a/src/Processors/QueryPlan/DistinctStep.h b/src/Processors/QueryPlan/DistinctStep.h index 57205b72030..d59adbbf92d 100644 --- a/src/Processors/QueryPlan/DistinctStep.h +++ b/src/Processors/QueryPlan/DistinctStep.h @@ -18,6 +18,7 @@ public: bool optimize_distinct_in_order_); String getName() const override { return "Distinct"; } + const Names & getColumnNames() const { return columns; } void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) override; @@ -33,7 +34,7 @@ private: SizeLimits set_size_limits; UInt64 limit_hint; - Names columns; + const Names columns; bool pre_distinct; bool optimize_distinct_in_order; }; diff --git a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp index 576525bf9a3..e37e08cd7b8 100644 --- a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp +++ b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp @@ -1,14 +1,22 @@ #include #include +#include #include +#include #include #include #include #include #include #include +#include +#include #include +#include +#include +#include #include +#include #include #include @@ -17,23 +25,29 @@ namespace DB::QueryPlanOptimizations namespace { - constexpr bool debug_logging_enabled = false; + constexpr bool debug_logging_enabled = true; void logActionsDAG(const String & prefix, const ActionsDAGPtr & actions) { if constexpr (debug_logging_enabled) - { LOG_DEBUG(&Poco::Logger::get("redundantDistinct"), "{}: {}", prefix, actions->dumpDAG()); - } } - std::set getDistinctColumns(const DistinctStep * distinct) + void logDebug(String key, String value) + { + if constexpr (debug_logging_enabled) + LOG_DEBUG(&Poco::Logger::get("redundantDistinct"), "{} : {}", key, value); + } + + using DistinctColumns = std::set; + DistinctColumns getDistinctColumns(const DistinctStep * distinct) { /// find non-const columns in DISTINCT const ColumnsWithTypeAndName & distinct_columns = distinct->getOutputStream().header.getColumnsWithTypeAndName(); std::set non_const_columns; + const Names & column_names = distinct->getColumnNames(); for (const auto & column : distinct_columns) { - if (!isColumnConst(*column.column)) + if (!isColumnConst(*column.column) && find(cbegin(column_names), cend(column_names), column.name) != column_names.cend()) non_const_columns.emplace(column.name); } return non_const_columns; @@ -67,42 +81,138 @@ namespace return node; } - bool canRemoveDistinct(const QueryPlan::Node * distinct_node) + bool compareAggregationKeysWithDistinctColumns( + const Names & aggregation_keys, const DistinctColumns & distinct_columns, const ActionsDAGPtr & path_actions) + { + if (aggregation_keys.size() != distinct_columns.size()) + return false; + + /// compare columns of two DISTINCTs + for (const auto & column : distinct_columns) + { + const auto * alias_node = getOriginalNodeForOutputAlias(path_actions, String(column)); + if (!alias_node) + return false; + + if (std::find(cbegin(aggregation_keys), cend(aggregation_keys), alias_node->result_name) == aggregation_keys.cend()) + return false; + } + return true; + } + + bool checkStepToAllowOptimization(const IQueryPlanStep * step) + { + if (typeid_cast(step)) + return true; + + if (const auto * const expr = typeid_cast(step); expr) + return !expr->getExpression()->hasArrayJoin(); + + if (const auto * const filter = typeid_cast(step); filter) + return !filter->getExpression()->hasArrayJoin(); + + if (typeid_cast(step) || typeid_cast(step) || typeid_cast(step) + || typeid_cast(step)) + return true; + + return false; + } + + /// build actions DAG from stack of steps + ActionsDAGPtr buildActionsForPlanPath(std::vector & dag_stack) + { + if (dag_stack.empty()) + return nullptr; + + ActionsDAGPtr path_actions = dag_stack.back(); + dag_stack.pop_back(); + while (!dag_stack.empty()) + { + ActionsDAGPtr clone = dag_stack.back()->clone(); + dag_stack.pop_back(); + path_actions->mergeInplace(std::move(*clone)); + } + return path_actions; + } + + bool passTillAggregation(const QueryPlan::Node * distinct_node) { const DistinctStep * distinct_step = typeid_cast(distinct_node->step.get()); chassert(distinct_step); + std::vector dag_stack; + const DistinctStep * inner_distinct_step = nullptr; + const IQueryPlanStep * aggregation_before_distinct = nullptr; + const QueryPlan::Node * node = distinct_node; + while (!node->children.empty()) + { + const IQueryPlanStep * current_step = node->step.get(); + if (typeid_cast(current_step) || typeid_cast(current_step) + || typeid_cast(current_step) || typeid_cast(current_step)) + { + aggregation_before_distinct = current_step; + break; + } + if (!checkStepToAllowOptimization(current_step)) + { + logDebug("aggregation pass: stopped by allow check on step", current_step->getName()); + break; + } + + if (const auto * const expr = typeid_cast(current_step); expr) + dag_stack.push_back(expr->getExpression()); + else if (const auto * const filter = typeid_cast(current_step); filter) + dag_stack.push_back(filter->getExpression()); + + node = node->children.front(); + if (inner_distinct_step = typeid_cast(node->step.get()); inner_distinct_step) + break; + } + if (inner_distinct_step) + return false; + + if (aggregation_before_distinct) + { + ActionsDAGPtr actions = buildActionsForPlanPath(dag_stack); + logActionsDAG("aggregation pass: merged DAG:\n{}", actions); + + const auto distinct_columns = getDistinctColumns(distinct_step); + + if (const auto * aggregating_step = typeid_cast(aggregation_before_distinct); aggregating_step) + return compareAggregationKeysWithDistinctColumns(aggregating_step->getParams().keys, distinct_columns, actions); + + if (const auto * cube_step = typeid_cast(aggregation_before_distinct); cube_step) + return compareAggregationKeysWithDistinctColumns(cube_step->getParams().keys, distinct_columns, actions); + + if (const auto * rollup_step = typeid_cast(aggregation_before_distinct); rollup_step) + return compareAggregationKeysWithDistinctColumns(rollup_step->getParams().keys, distinct_columns, actions); + } + + return false; + } + + bool passTillDistinct(const QueryPlan::Node * distinct_node) + { + const DistinctStep * distinct_step = typeid_cast(distinct_node->step.get()); + chassert(distinct_step); + const auto distinct_columns = getDistinctColumns(distinct_step); + std::vector dag_stack; const DistinctStep * inner_distinct_step = nullptr; const QueryPlan::Node * node = distinct_node; while (!node->children.empty()) { const IQueryPlanStep * current_step = node->step.get(); - - /// don't try to remove DISTINCT after step with many inputs, like union/join/intersect/except - if (current_step->getInputStreams().size() > 1) - break; - - /// do not remove DISTINCT if there are steps which can generate new rows - if (typeid_cast(current_step) || typeid_cast(current_step)) + if (!checkStepToAllowOptimization(current_step)) + { + logDebug("distinct pass: stopped by allow check on step", current_step->getName()); break; + } if (const auto * const expr = typeid_cast(current_step); expr) - { - /// arrayJoin() can generate new rows - if (expr->getExpression()->hasArrayJoin()) - break; - dag_stack.push_back(expr->getExpression()); - } - if (const auto * const filter = typeid_cast(current_step); filter) - { - /// arrayJoin() can generate new rows - if (filter->getExpression()->hasArrayJoin()) - break; - + else if (const auto * const filter = typeid_cast(current_step); filter) dag_stack.push_back(filter->getExpression()); - } node = node->children.front(); inner_distinct_step = typeid_cast(node->step.get()); @@ -120,7 +230,6 @@ namespace if (inner_distinct_step->isPreliminary()) return false; - const auto distinct_columns = getDistinctColumns(distinct_step); auto inner_distinct_columns = getDistinctColumns(inner_distinct_step); if (distinct_columns.size() != inner_distinct_columns.size()) return false; @@ -129,16 +238,8 @@ namespace if (!dag_stack.empty()) { /// build actions DAG to find original column names - path_actions = dag_stack.back(); - dag_stack.pop_back(); - while (!dag_stack.empty()) - { - ActionsDAGPtr clone = dag_stack.back()->clone(); - dag_stack.pop_back(); - path_actions->mergeInplace(std::move(*clone)); - } - - logActionsDAG("merged DAG:\n{}", path_actions); + path_actions = buildActionsForPlanPath(dag_stack); + logActionsDAG("distinct pass: merged DAG:\n{}", path_actions); /// compare columns of two DISTINCTs for (const auto & column : distinct_columns) @@ -162,6 +263,17 @@ namespace return true; } + + bool canRemoveDistinct(const QueryPlan::Node * distinct_node) + { + if (passTillAggregation(distinct_node)) + return true; + + if (passTillDistinct(distinct_node)) + return true; + + return false; + } } /// @@ -188,5 +300,4 @@ size_t tryRemoveRedundantDistinct(QueryPlan::Node * parent_node, QueryPlan::Node return applied; } - } diff --git a/src/Processors/QueryPlan/RollupStep.h b/src/Processors/QueryPlan/RollupStep.h index 866de7178fa..0f9c1592969 100644 --- a/src/Processors/QueryPlan/RollupStep.h +++ b/src/Processors/QueryPlan/RollupStep.h @@ -19,6 +19,8 @@ public: void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) override; + const Aggregator::Params & getParams() const { return params; } + private: void updateOutputStream() override; diff --git a/tests/queries/0_stateless/02500_remove_redundant_distinct.reference b/tests/queries/0_stateless/02500_remove_redundant_distinct.reference index ffee4a54ee6..ccce269f694 100644 --- a/tests/queries/0_stateless/02500_remove_redundant_distinct.reference +++ b/tests/queries/0_stateless/02500_remove_redundant_distinct.reference @@ -208,3 +208,64 @@ Expression (Project names) ReadFromStorage (SystemOne) -- execute ['Istanbul','Berlin','Bensheim'] +-- GROUP BY before DISTINCT with on the same columns => remove DISTINCT +-- query +SELECT DISTINCT a +FROM +( + SELECT + a, + sum(b) AS c + FROM + ( + SELECT + x.number AS a, + y.number AS b + FROM numbers(3) AS x, numbers(3, 3) AS y + ) + GROUP BY a +) +-- explain +Expression ((Project names + (Projection + (Change column names to column identifiers + (Project names + Projection))))) + Aggregating + Expression ((Before GROUP BY + (Change column names to column identifiers + (Project names + (Projection + DROP unused columns after JOIN))))) + Join (JOIN FillRightFirst) + Expression (Change column names to column identifiers) + ReadFromStorage (SystemNumbers) + Expression (Change column names to column identifiers) + ReadFromStorage (SystemNumbers) +-- execute +0 +2 +1 +-- GROUP BY before DISTINCT with on different columns => do _not_ remove DISTINCT +-- query +SELECT DISTINCT c +FROM +( + SELECT + a, + sum(b) AS c + FROM + ( + SELECT + x.number AS a, + y.number AS b + FROM numbers(3) AS x, numbers(3, 3) AS y + ) + GROUP BY a +) +-- explain +Expression (Project names) + Distinct (DISTINCT) + Distinct (Preliminary DISTINCT) + Expression ((Projection + (Change column names to column identifiers + (Project names + Projection)))) + Aggregating + Expression ((Before GROUP BY + (Change column names to column identifiers + (Project names + (Projection + DROP unused columns after JOIN))))) + Join (JOIN FillRightFirst) + Expression (Change column names to column identifiers) + ReadFromStorage (SystemNumbers) + Expression (Change column names to column identifiers) + ReadFromStorage (SystemNumbers) +-- execute +12 diff --git a/tests/queries/0_stateless/02500_remove_redundant_distinct.sh b/tests/queries/0_stateless/02500_remove_redundant_distinct.sh index 4d9bbd301a1..9faf32abc16 100755 --- a/tests/queries/0_stateless/02500_remove_redundant_distinct.sh +++ b/tests/queries/0_stateless/02500_remove_redundant_distinct.sh @@ -112,3 +112,39 @@ FROM ) WHERE arrayJoin(cities) IN ['Berlin', 'Bensheim']" run_query "$query" + +echo "-- GROUP BY before DISTINCT with on the same columns => remove DISTINCT" +query="SELECT DISTINCT a +FROM +( + SELECT + a, + sum(b) AS c + FROM + ( + SELECT + x.number AS a, + y.number AS b + FROM numbers(3) AS x, numbers(3, 3) AS y + ) + GROUP BY a +)" +run_query "$query" + +echo "-- GROUP BY before DISTINCT with on different columns => do _not_ remove DISTINCT" +query="SELECT DISTINCT c +FROM +( + SELECT + a, + sum(b) AS c + FROM + ( + SELECT + x.number AS a, + y.number AS b + FROM numbers(3) AS x, numbers(3, 3) AS y + ) + GROUP BY a +)" +run_query "$query" From 01c42dd6f3a0870912a565e2197626e51238702c Mon Sep 17 00:00:00 2001 From: avogar Date: Thu, 9 Feb 2023 12:03:56 +0000 Subject: [PATCH 144/717] Fix --- src/Interpreters/inplaceBlockConversions.cpp | 6 +++++- .../00690_insert_select_converting_exception_message.sh | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/Interpreters/inplaceBlockConversions.cpp b/src/Interpreters/inplaceBlockConversions.cpp index ed606fff825..55d2f7ce5a8 100644 --- a/src/Interpreters/inplaceBlockConversions.cpp +++ b/src/Interpreters/inplaceBlockConversions.cpp @@ -101,7 +101,11 @@ void addDefaultRequiredExpressionsRecursively( auto default_value = column.type->getDefault(); ASTPtr expr = std::make_shared(default_value); if (is_column_in_query && convert_null_to_default) - expr = makeASTFunction("ifNull", std::make_shared(required_column_name), std::move(expr)); + { + /// We should CAST default value to required type, otherwise the result of ifNull function can be different type. + auto cast_expr = makeASTFunction("_CAST", std::move(expr), std::make_shared(columns.get(required_column_name).type->getName())); + expr = makeASTFunction("ifNull", std::make_shared(required_column_name), std::move(cast_expr)); + } default_expr_list_accum->children.emplace_back(setAlias(expr, required_column_name)); added_columns.emplace(required_column_name); } diff --git a/tests/queries/0_stateless/00690_insert_select_converting_exception_message.sh b/tests/queries/0_stateless/00690_insert_select_converting_exception_message.sh index 3f22726877a..254e8314474 100755 --- a/tests/queries/0_stateless/00690_insert_select_converting_exception_message.sh +++ b/tests/queries/0_stateless/00690_insert_select_converting_exception_message.sh @@ -10,7 +10,7 @@ ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS test_00690;" ${CLICKHOUSE_CLIENT} --query "CREATE TABLE test_00690 (val Int64) engine = Memory;" ${CLICKHOUSE_CLIENT} --query "INSERT INTO test_00690 SELECT 1;" -${CLICKHOUSE_CLIENT} --query "INSERT INTO test_00690 SELECT NULL AS src;" 2>&1 | grep -oF 'while converting source column src to destination column val'; -${CLICKHOUSE_CLIENT} --query "INSERT INTO test_00690 SELECT number % 2 ? 1 : NULL AS src FROM numbers(10);" 2>&1 | grep -oF 'while converting source column src to destination column val'; +${CLICKHOUSE_CLIENT} --query "INSERT INTO test_00690 SELECT NULL AS src SETTINGS insert_null_as_default=0;" 2>&1 | grep -oF 'while converting source column src to destination column val'; +${CLICKHOUSE_CLIENT} --query "INSERT INTO test_00690 SELECT number % 2 ? 1 : NULL AS src FROM numbers(10) SETTINGS insert_null_as_default=0;" 2>&1 | grep -oF 'while converting source column src to destination column val'; ${CLICKHOUSE_CLIENT} --query "DROP TABLE test_00690;" From fd52979991439905a871ff22cefb1fd2775f6ebd Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 9 Feb 2023 14:56:13 +0100 Subject: [PATCH 145/717] Fix snapshot for ordinary merge tree --- src/Storages/StorageMergeTree.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 5aa4a2bc98d..78b29c943dc 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -1262,7 +1262,10 @@ bool StorageMergeTree::scheduleDataProcessingJob(BackgroundJobsAssignee & assign } if (mutate_entry) { - auto task = std::make_shared(*this, metadata_snapshot, mutate_entry, shared_lock, common_assignee_trigger); + /// We take new metadata snapshot here. It's because mutation commands can be executed only with metadata snapshot + /// which is equal or more fresh than commands themselves. In extremely rare case it can happen that we will have alter + /// in between we took snapshot above and selected commands. That is why we take new snapshot here. + auto task = std::make_shared(*this, getInMemoryMetadataPtr(), mutate_entry, shared_lock, common_assignee_trigger); assignee.scheduleMergeMutateTask(task); return true; } From 09d7ca26e61d2b241430e9d3989b0d8081fbfd04 Mon Sep 17 00:00:00 2001 From: flynn Date: Fri, 10 Feb 2023 10:22:55 +0000 Subject: [PATCH 146/717] fix --- src/Storages/IStorageDataLake.h | 13 ++++++++----- src/Storages/StorageDeltaLake.cpp | 10 ---------- src/Storages/StorageHudi.cpp | 4 ++-- src/Storages/StorageIceberg.cpp | 17 +++++++++-------- src/Storages/StorageIceberg.h | 13 +------------ 5 files changed, 20 insertions(+), 37 deletions(-) diff --git a/src/Storages/IStorageDataLake.h b/src/Storages/IStorageDataLake.h index 8deba134382..1b471d48525 100644 --- a/src/Storages/IStorageDataLake.h +++ b/src/Storages/IStorageDataLake.h @@ -79,7 +79,7 @@ public: : IStorage(table_id_) , base_configuration{configuration_} , log(&Poco::Logger::get("Storage" + String(name) + "(" + table_id_.table_name + ")")) - , table_path(base_configuration.uri.key) + , table_path(base_configuration.url.key) { StorageInMemoryMetadata storage_metadata; StorageS3::updateS3Configuration(context_, base_configuration); @@ -141,12 +141,15 @@ public: static StorageS3::Configuration getAdjustedS3Configuration(const ContextPtr & context, StorageS3::Configuration & configuration, Poco::Logger * log) { - MetaParser parser{configuration, configuration.url.key, context}; + MetaParser parser{configuration, context}; auto keys = parser.getFiles(); - auto new_uri = std::filesystem::path(configuration.url.uri.toString()) / Name::data_directory_prefix + String new_uri = std::filesystem::path(configuration.url.uri.toString()) / Name::data_directory_prefix / MetaParser::generateQueryFromKeys(keys, configuration.format); + StorageS3::Configuration new_configuration(configuration); + new_configuration.url = S3::URI(new_uri); + LOG_DEBUG(log, "Table path: {}, new uri: {}", configuration.url.key, new_uri); return new_configuration; @@ -199,7 +202,7 @@ public: for (auto & engine_arg : engine_args) engine_arg = evaluateConstantExpressionOrIdentifierAsLiteral(engine_arg, local_context); - configuration.url = S3::{checkAndGetLiteralArgument(engine_args[0], "url")}; + configuration.url = S3::URI{checkAndGetLiteralArgument(engine_args[0], "url")}; configuration.auth_settings.access_key_id = checkAndGetLiteralArgument(engine_args[1], "access_key_id"); configuration.auth_settings.secret_access_key = checkAndGetLiteralArgument(engine_args[2], "secret_access_key"); @@ -215,7 +218,7 @@ public: private: - StorageS3::S3Configuration base_configuration; + StorageS3::Configuration base_configuration; std::shared_ptr s3engine; Poco::Logger * log; String table_path; diff --git a/src/Storages/StorageDeltaLake.cpp b/src/Storages/StorageDeltaLake.cpp index 045bbe58e52..8688bc60539 100644 --- a/src/Storages/StorageDeltaLake.cpp +++ b/src/Storages/StorageDeltaLake.cpp @@ -95,17 +95,7 @@ void DeltaLakeMetaParser::init(ContextPtr context) std::vector DeltaLakeMetaParser::getJsonLogFiles() const { - const auto & client = base_configuration.client; const auto table_path = base_configuration.url.key; - const auto bucket = base_configuration.url.bucket; - - std::vector keys; - S3::ListObjectsV2Request request; - Aws::S3::Model::ListObjectsV2Outcome outcome; - - bool is_finished{false}; - - request.SetBucket(bucket); /// DeltaLake format stores all metadata json files in _delta_log directory static constexpr auto deltalake_metadata_directory = "_delta_log"; diff --git a/src/Storages/StorageHudi.cpp b/src/Storages/StorageHudi.cpp index 85c6a7d516e..68a65d3b757 100644 --- a/src/Storages/StorageHudi.cpp +++ b/src/Storages/StorageHudi.cpp @@ -26,7 +26,7 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -HudiMetaParser::HudiMetaParser(const StorageS3::S3Configuration & configuration_, ContextPtr context_) +HudiMetaParser::HudiMetaParser(const StorageS3::Configuration & configuration_, ContextPtr context_) : configuration(configuration_), context(context_), log(&Poco::Logger::get("StorageHudi")) { } @@ -90,7 +90,7 @@ String HudiMetaParser::generateQueryFromKeys(const std::vector & ke std::vector HudiMetaParser::getFiles() const { const auto & client = configuration.client; - const auto & table_path = configuration.table_path; + const auto & table_path = configuration.url.key; const auto & bucket = configuration.url.bucket; std::vector keys; diff --git a/src/Storages/StorageIceberg.cpp b/src/Storages/StorageIceberg.cpp index 99ca7731850..93aa16cd29a 100644 --- a/src/Storages/StorageIceberg.cpp +++ b/src/Storages/StorageIceberg.cpp @@ -46,8 +46,8 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; } -IcebergMetaParser::IcebergMetaParser(const StorageS3::Configuration & configuration_, const String & table_path_, ContextPtr context_) - : base_configuration(configuration_), table_path(table_path_), context(context_) +IcebergMetaParser::IcebergMetaParser(const StorageS3::Configuration & configuration_, ContextPtr context_) + : base_configuration(configuration_), context(context_) { } @@ -71,9 +71,10 @@ String IcebergMetaParser::getNewestMetaFile() const /// Iceberg stores all the metadata.json in metadata directory, and the /// newest version has the max version name, so we should list all of them /// then find the newest metadata. + const auto & table_path = base_configuration.url.key; std::vector metadata_files = S3::listFiles( *base_configuration.client, - base_configuration.uri.bucket, + base_configuration.url.bucket, table_path, std::filesystem::path(table_path) / metadata_directory, ".json"); @@ -106,7 +107,7 @@ String IcebergMetaParser::getManiFestList(const String & metadata_name) const if (snapshot->getValue("snapshot-id") == current_snapshot_id) { auto path = snapshot->getValue("manifest-list"); - return std::filesystem::path(table_path) / metadata_directory / std::filesystem::path(path).filename(); + return std::filesystem::path(base_configuration.url.key) / metadata_directory / std::filesystem::path(path).filename(); } } @@ -156,7 +157,7 @@ std::vector IcebergMetaParser::getManifestFiles(const String & manifest_ auto file_path = col_str->getDataAt(i).toView(); /// We just need obtain the file name std::filesystem::path path(file_path); - res.emplace_back(std::filesystem::path(table_path) / metadata_directory / path.filename()); + res.emplace_back(std::filesystem::path(base_configuration.url.key) / metadata_directory / path.filename()); } return res; @@ -227,9 +228,9 @@ std::shared_ptr IcebergMetaParser::createS3ReadBuffer(const String & request_settings.max_single_read_retries = context->getSettingsRef().s3_max_single_read_retries; return std::make_shared( base_configuration.client, - base_configuration.uri.bucket, + base_configuration.url.bucket, key, - base_configuration.uri.version_id, + base_configuration.url.version_id, request_settings, context->getReadSettings()); } @@ -249,7 +250,7 @@ void registerStorageIceberg(StorageFactory & factory) [](const StorageFactory::Arguments & args) { auto & engine_args = args.engine_args; - StorageS3Configuration configuration = StorageIceberg::getConfiguration(engine_args, args.getLocalContext()); + StorageS3::Configuration configuration = StorageIceberg::getConfiguration(engine_args, args.getLocalContext()); auto format_settings = getFormatSettings(args.getContext()); diff --git a/src/Storages/StorageIceberg.h b/src/Storages/StorageIceberg.h index 622348d6caf..d351f073c8b 100644 --- a/src/Storages/StorageIceberg.h +++ b/src/Storages/StorageIceberg.h @@ -9,16 +9,6 @@ # include # include -namespace Poco -{ -class Logger; -} - -namespace Aws::S3 -{ -class S3Client; -} - namespace DB { @@ -31,7 +21,7 @@ namespace DB class IcebergMetaParser { public: - IcebergMetaParser(const StorageS3::Configuration & configuration_, const String & table_path_, ContextPtr context_); + IcebergMetaParser(const StorageS3::Configuration & configuration_, ContextPtr context_); std::vector getFiles() const; @@ -40,7 +30,6 @@ public: private: static constexpr auto metadata_directory = "metadata"; StorageS3::Configuration base_configuration; - String table_path; ContextPtr context; /// Just get file name From 9743a05cde7e54066e5a784406b246b23ab6668d Mon Sep 17 00:00:00 2001 From: flynn Date: Fri, 10 Feb 2023 10:30:43 +0000 Subject: [PATCH 147/717] fix style --- src/IO/S3Common.cpp | 3 --- src/Storages/StorageDeltaLake.cpp | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp index 5462df274cf..54aaddd9056 100644 --- a/src/IO/S3Common.cpp +++ b/src/IO/S3Common.cpp @@ -25,9 +25,6 @@ # include # include # include -# include -# include -# include # include # include diff --git a/src/Storages/StorageDeltaLake.cpp b/src/Storages/StorageDeltaLake.cpp index 8688bc60539..6bcdb6b58b8 100644 --- a/src/Storages/StorageDeltaLake.cpp +++ b/src/Storages/StorageDeltaLake.cpp @@ -56,7 +56,7 @@ std::vector DeltaLakeMetadata::listCurrentFiles() && return keys; } -DeltaLakeMetaParser::DeltaLakeMetaParser(const StorageS3::Configuration & configuration_, ContextPtr context) +DeltaLakeMetaParser::DeltaLakeMetaParser(const StorageS3::Configuration & configuration_, ContextPtr context) : base_configuration(configuration_) { init(context); From 7fee8995d373e47ef06257a40a1c72709cafd88a Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Fri, 10 Feb 2023 12:04:05 +0100 Subject: [PATCH 148/717] Addressed review comments and moved concurrency check to Backup/Restore Coordination - Use cluster state data to check concurrent backup/restore --- src/Backups/BackupCoordinationLocal.cpp | 5 + src/Backups/BackupCoordinationLocal.h | 2 + src/Backups/BackupCoordinationRemote.cpp | 55 ++++++++- src/Backups/BackupCoordinationRemote.h | 11 +- src/Backups/BackupsWorker.cpp | 144 ++-------------------- src/Backups/BackupsWorker.h | 4 - src/Backups/IBackupCoordination.h | 4 + src/Backups/IRestoreCoordination.h | 4 + src/Backups/RestoreCoordinationLocal.cpp | 5 + src/Backups/RestoreCoordinationLocal.h | 2 + src/Backups/RestoreCoordinationRemote.cpp | 56 ++++++++- src/Backups/RestoreCoordinationRemote.h | 6 +- 12 files changed, 146 insertions(+), 152 deletions(-) diff --git a/src/Backups/BackupCoordinationLocal.cpp b/src/Backups/BackupCoordinationLocal.cpp index e199e43fe01..dd5afbed8e8 100644 --- a/src/Backups/BackupCoordinationLocal.cpp +++ b/src/Backups/BackupCoordinationLocal.cpp @@ -202,4 +202,9 @@ Strings BackupCoordinationLocal::getAllArchiveSuffixes() const return archive_suffixes; } +bool BackupCoordinationLocal::hasConcurrentBackups(const String &, const String &, const std::atomic & num_active_backups) const +{ + return (num_active_backups > 1); +} + } diff --git a/src/Backups/BackupCoordinationLocal.h b/src/Backups/BackupCoordinationLocal.h index 43145a42bf6..6e8a793ccd4 100644 --- a/src/Backups/BackupCoordinationLocal.h +++ b/src/Backups/BackupCoordinationLocal.h @@ -52,6 +52,8 @@ public: String getNextArchiveSuffix() override; Strings getAllArchiveSuffixes() const override; + bool hasConcurrentBackups(const String & backup_id, const String & common_backups_path, const std::atomic & num_active_backups) const override; + private: mutable std::mutex mutex; BackupCoordinationReplicatedTables replicated_tables TSA_GUARDED_BY(mutex); diff --git a/src/Backups/BackupCoordinationRemote.cpp b/src/Backups/BackupCoordinationRemote.cpp index ad164768333..f613eb7d198 100644 --- a/src/Backups/BackupCoordinationRemote.cpp +++ b/src/Backups/BackupCoordinationRemote.cpp @@ -7,6 +7,7 @@ #include #include #include +#include namespace DB @@ -18,6 +19,8 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } +namespace Stage = BackupCoordinationStage; + /// zookeeper_path/file_names/file_name->checksum_and_size /// zookeeper_path/file_infos/checksum_and_size->info /// zookeeper_path/archive_suffixes @@ -163,10 +166,10 @@ namespace } BackupCoordinationRemote::BackupCoordinationRemote( - const String & zookeeper_path_, zkutil::GetZooKeeper get_zookeeper_, bool remove_zk_nodes_in_destructor_) + const String & zookeeper_path_, zkutil::GetZooKeeper get_zookeeper_, bool is_internal_) : zookeeper_path(zookeeper_path_) , get_zookeeper(get_zookeeper_) - , remove_zk_nodes_in_destructor(remove_zk_nodes_in_destructor_) + , is_internal(is_internal_) { createRootNodes(); stage_sync.emplace( @@ -177,7 +180,7 @@ BackupCoordinationRemote::~BackupCoordinationRemote() { try { - if (remove_zk_nodes_in_destructor) + if (!is_internal) removeAllNodes(); } catch (...) @@ -592,4 +595,50 @@ Strings BackupCoordinationRemote::getAllArchiveSuffixes() const return node_names; } +bool BackupCoordinationRemote::hasConcurrentBackups(const String & backup_id, const String & common_backup_path, const std::atomic &) const +{ + /// If its internal concurrency will be checked for the base backup + if (is_internal) + return false; + + auto zk = getZooKeeper(); + std::string backup_stage_path = common_backup_path + "/backup-" + toString(backup_id) +"/stage"; + + if (!zk->exists(common_backup_path)) + zk->createAncestors(common_backup_path); + + for (size_t attempt = 0; attempt < MAX_ZOOKEEPER_ATTEMPTS; ++attempt) + { + Coordination::Stat stat; + zk->get(common_backup_path, &stat); + Strings existing_backup_paths = zk->getChildren(common_backup_path); + + for (const auto & existing_backup_path : existing_backup_paths) + { + if (startsWith(existing_backup_path, "restore-")) + continue; + + String existing_backup_id = existing_backup_path; + existing_backup_id.erase(0, String("backup-").size()); + + if (existing_backup_id == toString(backup_id)) + continue; + + const auto status = zk->get(common_backup_path + "/" + existing_backup_path + "/stage"); + if (status != Stage::COMPLETED) + return true; + } + + zk->createIfNotExists(backup_stage_path, ""); + auto code = zk->trySet(backup_stage_path, Stage::SCHEDULED_TO_START, stat.version); + if (code == Coordination::Error::ZOK) + break; + bool is_last_attempt = (attempt == MAX_ZOOKEEPER_ATTEMPTS - 1); + if ((code != Coordination::Error::ZBADVERSION) || is_last_attempt) + throw zkutil::KeeperException(code, backup_stage_path); + } + return false; +} + + } diff --git a/src/Backups/BackupCoordinationRemote.h b/src/Backups/BackupCoordinationRemote.h index b1a7a12e109..50b98fcd77d 100644 --- a/src/Backups/BackupCoordinationRemote.h +++ b/src/Backups/BackupCoordinationRemote.h @@ -5,17 +5,18 @@ #include #include -/// We try to store data to zookeeper several times due to possible version conflicts. -constexpr size_t MAX_ZOOKEEPER_ATTEMPTS = 10; namespace DB { +/// We try to store data to zookeeper several times due to possible version conflicts. +constexpr size_t MAX_ZOOKEEPER_ATTEMPTS = 10; + /// Implementation of the IBackupCoordination interface performing coordination via ZooKeeper. It's necessary for "BACKUP ON CLUSTER". class BackupCoordinationRemote : public IBackupCoordination { public: - BackupCoordinationRemote(const String & zookeeper_path_, zkutil::GetZooKeeper get_zookeeper_, bool remove_zk_nodes_in_destructor_); + BackupCoordinationRemote(const String & zookeeper_path_, zkutil::GetZooKeeper get_zookeeper_, bool is_internal_); ~BackupCoordinationRemote() override; void setStage(const String & current_host, const String & new_stage, const String & message) override; @@ -57,6 +58,8 @@ public: String getNextArchiveSuffix() override; Strings getAllArchiveSuffixes() const override; + bool hasConcurrentBackups(const String & backup_id, const String & common_backups_path, const std::atomic & num_active_backups) const override; + private: zkutil::ZooKeeperPtr getZooKeeper() const; zkutil::ZooKeeperPtr getZooKeeperNoLock() const; @@ -67,7 +70,7 @@ private: const String zookeeper_path; const zkutil::GetZooKeeper get_zookeeper; - const bool remove_zk_nodes_in_destructor; + const bool is_internal; std::optional stage_sync; diff --git a/src/Backups/BackupsWorker.cpp b/src/Backups/BackupsWorker.cpp index 743f3329f5c..133e6989ff5 100644 --- a/src/Backups/BackupsWorker.cpp +++ b/src/Backups/BackupsWorker.cpp @@ -43,7 +43,7 @@ namespace if (!coordination_zk_path.empty()) { auto get_zookeeper = [global_context = context->getGlobalContext()] { return global_context->getZooKeeper(); }; - return std::make_shared(coordination_zk_path, get_zookeeper, !is_internal_backup); + return std::make_shared(coordination_zk_path, get_zookeeper, is_internal_backup); } else { @@ -56,7 +56,7 @@ namespace if (!coordination_zk_path.empty()) { auto get_zookeeper = [global_context = context->getGlobalContext()] { return global_context->getZooKeeper(); }; - return std::make_shared(coordination_zk_path, get_zookeeper, !is_internal_backup); + return std::make_shared(coordination_zk_path, get_zookeeper, is_internal_backup); } else { @@ -261,15 +261,17 @@ void BackupsWorker::doBackup( if (!on_cluster) context->checkAccess(required_access); + String root_zk_path; + ClusterPtr cluster; if (on_cluster) { + root_zk_path = context->getConfigRef().getString("backups.zookeeper_path", "/clickhouse/backups"); backup_query->cluster = context->getMacros()->expand(backup_query->cluster); cluster = context->getCluster(backup_query->cluster); backup_settings.cluster_host_ids = cluster->getHostIDs(); if (backup_settings.coordination_zk_path.empty()) { - String root_zk_path = context->getConfigRef().getString("backups.zookeeper_path", "/clickhouse/backups"); backup_settings.coordination_zk_path = root_zk_path + "/backup-" + toString(*backup_settings.backup_uuid); } } @@ -278,7 +280,7 @@ void BackupsWorker::doBackup( if (!backup_coordination) backup_coordination = makeBackupCoordination(backup_settings.coordination_zk_path, context, backup_settings.internal); - if (!allow_concurrent_backups && !backup_settings.internal && hasConcurrentBackups(backup_id, context, on_cluster)) + if (!allow_concurrent_backups && backup_coordination->hasConcurrentBackups(backup_id, root_zk_path, std::ref(num_active_backups))) throw Exception(ErrorCodes::CONCURRENT_ACCESS_NOT_SUPPORTED, "Concurrent backups not supported, turn on setting 'allow_concurrent_backups'"); /// Opens a backup for writing. @@ -487,13 +489,14 @@ void BackupsWorker::doRestore( BackupPtr backup = BackupFactory::instance().createBackup(backup_open_params); String current_database = context->getCurrentDatabase(); - + String root_zk_path; /// Checks access rights if this is ON CLUSTER query. /// (If this isn't ON CLUSTER query RestorerFromBackup will check access rights later.) ClusterPtr cluster; bool on_cluster = !restore_query->cluster.empty(); if (on_cluster) { + root_zk_path = context->getConfigRef().getString("backups.zookeeper_path", "/clickhouse/backups"); restore_query->cluster = context->getMacros()->expand(restore_query->cluster); cluster = context->getCluster(restore_query->cluster); restore_settings.cluster_host_ids = cluster->getHostIDs(); @@ -517,14 +520,13 @@ void BackupsWorker::doRestore( /// Make a restore coordination. if (on_cluster && restore_settings.coordination_zk_path.empty()) { - String root_zk_path = context->getConfigRef().getString("backups.zookeeper_path", "/clickhouse/backups"); restore_settings.coordination_zk_path = root_zk_path + "/restore-" + toString(restore_id); } if (!restore_coordination) restore_coordination = makeRestoreCoordination(restore_settings.coordination_zk_path, context, restore_settings.internal); - if (!allow_concurrent_restores && !restore_settings.internal && hasConcurrentRestores(restore_id, context, on_cluster)) + if (!allow_concurrent_restores && restore_coordination->hasConcurrentRestores(restore_id, root_zk_path, std::ref(num_active_restores))) throw Exception(ErrorCodes::CONCURRENT_ACCESS_NOT_SUPPORTED, "Concurrent restores not supported, turn on setting 'allow_concurrent_restores'"); /// Do RESTORE. @@ -709,134 +711,6 @@ std::vector BackupsWorker::getAllInfos() const return res_infos; } -std::vector BackupsWorker::getAllActiveBackupInfos() const -{ - std::vector res_infos; - std::lock_guard lock{infos_mutex}; - for (const auto & info : infos | boost::adaptors::map_values) - { - if (info.status==BackupStatus::CREATING_BACKUP) - res_infos.push_back(info); - } - return res_infos; -} - -std::vector BackupsWorker::getAllActiveRestoreInfos() const -{ - std::vector res_infos; - std::lock_guard lock{infos_mutex}; - for (const auto & info : infos | boost::adaptors::map_values) - { - if (info.status==BackupStatus::RESTORING) - res_infos.push_back(info); - } - return res_infos; -} - -bool BackupsWorker::hasConcurrentBackups(const OperationID & backup_id, const ContextPtr & context, bool on_cluster) const -{ - if (on_cluster) - { - String common_backup_path = context->getConfigRef().getString("backups.zookeeper_path", "/clickhouse/backups") ; - auto zookeeper = context->getGlobalContext()->getZooKeeper(); - std::string backup_stage_path = common_backup_path + "/backup-" + toString(backup_id) +"/stage"; - - if (!zookeeper->exists(common_backup_path)) - zookeeper->createAncestors(common_backup_path); - - for (size_t attempt = 0; attempt < MAX_ZOOKEEPER_ATTEMPTS; ++attempt) - { - Coordination::Stat stat; - zookeeper->get(common_backup_path, &stat); - Strings existing_backup_paths = zookeeper->getChildren(common_backup_path); - - for (const auto & existing_backup_path : existing_backup_paths) - { - if (startsWith(existing_backup_path, "restore-")) - continue; - - String existing_backup_id = existing_backup_path; - existing_backup_id.erase(0, String("backup-").size()); - - if (existing_backup_id == toString(backup_id)) - continue; - - const auto status = zookeeper->get(common_backup_path + "/" + existing_backup_path + "/stage"); - if (status != Stage::COMPLETED) - return true; - } - - zookeeper->createIfNotExists(backup_stage_path, ""); - auto code = zookeeper->trySet(backup_stage_path, Stage::SCHEDULED_TO_START, stat.version); - if (code == Coordination::Error::ZOK) - break; - bool is_last_attempt = (attempt == MAX_ZOOKEEPER_ATTEMPTS - 1); - if ((code != Coordination::Error::ZBADVERSION) || is_last_attempt) - throw zkutil::KeeperException(code, backup_stage_path); - } - return false; - } - else - { - if (num_active_backups == 1) - return false; - else - return true; - } -} - -bool BackupsWorker::hasConcurrentRestores(const OperationID & restore_id, const ContextPtr & context, bool on_cluster) const -{ - if (on_cluster) - { - String common_restore_path = context->getConfigRef().getString("backups.zookeeper_path", "/clickhouse/backups") ; - auto zookeeper = context->getGlobalContext()->getZooKeeper(); - std::string path = common_restore_path + "/restore-" + toString(restore_id) +"/stage"; - - if (!zookeeper->exists(common_restore_path)) - zookeeper->createAncestors(common_restore_path); - - for (size_t attempt = 0; attempt < MAX_ZOOKEEPER_ATTEMPTS; ++attempt) - { - Coordination::Stat stat; - zookeeper->get(common_restore_path, &stat); - Strings existing_restore_paths = zookeeper->getChildren(common_restore_path); - for (const auto & existing_restore_path : existing_restore_paths) - { - if (startsWith(existing_restore_path, "backup-")) - continue; - - String existing_restore_id = existing_restore_path; - existing_restore_id.erase(0, String("restore-").size()); - - if (existing_restore_id == toString(restore_id)) - continue; - - - const auto status = zookeeper->get(common_restore_path + "/" + existing_restore_path + "/stage"); - if (status != Stage::COMPLETED) - return true; - } - - zookeeper->createIfNotExists(path, ""); - auto code = zookeeper->trySet(path, Stage::SCHEDULED_TO_START, stat.version); - if (code == Coordination::Error::ZOK) - break; - bool is_last_attempt = (attempt == MAX_ZOOKEEPER_ATTEMPTS - 1); - if ((code != Coordination::Error::ZBADVERSION) || is_last_attempt) - throw zkutil::KeeperException(code, path); - } - return false; - } - else - { - if (num_active_restores == 1) - return false; - else - return true; - } -} - void BackupsWorker::shutdown() { bool has_active_backups_and_restores = (num_active_backups || num_active_restores); diff --git a/src/Backups/BackupsWorker.h b/src/Backups/BackupsWorker.h index 994e5f1f8b7..e169a9a2597 100644 --- a/src/Backups/BackupsWorker.h +++ b/src/Backups/BackupsWorker.h @@ -111,10 +111,6 @@ private: void setStatus(const OperationID & id, BackupStatus status, bool throw_if_error = true); void setStatusSafe(const String & id, BackupStatus status) { setStatus(id, status, false); } void setNumFilesAndSize(const OperationID & id, size_t num_files, size_t num_processed_files, UInt64 processed_files_size, UInt64 uncompressed_size, UInt64 compressed_size); - std::vector getAllActiveBackupInfos() const; - std::vector getAllActiveRestoreInfos() const; - bool hasConcurrentBackups(const OperationID & backup_id, const ContextPtr & context, bool on_cluster) const; - bool hasConcurrentRestores(const OperationID & restore_id, const ContextPtr & context, bool on_cluster) const; ThreadPool backups_thread_pool; ThreadPool restores_thread_pool; diff --git a/src/Backups/IBackupCoordination.h b/src/Backups/IBackupCoordination.h index 7ff911488aa..787b86f8cbd 100644 --- a/src/Backups/IBackupCoordination.h +++ b/src/Backups/IBackupCoordination.h @@ -114,6 +114,10 @@ public: /// Returns the list of all the archive suffixes which were generated. virtual Strings getAllArchiveSuffixes() const = 0; + + /// This function is used to check if concurrent backups are running + /// other than the backup passed to the function + virtual bool hasConcurrentBackups(const String & backup_id, const String & common_backups_path, const std::atomic & num_active_backups) const = 0; }; } diff --git a/src/Backups/IRestoreCoordination.h b/src/Backups/IRestoreCoordination.h index 692054ae267..a2506e5c157 100644 --- a/src/Backups/IRestoreCoordination.h +++ b/src/Backups/IRestoreCoordination.h @@ -34,6 +34,10 @@ public: /// Sets that this replica is going to restore a ReplicatedAccessStorage. /// The function returns false if this access storage is being already restored by another replica. virtual bool acquireReplicatedAccessStorage(const String & access_storage_zk_path) = 0; + + /// This function is used to check if concurrent restores are running + /// other than the restore passed to the function + virtual bool hasConcurrentRestores(const String & restore_id, const String & common_restores_path, const std::atomic & num_active_restores) const = 0; }; } diff --git a/src/Backups/RestoreCoordinationLocal.cpp b/src/Backups/RestoreCoordinationLocal.cpp index b2a9849c38d..56951c56baa 100644 --- a/src/Backups/RestoreCoordinationLocal.cpp +++ b/src/Backups/RestoreCoordinationLocal.cpp @@ -42,4 +42,9 @@ bool RestoreCoordinationLocal::acquireReplicatedAccessStorage(const String &) return true; } +bool RestoreCoordinationLocal::hasConcurrentRestores(const String &, const String &, const std::atomic & num_active_restores) const +{ + return (num_active_restores > 1); +} + } diff --git a/src/Backups/RestoreCoordinationLocal.h b/src/Backups/RestoreCoordinationLocal.h index b4e70d83b72..fa6941a7577 100644 --- a/src/Backups/RestoreCoordinationLocal.h +++ b/src/Backups/RestoreCoordinationLocal.h @@ -35,6 +35,8 @@ public: /// The function returns false if this access storage is being already restored by another replica. bool acquireReplicatedAccessStorage(const String & access_storage_zk_path) override; + bool hasConcurrentRestores(const String & restore_id, const String & common_restores_path, const std::atomic & num_active_restores) const override; + private: std::set> acquired_tables_in_replicated_databases; std::unordered_set acquired_data_in_replicated_tables; diff --git a/src/Backups/RestoreCoordinationRemote.cpp b/src/Backups/RestoreCoordinationRemote.cpp index 89a9950aad2..95766bfcae3 100644 --- a/src/Backups/RestoreCoordinationRemote.cpp +++ b/src/Backups/RestoreCoordinationRemote.cpp @@ -1,16 +1,19 @@ #include #include #include - +#include +#include namespace DB { +namespace Stage = BackupCoordinationStage; + RestoreCoordinationRemote::RestoreCoordinationRemote( - const String & zookeeper_path_, zkutil::GetZooKeeper get_zookeeper_, bool remove_zk_nodes_in_destructor_) + const String & zookeeper_path_, zkutil::GetZooKeeper get_zookeeper_, bool is_internal_) : zookeeper_path(zookeeper_path_) , get_zookeeper(get_zookeeper_) - , remove_zk_nodes_in_destructor(remove_zk_nodes_in_destructor_) + , is_internal(is_internal_) { createRootNodes(); @@ -22,7 +25,7 @@ RestoreCoordinationRemote::~RestoreCoordinationRemote() { try { - if (remove_zk_nodes_in_destructor) + if (!is_internal) removeAllNodes(); } catch (...) @@ -129,4 +132,49 @@ void RestoreCoordinationRemote::removeAllNodes() zk->removeRecursive(zookeeper_path); } +bool RestoreCoordinationRemote::hasConcurrentRestores(const String & restore_id, const String & common_restores_path, const std::atomic &) const +{ + /// If its internal concurrency will be checked for the base restore + if (is_internal) + return false; + + auto zk = getZooKeeper(); + std::string path = common_restores_path + "/restore-" + toString(restore_id) +"/stage"; + + if (! zk->exists(common_restores_path)) + zk->createAncestors(common_restores_path); + + for (size_t attempt = 0; attempt < MAX_ZOOKEEPER_ATTEMPTS; ++attempt) + { + Coordination::Stat stat; + zk->get(common_restores_path, &stat); + Strings existing_restore_paths = zk->getChildren(common_restores_path); + for (const auto & existing_restore_path : existing_restore_paths) + { + if (startsWith(existing_restore_path, "backup-")) + continue; + + String existing_restore_id = existing_restore_path; + existing_restore_id.erase(0, String("restore-").size()); + + if (existing_restore_id == toString(restore_id)) + continue; + + + const auto status = zk->get(common_restores_path + "/" + existing_restore_path + "/stage"); + if (status != Stage::COMPLETED) + return true; + } + + zk->createIfNotExists(path, ""); + auto code = zk->trySet(path, Stage::SCHEDULED_TO_START, stat.version); + if (code == Coordination::Error::ZOK) + break; + bool is_last_attempt = (attempt == MAX_ZOOKEEPER_ATTEMPTS - 1); + if ((code != Coordination::Error::ZBADVERSION) || is_last_attempt) + throw zkutil::KeeperException(code, path); + } + return false; +} + } diff --git a/src/Backups/RestoreCoordinationRemote.h b/src/Backups/RestoreCoordinationRemote.h index 83760a2d883..67d78192e1e 100644 --- a/src/Backups/RestoreCoordinationRemote.h +++ b/src/Backups/RestoreCoordinationRemote.h @@ -11,7 +11,7 @@ namespace DB class RestoreCoordinationRemote : public IRestoreCoordination { public: - RestoreCoordinationRemote(const String & zookeeper_path_, zkutil::GetZooKeeper get_zookeeper_, bool remove_zk_nodes_in_destructor_); + RestoreCoordinationRemote(const String & zookeeper_path_, zkutil::GetZooKeeper get_zookeeper_, bool is_internal_); ~RestoreCoordinationRemote() override; /// Sets the current stage and waits for other hosts to come to this stage too. @@ -31,6 +31,8 @@ public: /// The function returns false if this access storage is being already restored by another replica. bool acquireReplicatedAccessStorage(const String & access_storage_zk_path) override; + bool hasConcurrentRestores(const String & restore_id, const String & common_restores_path, const std::atomic & num_active_restores) const override; + private: zkutil::ZooKeeperPtr getZooKeeper() const; void createRootNodes(); @@ -40,7 +42,7 @@ private: const String zookeeper_path; const zkutil::GetZooKeeper get_zookeeper; - const bool remove_zk_nodes_in_destructor; + const bool is_internal; std::optional stage_sync; From 94fba0b6649b7425ef6b59eeaad4274999e04bd2 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Fri, 10 Feb 2023 13:53:21 +0100 Subject: [PATCH 149/717] Fixed build issue caused after merge master in BackupsWorker.h - Use cluster state data to check concurrent backup/restore --- src/Backups/BackupsWorker.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Backups/BackupsWorker.h b/src/Backups/BackupsWorker.h index a704db720b5..d0f3ad99b5f 100644 --- a/src/Backups/BackupsWorker.h +++ b/src/Backups/BackupsWorker.h @@ -114,8 +114,9 @@ private: void addInfo(const OperationID & id, const String & name, bool internal, BackupStatus status); void setStatus(const OperationID & id, BackupStatus status, bool throw_if_error = true); void setStatusSafe(const String & id, BackupStatus status) { setStatus(id, status, false); } - void setNumFilesAndSize(const OperationID & id, size_t num_files, size_t num_processed_files, UInt64 processed_files_size, UInt64 uncompressed_size, UInt64 compressed_size); - + void setNumFilesAndSize(const OperationID & id, size_t num_files, UInt64 total_size, size_t num_entries, + UInt64 uncompressed_size, UInt64 compressed_size, size_t num_read_files, UInt64 num_read_bytes); + ThreadPool backups_thread_pool; ThreadPool restores_thread_pool; From 808a939ad2ca2730e186eb48dbd6d130663de3ec Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 10 Feb 2023 14:25:19 +0100 Subject: [PATCH 150/717] Small rehacktoing --- src/Storages/MergeTree/MergeTreeData.cpp | 10 -- src/Storages/MergeTree/MutateTask.cpp | 115 ++++++++---------- .../ReplicatedMergeTreeAttachThread.cpp | 5 +- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 20 +-- src/Storages/MutationCommands.cpp | 16 +++ src/Storages/MutationCommands.h | 5 + src/Storages/StorageInMemoryMetadata.cpp | 7 ++ src/Storages/StorageInMemoryMetadata.h | 9 +- src/Storages/StorageMergeTree.cpp | 15 +-- src/Storages/StorageReplicatedMergeTree.cpp | 5 +- .../02555_davengers_rename_chain.sh | 16 +-- 11 files changed, 106 insertions(+), 117 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index f8f8ab74b31..4461a1470b4 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -7574,7 +7574,6 @@ AlterConversions MergeTreeData::getAlterConversionsForPart(const MergeTreeDataPa AlterConversions result{}; auto & rename_map = result.rename_map; - /// Squash "log of renames" into single map for (const auto & [version, commands] : commands_map) { for (const auto & command : commands) @@ -7589,15 +7588,6 @@ AlterConversions MergeTreeData::getAlterConversionsForPart(const MergeTreeDataPa } } - for (auto it = rename_map.begin(); it != rename_map.end();) - { - if (it->rename_to == it->rename_from) - it = rename_map.erase(it); - else - ++it; - - } - return result; } diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index 340709bff2c..01f7b47a170 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -52,7 +52,7 @@ static bool checkOperationIsNotCanceled(ActionBlocker & merges_blocker, MergeLis * First part should be executed by mutations interpreter. * Other is just simple drop/renames, so they can be executed without interpreter. */ -static void splitMutationCommands( +static void splitAndModifyMutationCommands( MergeTreeData::DataPartPtr part, const MutationCommands & commands, MutationCommands & for_interpreter, @@ -101,29 +101,37 @@ static void splitMutationCommands( } auto alter_conversions = part->storage.getAlterConversionsForPart(part); - /// If it's compact part, then we don't need to actually remove files - /// from disk we just don't read dropped columns + /// We don't add renames from commands, instead we take them from rename_map. + /// It's important because required renames depend not only on part's data version (i.e. mutation version) + /// but also on part's metadata version. Why we have such logic only for renames? Because all other types of alter + /// can be deduced based on difference between part's schema and table schema. for (const auto & [rename_to, rename_from] : alter_conversions.rename_map) { if (part_columns.has(rename_from)) { + /// Actual rename for_interpreter.push_back( { .type = MutationCommand::Type::READ_COLUMN, .column_name = rename_to, }); + /// Not needed for compact parts (not executed), added here only to produce correct + /// set of columns for new part and their serializations for_file_renames.push_back( { .type = MutationCommand::Type::RENAME_COLUMN, .column_name = rename_from, .rename_to = rename_to }); + part_columns.rename(rename_from, rename_to); } } + /// If it's compact part, then we don't need to actually remove files + /// from disk we just don't read dropped columns for (const auto & column : part_columns) { if (!mutated_columns.contains(column.name)) @@ -162,6 +170,11 @@ static void splitMutationCommands( } auto alter_conversions = part->storage.getAlterConversionsForPart(part); + /// We don't add renames from commands, instead we take them from rename_map. + /// It's important because required renames depend not only on part's data version (i.e. mutation version) + /// but also on part's metadata version. Why we have such logic only for renames? Because all other types of alter + /// can be deduced based on difference between part's schema and table schema. + for (const auto & [rename_to, rename_from] : alter_conversions.rename_map) { for_file_renames.push_back({.type = MutationCommand::Type::RENAME_COLUMN, .column_name = rename_from, .rename_to = rename_to}); @@ -169,6 +182,41 @@ static void splitMutationCommands( } } +/// It's legal to squash renames because commands with rename are always "barrier" +/// and executed separately from other types of commands. +static MutationCommands squashRenamesInCommands(const MutationCommands & commands) +{ + NameToNameMap squashed_renames; + for (const auto & command : commands) + { + std::string result_name = command.rename_to; + + bool squashed = false; + for (const auto & [name_from, name_to] : squashed_renames) + { + if (name_to == command.column_name) + { + squashed = true; + squashed_renames[name_from] = result_name; + break; + } + } + if (!squashed) + squashed_renames[command.column_name] = result_name; + } + + MutationCommands squashed_commands; + for (const auto & command : commands) + { + if (squashed_renames.contains(command.column_name)) + { + squashed_commands.push_back(command); + squashed_commands.back().rename_to = squashed_renames[command.column_name]; + } + } + return squashed_commands; +} + /// Get the columns list of the resulting part in the same order as storage_columns. static std::pair getColumnsForNewDataPart( @@ -198,35 +246,7 @@ getColumnsForNewDataPart( storage_columns.emplace_back(column); } - NameToNameMap squashed_renames; - for (const auto & command : all_commands) - { - std::string result_name = command.rename_to; - - bool squashed = false; - for (const auto & [name_from, name_to] : squashed_renames) - { - if (name_to == command.column_name) - { - squashed = true; - squashed_renames[name_from] = result_name; - break; - } - } - if (!squashed) - squashed_renames[command.column_name] = result_name; - } - - MutationCommands squashed_commands; - for (const auto & command : all_commands) - { - if (squashed_renames.contains(command.column_name)) - { - squashed_commands.push_back(command); - squashed_commands.back().rename_to = squashed_renames[command.column_name]; - } - } - + MutationCommands squashed_commands = squashRenamesInCommands(all_commands); for (const auto & command : squashed_commands) { @@ -618,34 +638,7 @@ static NameToNameVector collectFilesForRenames( rename_vector.emplace_back(file_rename_from, file_rename_to); }; - NameToNameMap squashed_renames; - for (const auto & command : commands_for_removes) - { - std::string result_name = command.rename_to; - - bool squashed = false; - for (const auto & [name_from, name_to] : squashed_renames) - { - if (name_to == command.column_name) - { - squashed = true; - squashed_renames[name_from] = result_name; - break; - } - } - if (!squashed) - squashed_renames[command.column_name] = result_name; - } - - MutationCommands squashed_commands; - for (const auto & command : commands_for_removes) - { - if (squashed_renames.contains(command.column_name)) - { - squashed_commands.push_back(command); - squashed_commands.back().rename_to = squashed_renames[command.column_name]; - } - } + MutationCommands squashed_commands = squashRenamesInCommands(commands_for_removes); /// Remove old data for (const auto & command : squashed_commands) @@ -1724,7 +1717,7 @@ bool MutateTask::prepare() context_for_reading->setSetting("allow_asynchronous_read_from_io_pool_for_merge_tree", false); context_for_reading->setSetting("max_streams_for_merge_tree_reading", Field(0)); - MutationHelpers::splitMutationCommands(ctx->source_part, ctx->commands_for_part, ctx->for_interpreter, ctx->for_file_renames); + MutationHelpers::splitAndModifyMutationCommands(ctx->source_part, ctx->commands_for_part, ctx->for_interpreter, ctx->for_file_renames); ctx->stage_progress = std::make_unique(1.0); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp index 941d0850f67..f1bdc9f43af 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp @@ -149,10 +149,7 @@ void ReplicatedMergeTreeAttachThread::runImpl() const bool replica_metadata_version_exists = zookeeper->tryGet(replica_path + "/metadata_version", replica_metadata_version); if (replica_metadata_version_exists) { - StorageInMemoryMetadata metadata_with_new_version(*metadata_snapshot); - - metadata_with_new_version.setMetadataVersion(parse(replica_metadata_version)); - storage.setInMemoryMetadata(metadata_with_new_version); + storage.setInMemoryMetadata(metadata_snapshot->withMetadataVersion(parse(replica_metadata_version))); } else { diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index f491cdc8cd1..e3e558db77b 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -1831,28 +1831,16 @@ MutationCommands ReplicatedMergeTreeQueue::getMutationCommands( MutationCommands commands; for (auto it = begin; it != end; ++it) { - bool rename_command = false; - if (it->second->entry->isAlterMutation()) - { - const auto & single_mutation_commands = it->second->entry->commands; - for (const auto & command : single_mutation_commands) - { - if (command.type == MutationCommand::Type::RENAME_COLUMN) - { - rename_command = true; - break; - } - } - } + const auto & commands_from_entry = it->second->entry->commands; - if (rename_command) + if (commands_from_entry.containBarrierCommand()) { if (commands.empty()) - commands.insert(commands.end(), it->second->entry->commands.begin(), it->second->entry->commands.end()); + commands.insert(commands.end(), commands_from_entry.begin(), commands_from_entry.end()); break; } else - commands.insert(commands.end(), it->second->entry->commands.begin(), it->second->entry->commands.end()); + commands.insert(commands.end(), commands_from_entry.begin(), commands_from_entry.end()); } return commands; diff --git a/src/Storages/MutationCommands.cpp b/src/Storages/MutationCommands.cpp index 0c9e9223929..aa77988348d 100644 --- a/src/Storages/MutationCommands.cpp +++ b/src/Storages/MutationCommands.cpp @@ -23,6 +23,12 @@ namespace ErrorCodes extern const int MULTIPLE_ASSIGNMENTS_TO_COLUMN; } + +bool MutationCommand::isBarrierCommand() const +{ + return type == RENAME_COLUMN; +} + std::optional MutationCommand::parse(ASTAlterCommand * command, bool parse_alter_commands) { if (command->type == ASTAlterCommand::DELETE) @@ -212,4 +218,14 @@ bool MutationCommands::hasNonEmptyMutationCommands() const return false; } +bool MutationCommands::containBarrierCommand() const +{ + for (const auto & command : *this) + { + if (command.isBarrierCommand()) + return true; + } + return false; +} + } diff --git a/src/Storages/MutationCommands.h b/src/Storages/MutationCommands.h index aca91c16e85..079b456fa3b 100644 --- a/src/Storages/MutationCommands.h +++ b/src/Storages/MutationCommands.h @@ -67,6 +67,9 @@ struct MutationCommand /// If parse_alter_commands, than consider more Alter commands as mutation commands static std::optional parse(ASTAlterCommand * command, bool parse_alter_commands = false); + + /// This command shouldn't stick with other commands + bool isBarrierCommand() const; }; /// Multiple mutation commands, possible from different ALTER queries @@ -79,6 +82,8 @@ public: void readText(ReadBuffer & in); std::string toString() const; bool hasNonEmptyMutationCommands() const; + + bool containBarrierCommand() const; }; using MutationCommandsConstPtr = std::shared_ptr; diff --git a/src/Storages/StorageInMemoryMetadata.cpp b/src/Storages/StorageInMemoryMetadata.cpp index 5250c6f5330..45abd4bebef 100644 --- a/src/Storages/StorageInMemoryMetadata.cpp +++ b/src/Storages/StorageInMemoryMetadata.cpp @@ -129,6 +129,13 @@ void StorageInMemoryMetadata::setMetadataVersion(int32_t metadata_version_) metadata_version = metadata_version_; } +StorageInMemoryMetadata StorageInMemoryMetadata::withMetadataVersion(int32_t metadata_version_) const +{ + StorageInMemoryMetadata copy(*this); + copy.setMetadataVersion(metadata_version_); + return copy; +} + const ColumnsDescription & StorageInMemoryMetadata::getColumns() const { return columns; diff --git a/src/Storages/StorageInMemoryMetadata.h b/src/Storages/StorageInMemoryMetadata.h index dfb5e9ddb54..25618c5b03f 100644 --- a/src/Storages/StorageInMemoryMetadata.h +++ b/src/Storages/StorageInMemoryMetadata.h @@ -50,6 +50,8 @@ struct StorageInMemoryMetadata String comment; + /// Version of metadata. Managed properly by ReplicatedMergeTree only + /// (zero-initialization is important) int32_t metadata_version = 0; StorageInMemoryMetadata() = default; @@ -60,7 +62,7 @@ struct StorageInMemoryMetadata StorageInMemoryMetadata(StorageInMemoryMetadata && other) = default; StorageInMemoryMetadata & operator=(StorageInMemoryMetadata && other) = default; - /// NOTE: Thread unsafe part. You should modify same StorageInMemoryMetadata + /// NOTE: Thread unsafe part. You should not modify same StorageInMemoryMetadata /// structure from different threads. It should be used as MultiVersion /// object. See example in IStorage. @@ -92,7 +94,10 @@ struct StorageInMemoryMetadata /// Set SELECT query for (Materialized)View void setSelectQuery(const SelectQueryDescription & select_); + /// Set version of metadata. void setMetadataVersion(int32_t metadata_version_); + /// Get copy of current metadata with metadata_version_ + StorageInMemoryMetadata withMetadataVersion(int32_t metadata_version_) const; /// Returns combined set of columns const ColumnsDescription & getColumns() const; @@ -222,8 +227,8 @@ struct StorageInMemoryMetadata const SelectQueryDescription & getSelectQuery() const; bool hasSelectQuery() const; + /// Get version of metadata int32_t getMetadataVersion() const { return metadata_version; } - bool hasMetadataVersion() const { return metadata_version != -1; } /// Check that all the requested names are in the table and have the correct types. void check(const NamesAndTypesList & columns) const; diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 78b29c943dc..3ee56a2e62d 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -1141,22 +1141,13 @@ MergeMutateSelectedEntryPtr StorageMergeTree::selectPartsToMutate( if (current_ast_elements + commands_size >= max_ast_elements) break; - bool rename_command = false; const auto & single_mutation_commands = it->second.commands; - for (const auto & command : single_mutation_commands) - { - if (command.type == MutationCommand::Type::RENAME_COLUMN) - { - rename_command = true; - break; - } - } - if (rename_command) + if (single_mutation_commands.containBarrierCommand()) { if (commands->empty()) { - commands->insert(commands->end(), it->second.commands.begin(), it->second.commands.end()); + commands->insert(commands->end(), single_mutation_commands.begin(), single_mutation_commands.end()); last_mutation_to_apply = it; } break; @@ -1164,7 +1155,7 @@ MergeMutateSelectedEntryPtr StorageMergeTree::selectPartsToMutate( else { current_ast_elements += commands_size; - commands->insert(commands->end(), it->second.commands.begin(), it->second.commands.end()); + commands->insert(commands->end(), single_mutation_commands.begin(), single_mutation_commands.end()); last_mutation_to_apply = it; } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 8e6e809a54a..806e535b711 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -455,10 +455,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( Coordination::Stat metadata_stat; current_zookeeper->get(zookeeper_path + "/metadata", &metadata_stat); - - StorageInMemoryMetadata storage_metadata(*metadata_snapshot); - storage_metadata.setMetadataVersion(metadata_stat.version); - setInMemoryMetadata(storage_metadata); + setInMemoryMetadata(metadata_snapshot->withMetadataVersion(metadata_stat.version)); } catch (Coordination::Exception & e) { diff --git a/tests/queries/0_stateless/02555_davengers_rename_chain.sh b/tests/queries/0_stateless/02555_davengers_rename_chain.sh index 71201537170..f4af5e091ad 100755 --- a/tests/queries/0_stateless/02555_davengers_rename_chain.sh +++ b/tests/queries/0_stateless/02555_davengers_rename_chain.sh @@ -27,8 +27,8 @@ counter=0 retries=60 I=0 while [[ $counter -lt $retries ]]; do I=$((I + 1)) - result=$($CLICKHOUSE_CLIENT --query "show create table wrong_metadata") - if [[ $result == *"a1 UInt64"* ]]; then + result=$($CLICKHOUSE_CLIENT --query "SELECT * FROM system.mutations WHERE table = 'wrong_metadata' AND database='${CLICKHOUSE_DATABASE}'") + if [[ $result == *"a TO a1"* ]]; then break; fi sleep 0.1 @@ -53,8 +53,8 @@ counter=0 retries=60 I=0 while [[ $counter -lt $retries ]]; do I=$((I + 1)) - result=$($CLICKHOUSE_CLIENT --query "show create table wrong_metadata") - if [[ $result == *"b UInt64"* ]]; then + result=$($CLICKHOUSE_CLIENT --query "SELECT * FROM system.mutations WHERE table = 'wrong_metadata' AND database='${CLICKHOUSE_DATABASE}'") + if [[ $result == *"b1 TO a"* ]]; then break; fi sleep 0.1 @@ -98,8 +98,8 @@ counter=0 retries=60 I=0 while [[ $counter -lt $retries ]]; do I=$((I + 1)) - result=$($CLICKHOUSE_CLIENT --query "show create table wrong_metadata_compact") - if [[ $result == *"b1 UInt64"* ]]; then + result=$($CLICKHOUSE_CLIENT --query "SELECT * FROM system.mutations WHERE table = 'wrong_metadata_compact' AND database='${CLICKHOUSE_DATABASE}'") + if [[ $result == *"a TO a1"* ]]; then break; fi sleep 0.1 @@ -120,8 +120,8 @@ counter=0 retries=60 I=0 while [[ $counter -lt $retries ]]; do I=$((I + 1)) - result=$($CLICKHOUSE_CLIENT --query "show create table wrong_metadata_compact") - if [[ $result == *"b UInt64"* ]]; then + result=$($CLICKHOUSE_CLIENT --query "SELECT * FROM system.mutations WHERE table = 'wrong_metadata_compact' AND database='${CLICKHOUSE_DATABASE}'") + if [[ $result == *"b1 TO a"* ]]; then break; fi sleep 0.1 From 99dc7fbde8ec125b85176c481396a2ca916854d2 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 10 Feb 2023 14:27:59 +0100 Subject: [PATCH 151/717] Update src/Storages/MutationCommands.h --- src/Storages/MutationCommands.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Storages/MutationCommands.h b/src/Storages/MutationCommands.h index 079b456fa3b..5ef0cfda1be 100644 --- a/src/Storages/MutationCommands.h +++ b/src/Storages/MutationCommands.h @@ -83,6 +83,9 @@ public: std::string toString() const; bool hasNonEmptyMutationCommands() const; + /// These set of commands contain barrier command and shouldn't + /// stick with other commands. Commands from one set have already been validated + /// to be executed without issues on the creation state. bool containBarrierCommand() const; }; From 2ce67830c8c4b822ddf98fc7c4ca77966ec88b61 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Fri, 10 Feb 2023 14:41:43 +0100 Subject: [PATCH 152/717] Fixed style check by removing trailing whitespaces in BackupsWorker.h - Use cluster state data to check concurrent backup/restore --- src/Backups/BackupsWorker.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Backups/BackupsWorker.h b/src/Backups/BackupsWorker.h index d0f3ad99b5f..0f5c16cd71f 100644 --- a/src/Backups/BackupsWorker.h +++ b/src/Backups/BackupsWorker.h @@ -116,7 +116,7 @@ private: void setStatusSafe(const String & id, BackupStatus status) { setStatus(id, status, false); } void setNumFilesAndSize(const OperationID & id, size_t num_files, UInt64 total_size, size_t num_entries, UInt64 uncompressed_size, UInt64 compressed_size, size_t num_read_files, UInt64 num_read_bytes); - + ThreadPool backups_thread_pool; ThreadPool restores_thread_pool; From d1efd024809cea35b2bb099e32208ad88603c37a Mon Sep 17 00:00:00 2001 From: avogar Date: Fri, 10 Feb 2023 16:40:14 +0000 Subject: [PATCH 153/717] Extend setting input_format_null_as_default for more formats --- src/Columns/ColumnLowCardinality.cpp | 28 ++++++++ src/Columns/ColumnLowCardinality.h | 2 + src/Columns/ColumnNullable.cpp | 22 ++++++ src/Columns/ColumnNullable.h | 2 + src/Core/Settings.h | 3 +- src/DataTypes/IDataType.h | 5 ++ src/Formats/FormatFactory.cpp | 1 - src/Formats/FormatSettings.h | 1 - src/Formats/NativeReader.cpp | 31 +++++++-- src/Formats/NativeReader.h | 10 ++- src/Formats/insertNullAsDefaultIfNeeded.cpp | 37 ++++++++++ src/Formats/insertNullAsDefaultIfNeeded.h | 10 +++ .../Formats/Impl/ArrowBlockInputFormat.cpp | 9 +-- .../Formats/Impl/ArrowBlockInputFormat.h | 1 - .../Formats/Impl/ArrowColumnToCHColumn.cpp | 39 ++++------- .../Formats/Impl/ArrowColumnToCHColumn.h | 9 ++- .../Formats/Impl/AvroRowInputFormat.cpp | 69 +++++++++++++++---- .../Formats/Impl/AvroRowInputFormat.h | 7 +- .../Formats/Impl/MsgPackRowInputFormat.cpp | 44 +++++++----- .../Formats/Impl/MsgPackRowInputFormat.h | 10 ++- src/Processors/Formats/Impl/NativeFormat.cpp | 12 +++- .../Formats/Impl/ORCBlockInputFormat.cpp | 8 +-- .../Formats/Impl/ORCBlockInputFormat.h | 1 - .../Formats/Impl/ParquetBlockInputFormat.cpp | 13 ++-- .../Formats/Impl/ParquetBlockInputFormat.h | 1 - ...561_null_as_default_more_formats.reference | 36 ++++++++++ .../02561_null_as_default_more_formats.sh | 21 ++++++ ...2_native_null_on_missing_columns.reference | 4 ++ .../02562_native_null_on_missing_columns.sh | 16 +++++ 29 files changed, 352 insertions(+), 100 deletions(-) create mode 100644 src/Formats/insertNullAsDefaultIfNeeded.cpp create mode 100644 src/Formats/insertNullAsDefaultIfNeeded.h create mode 100644 tests/queries/0_stateless/02561_null_as_default_more_formats.reference create mode 100755 tests/queries/0_stateless/02561_null_as_default_more_formats.sh create mode 100644 tests/queries/0_stateless/02562_native_null_on_missing_columns.reference create mode 100755 tests/queries/0_stateless/02562_native_null_on_missing_columns.sh diff --git a/src/Columns/ColumnLowCardinality.cpp b/src/Columns/ColumnLowCardinality.cpp index ecdaf240e5e..109bf201836 100644 --- a/src/Columns/ColumnLowCardinality.cpp +++ b/src/Columns/ColumnLowCardinality.cpp @@ -830,4 +830,32 @@ void ColumnLowCardinality::Dictionary::compact(ColumnPtr & positions) shared = false; } +ColumnPtr ColumnLowCardinality::cloneWithDefaultOnNull() const +{ + if (!nestedIsNullable()) + return getPtr(); + + auto res = cloneEmpty(); + auto & lc_res = assert_cast(*res); + lc_res.nestedRemoveNullable(); + size_t end = size(); + size_t start = 0; + while (start < end) + { + size_t next_null_index = start; + while (next_null_index < end && !isNullAt(next_null_index)) + ++next_null_index; + + if (next_null_index != start) + lc_res.insertRangeFrom(*this, start, next_null_index - start); + + if (next_null_index < end) + lc_res.insertDefault(); + + start = next_null_index + 1; + } + + return res; +} + } diff --git a/src/Columns/ColumnLowCardinality.h b/src/Columns/ColumnLowCardinality.h index e895bc6b54e..3d42f82a867 100644 --- a/src/Columns/ColumnLowCardinality.h +++ b/src/Columns/ColumnLowCardinality.h @@ -220,6 +220,8 @@ public: void nestedToNullable() { dictionary.getColumnUnique().nestedToNullable(); } void nestedRemoveNullable() { dictionary.getColumnUnique().nestedRemoveNullable(); } + ColumnPtr cloneWithDefaultOnNull() const; + const IColumnUnique & getDictionary() const { return dictionary.getColumnUnique(); } IColumnUnique & getDictionary() { return dictionary.getColumnUnique(); } const ColumnPtr & getDictionaryPtr() const { return dictionary.getColumnUniquePtr(); } diff --git a/src/Columns/ColumnNullable.cpp b/src/Columns/ColumnNullable.cpp index 9398c66bef0..99d377f10eb 100644 --- a/src/Columns/ColumnNullable.cpp +++ b/src/Columns/ColumnNullable.cpp @@ -781,6 +781,28 @@ ColumnPtr ColumnNullable::createWithOffsets(const IColumn::Offsets & offsets, co return ColumnNullable::create(new_values, new_null_map); } +ColumnPtr ColumnNullable::getNestedColumnWithDefaultOnNull() const +{ + auto res = nested_column->cloneEmpty(); + const auto & null_map_data = getNullMapData(); + size_t start = 0; + while (start < nested_column->size()) + { + size_t next_null_index = start; + while (next_null_index < null_map->size() && !null_map_data[next_null_index]) + ++next_null_index; + + if (next_null_index != start) + res->insertRangeFrom(*nested_column, start, next_null_index - start); + + if (next_null_index < null_map->size()) + res->insertDefault(); + + start = next_null_index + 1; + } + return res; +} + ColumnPtr makeNullable(const ColumnPtr & column) { if (isColumnNullable(*column)) diff --git a/src/Columns/ColumnNullable.h b/src/Columns/ColumnNullable.h index 85bf095a9d1..1ec037092b5 100644 --- a/src/Columns/ColumnNullable.h +++ b/src/Columns/ColumnNullable.h @@ -188,6 +188,8 @@ public: NullMap & getNullMapData() { return getNullMapColumn().getData(); } const NullMap & getNullMapData() const { return getNullMapColumn().getData(); } + ColumnPtr getNestedColumnWithDefaultOnNull() const; + /// Apply the null byte map of a specified nullable column onto the /// null byte map of the current column by performing an element-wise OR /// between both byte maps. This method is used to determine the null byte diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 6e085fd27ac..481929d915f 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -751,7 +751,7 @@ class IColumn; M(Bool, input_format_csv_empty_as_default, true, "Treat empty fields in CSV input as default values.", 0) \ M(Bool, input_format_tsv_empty_as_default, false, "Treat empty fields in TSV input as default values.", 0) \ M(Bool, input_format_tsv_enum_as_number, false, "Treat inserted enum values in TSV formats as enum indices.", 0) \ - M(Bool, input_format_null_as_default, true, "For text input formats initialize null fields with default values if data type of this field is not nullable", 0) \ + M(Bool, input_format_null_as_default, true, "For most input formats initialize null fields with default values if data type of this field is not nullable", 0) \ M(Bool, input_format_arrow_import_nested, false, "Allow to insert array of structs into Nested table in Arrow input format.", 0) \ M(Bool, input_format_arrow_case_insensitive_column_matching, false, "Ignore case when matching Arrow columns with CH columns.", 0) \ M(Bool, input_format_orc_import_nested, false, "Allow to insert array of structs into Nested table in ORC input format.", 0) \ @@ -811,6 +811,7 @@ class IColumn; M(Bool, input_format_values_deduce_templates_of_expressions, true, "For Values format: if the field could not be parsed by streaming parser, run SQL parser, deduce template of the SQL expression, try to parse all rows using template and then interpret expression for all rows.", 0) \ M(Bool, input_format_values_accurate_types_of_literals, true, "For Values format: when parsing and interpreting expressions using template, check actual type of literal to avoid possible overflow and precision issues.", 0) \ M(Bool, input_format_avro_allow_missing_fields, false, "For Avro/AvroConfluent format: when field is not found in schema use default value instead of error", 0) \ + /** This setting is obsolete and do nothing, left for compatibility reasons. */ \ M(Bool, input_format_avro_null_as_default, false, "For Avro/AvroConfluent format: insert default in case of null and non Nullable column", 0) \ M(UInt64, format_binary_max_string_size, 1_GiB, "The maximum allowed size for String in RowBinary format. It prevents allocating large amount of memory in case of corrupted data. 0 means there is no limit", 0) \ M(URI, format_avro_schema_registry_url, "", "For AvroConfluent format: Confluent Schema Registry URL.", 0) \ diff --git a/src/DataTypes/IDataType.h b/src/DataTypes/IDataType.h index bafe03dbc3a..f2230b70cab 100644 --- a/src/DataTypes/IDataType.h +++ b/src/DataTypes/IDataType.h @@ -548,6 +548,11 @@ inline bool isAggregateFunction(const DataTypePtr & data_type) return which.isAggregateFunction(); } +inline bool isNullableOrLowCardinalityNullable(const DataTypePtr & data_type) +{ + return data_type->isNullable() || data_type->isLowCardinalityNullable(); +} + template constexpr bool IsDataTypeDecimal = false; template constexpr bool IsDataTypeNumber = false; template constexpr bool IsDataTypeDateOrDateTime = false; diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 3fcecd23f5b..8d8ffebe270 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -56,7 +56,6 @@ FormatSettings getFormatSettings(ContextPtr context, const Settings & settings) format_settings.avro.schema_registry_url = settings.format_avro_schema_registry_url.toString(); format_settings.avro.string_column_pattern = settings.output_format_avro_string_column_pattern.toString(); format_settings.avro.output_rows_in_file = settings.output_format_avro_rows_in_file; - format_settings.avro.null_as_default = settings.input_format_avro_null_as_default; format_settings.csv.allow_double_quotes = settings.format_csv_allow_double_quotes; format_settings.csv.allow_single_quotes = settings.format_csv_allow_single_quotes; format_settings.csv.crlf_end_of_line = settings.output_format_csv_crlf_end_of_line; diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index 92e499abb10..2bf8e136c63 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -104,7 +104,6 @@ struct FormatSettings bool allow_missing_fields = false; String string_column_pattern; UInt64 output_rows_in_file = 1; - bool null_as_default = false; } avro; String bool_true_representation = "true"; diff --git a/src/Formats/NativeReader.cpp b/src/Formats/NativeReader.cpp index 58baee5931b..9f8d4ba1930 100644 --- a/src/Formats/NativeReader.cpp +++ b/src/Formats/NativeReader.cpp @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -32,8 +33,19 @@ NativeReader::NativeReader(ReadBuffer & istr_, UInt64 server_revision_) { } -NativeReader::NativeReader(ReadBuffer & istr_, const Block & header_, UInt64 server_revision_, bool skip_unknown_columns_) - : istr(istr_), header(header_), server_revision(server_revision_), skip_unknown_columns(skip_unknown_columns_) +NativeReader::NativeReader( + ReadBuffer & istr_, + const Block & header_, + UInt64 server_revision_, + bool skip_unknown_columns_, + bool null_as_default_, + BlockMissingValues * block_missing_values_) + : istr(istr_) + , header(header_) + , server_revision(server_revision_) + , skip_unknown_columns(skip_unknown_columns_) + , null_as_default(null_as_default_) + , block_missing_values(block_missing_values_) { } @@ -187,8 +199,12 @@ Block NativeReader::read() { if (header.has(column.name)) { - /// Support insert from old clients without low cardinality type. auto & header_column = header.getByName(column.name); + + if (null_as_default) + insertNullAsDefaultIfNeeded(column, header_column, header.getPositionByName(column.name), block_missing_values); + + /// Support insert from old clients without low cardinality type. if (!header_column.type->equals(*column.type)) { column.column = recursiveTypeConversion(column.column, column.type, header.safeGetByPosition(i).type); @@ -225,12 +241,19 @@ Block NativeReader::read() /// Allow to skip columns. Fill them with default values. Block tmp_res; - for (auto & col : header) + for (size_t column_i = 0; column_i != header.columns(); ++column_i) { + auto & col = header.getByPosition(column_i); if (res.has(col.name)) + { tmp_res.insert(res.getByName(col.name)); + } else + { tmp_res.insert({col.type->createColumn()->cloneResized(rows), col.type, col.name}); + if (block_missing_values) + block_missing_values->setBits(column_i, rows); + } } tmp_res.info = res.info; diff --git a/src/Formats/NativeReader.h b/src/Formats/NativeReader.h index 3ae53d45faf..64d3e4d6df0 100644 --- a/src/Formats/NativeReader.h +++ b/src/Formats/NativeReader.h @@ -24,7 +24,13 @@ public: /// For cases when data structure (header) is known in advance. /// NOTE We may use header for data validation and/or type conversions. It is not implemented. - NativeReader(ReadBuffer & istr_, const Block & header_, UInt64 server_revision_, bool skip_unknown_columns_ = false); + NativeReader( + ReadBuffer & istr_, + const Block & header_, + UInt64 server_revision_, + bool skip_unknown_columns_ = false, + bool null_as_default_ = false, + BlockMissingValues * block_missing_values_ = nullptr); /// For cases when we have an index. It allows to skip columns. Only columns specified in the index will be read. NativeReader(ReadBuffer & istr_, UInt64 server_revision_, @@ -44,6 +50,8 @@ private: Block header; UInt64 server_revision; bool skip_unknown_columns; + bool null_as_default; + BlockMissingValues * block_missing_values; bool use_index = false; IndexForNativeFormat::Blocks::const_iterator index_block_it; diff --git a/src/Formats/insertNullAsDefaultIfNeeded.cpp b/src/Formats/insertNullAsDefaultIfNeeded.cpp new file mode 100644 index 00000000000..767892718c5 --- /dev/null +++ b/src/Formats/insertNullAsDefaultIfNeeded.cpp @@ -0,0 +1,37 @@ +#include +#include +#include +#include +#include + +namespace DB +{ + +void insertNullAsDefaultIfNeeded(ColumnWithTypeAndName & input_column, const ColumnWithTypeAndName & header_column, size_t column_i, BlockMissingValues * block_missing_values) +{ + if (!isNullableOrLowCardinalityNullable(input_column.type) || isNullableOrLowCardinalityNullable(header_column.type)) + return; + + if (block_missing_values) + { + for (size_t i = 0; i < input_column.column->size(); ++i) + { + if (input_column.column->isNullAt(i)) + block_missing_values->setBit(column_i, i); + } + } + + if (input_column.type->isNullable()) + { + input_column.column = assert_cast(input_column.column.get())->getNestedColumnWithDefaultOnNull(); + input_column.type = removeNullable(input_column.type); + } + else + { + input_column.column = assert_cast(input_column.column.get())->cloneWithDefaultOnNull(); + const auto * lc_type = assert_cast(input_column.type.get()); + input_column.type = std::make_shared(removeNullable(lc_type->getDictionaryType())); + } +} + +} diff --git a/src/Formats/insertNullAsDefaultIfNeeded.h b/src/Formats/insertNullAsDefaultIfNeeded.h new file mode 100644 index 00000000000..3e4dcd1e74a --- /dev/null +++ b/src/Formats/insertNullAsDefaultIfNeeded.h @@ -0,0 +1,10 @@ +#pragma once + +#include + +namespace DB +{ + +void insertNullAsDefaultIfNeeded(ColumnWithTypeAndName & input_column, const ColumnWithTypeAndName & header_column, size_t column_i, BlockMissingValues * block_missing_values); + +} diff --git a/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp b/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp index ed963d8a500..cd8facb83eb 100644 --- a/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp @@ -71,13 +71,10 @@ Chunk ArrowBlockInputFormat::generate() ++record_batch_current; - arrow_column_to_ch_column->arrowTableToCHChunk(res, *table_result, (*table_result)->num_rows()); - /// If defaults_for_omitted_fields is true, calculate the default values from default expression for omitted fields. /// Otherwise fill the missing columns with zero values of its type. - if (format_settings.defaults_for_omitted_fields) - for (const auto & column_idx : missing_columns) - block_missing_values.setBits(column_idx, res.getNumRows()); + BlockMissingValues * block_missing_values_ptr = format_settings.defaults_for_omitted_fields ? &block_missing_values : nullptr; + arrow_column_to_ch_column->arrowTableToCHChunk(res, *table_result, (*table_result)->num_rows(), block_missing_values_ptr); return res; } @@ -143,8 +140,8 @@ void ArrowBlockInputFormat::prepareReader() "Arrow", format_settings.arrow.import_nested, format_settings.arrow.allow_missing_columns, + format_settings.null_as_default, format_settings.arrow.case_insensitive_column_matching); - missing_columns = arrow_column_to_ch_column->getMissingColumns(*schema); if (stream) record_batch_total = -1; diff --git a/src/Processors/Formats/Impl/ArrowBlockInputFormat.h b/src/Processors/Formats/Impl/ArrowBlockInputFormat.h index 02648d28048..3db76777891 100644 --- a/src/Processors/Formats/Impl/ArrowBlockInputFormat.h +++ b/src/Processors/Formats/Impl/ArrowBlockInputFormat.h @@ -47,7 +47,6 @@ private: int record_batch_total = 0; int record_batch_current = 0; - std::vector missing_columns; BlockMissingValues block_missing_values; const FormatSettings format_settings; diff --git a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp index 68c40527097..80172ca9c05 100644 --- a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp +++ b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -384,9 +385,10 @@ static ColumnWithTypeAndName readColumnWithIndexesDataImpl(std::shared_ptr(buffer->data()); /// Check that indexes are correct (protection against corrupted files) + /// Note that on null values index can be arbitrary value. for (int64_t i = 0; i != chunk->length(); ++i) { - if (data[i] < 0 || data[i] >= dict_size) + if (!chunk->IsNull(i) && (data[i] < 0 || data[i] >= dict_size)) throw Exception(ErrorCodes::INCORRECT_DATA, "Index {} in Dictionary column is out of bounds, dictionary size is {}", Int64(data[i]), UInt64(dict_size)); @@ -805,16 +807,18 @@ ArrowColumnToCHColumn::ArrowColumnToCHColumn( const std::string & format_name_, bool import_nested_, bool allow_missing_columns_, + bool null_as_default_, bool case_insensitive_matching_) : header(header_) , format_name(format_name_) , import_nested(import_nested_) , allow_missing_columns(allow_missing_columns_) + , null_as_default(null_as_default_) , case_insensitive_matching(case_insensitive_matching_) { } -void ArrowColumnToCHColumn::arrowTableToCHChunk(Chunk & res, std::shared_ptr & table, size_t num_rows) +void ArrowColumnToCHColumn::arrowTableToCHChunk(Chunk & res, std::shared_ptr & table, size_t num_rows, BlockMissingValues * block_missing_values) { NameToColumnPtr name_to_column_ptr; for (auto column_name : table->ColumnNames()) @@ -828,10 +832,10 @@ void ArrowColumnToCHColumn::arrowTableToCHChunk(Chunk & res, std::shared_ptrcloneResized(num_rows); columns_list.push_back(std::move(column.column)); + if (block_missing_values) + block_missing_values->setBits(column_i, num_rows); continue; } } @@ -906,6 +912,9 @@ void ArrowColumnToCHColumn::arrowColumnsToCHChunk(Chunk & res, NameToColumnPtr & arrow_column, header_column.name, format_name, false, dictionary_infos, true, false, skipped, header_column.type); } + if (null_as_default) + insertNullAsDefaultIfNeeded(column, header_column, column_i, block_missing_values); + try { column.column = castColumn(column, header_column.type); @@ -927,28 +936,6 @@ void ArrowColumnToCHColumn::arrowColumnsToCHChunk(Chunk & res, NameToColumnPtr & res.setColumns(columns_list, num_rows); } -std::vector ArrowColumnToCHColumn::getMissingColumns(const arrow::Schema & schema) const -{ - std::vector missing_columns; - auto block_from_arrow = arrowSchemaToCHHeader(schema, format_name, false, &header, case_insensitive_matching); - NestedColumnExtractHelper nested_columns_extractor(block_from_arrow, case_insensitive_matching); - - for (size_t i = 0, columns = header.columns(); i < columns; ++i) - { - const auto & header_column = header.getByPosition(i); - if (!block_from_arrow.has(header_column.name, case_insensitive_matching)) - { - if (!import_nested || !nested_columns_extractor.extractColumn(header_column.name)) - { - if (!allow_missing_columns) - throw Exception{ErrorCodes::THERE_IS_NO_COLUMN, "Column '{}' is not presented in input data.", header_column.name}; - missing_columns.push_back(i); - } - } - } - return missing_columns; -} - } #endif diff --git a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h index dd9f44eb94e..64ff99c70ac 100644 --- a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h +++ b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h @@ -26,14 +26,12 @@ public: const std::string & format_name_, bool import_nested_, bool allow_missing_columns_, + bool null_as_default_, bool case_insensitive_matching_ = false); - void arrowTableToCHChunk(Chunk & res, std::shared_ptr & table, size_t num_rows); + void arrowTableToCHChunk(Chunk & res, std::shared_ptr & table, size_t num_rows, BlockMissingValues * block_missing_values = nullptr); - void arrowColumnsToCHChunk(Chunk & res, NameToColumnPtr & name_to_column_ptr, size_t num_rows); - - /// Get missing columns that exists in header but not in arrow::Schema - std::vector getMissingColumns(const arrow::Schema & schema) const; + void arrowColumnsToCHChunk(Chunk & res, NameToColumnPtr & name_to_column_ptr, size_t num_rows, BlockMissingValues * block_missing_values = nullptr); /// Transform arrow schema to ClickHouse header. If hint_header is provided, /// we will skip columns in schema that are not in hint_header. @@ -58,6 +56,7 @@ private: bool import_nested; /// If false, throw exception if some columns in header not exists in arrow table. bool allow_missing_columns; + bool null_as_default; bool case_insensitive_matching; /// Map {column name : dictionary column}. diff --git a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp index 9a475efa195..e77f4132100 100644 --- a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp @@ -176,8 +176,9 @@ AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::Node { auto & lc_column = assert_cast(column); auto tmp_column = lc_column.getDictionary().getNestedColumn()->cloneEmpty(); - dict_deserialize(*tmp_column, decoder); + auto res = dict_deserialize(*tmp_column, decoder); lc_column.insertFromFullColumn(*tmp_column, 0); + return res; }; } @@ -198,6 +199,7 @@ AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::Node UUID uuid; parseUUID(reinterpret_cast(tmp.data()), std::reverse_iterator(reinterpret_cast(&uuid) + 16)); assert_cast(column).insertValue(uuid); + return true; }; } if (target.isString() || target.isFixedString()) @@ -206,6 +208,7 @@ AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::Node { decoder.decodeString(tmp); column.insertData(tmp.c_str(), tmp.length()); + return true; }; } break; @@ -215,6 +218,7 @@ AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::Node return [target](IColumn & column, avro::Decoder & decoder) { insertNumber(column, target, decoder.decodeInt()); + return true; }; } break; @@ -224,6 +228,7 @@ AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::Node return [target](IColumn & column, avro::Decoder & decoder) { insertNumber(column, target, decoder.decodeLong()); + return true; }; } break; @@ -233,6 +238,7 @@ AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::Node return [target](IColumn & column, avro::Decoder & decoder) { insertNumber(column, target, decoder.decodeFloat()); + return true; }; } break; @@ -242,6 +248,7 @@ AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::Node return [target](IColumn & column, avro::Decoder & decoder) { insertNumber(column, target, decoder.decodeDouble()); + return true; }; } break; @@ -251,6 +258,7 @@ AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::Node return [target](IColumn & column, avro::Decoder & decoder) { insertNumber(column, target, decoder.decodeBool()); + return true; }; } break; @@ -275,6 +283,7 @@ AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::Node } } offsets.push_back(offsets.back() + total); + return true; }; } break; @@ -301,24 +310,33 @@ AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::Node { col.insertDefault(); } + return true; }; } - - /// If the Union is ['Null', Nested-Type], since the Nested-Type can not be inside - /// Nullable, so we will get Nested-Type, instead of Nullable type. - if (null_as_default || !target.isNullable()) + else if (null_as_default) { auto nested_deserialize = this->createDeserializeFn(root_node->leafAt(non_null_union_index), target_type); return [non_null_union_index, nested_deserialize](IColumn & column, avro::Decoder & decoder) { int union_index = static_cast(decoder.decodeUnionIndex()); if (union_index == non_null_union_index) + { nested_deserialize(column, decoder); - else - column.insertDefault(); + return true; + } + column.insertDefault(); + return false; }; } - + else + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Cannot insert Avro Union(Null, {}) into non-nullable type {}. To use default value on NULL, enable setting " + "input_format_null_as_default", + avro::toString(root_node->leafAt(non_null_union_index)->type()), + target_type->getName()); + } } break; } @@ -331,6 +349,7 @@ AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::Node return [](IColumn &, avro::Decoder & decoder) { decoder.decodeNull(); + return true; }; } else @@ -340,10 +359,26 @@ AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::Node ColumnNullable & col = assert_cast(column); decoder.decodeNull(); col.insertDefault(); + return true; }; } } - break; + else if (null_as_default) + { + return [](IColumn & column, avro::Decoder & decoder) + { + decoder.decodeNull(); + column.insertDefault(); + return false; + }; + } + else + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Cannot insert Avro Null into non-nullable type {}. To use default value on NULL, enable setting " + "input_format_null_as_default", target_type->getName()); + } case avro::AVRO_ENUM: if (target.isString()) { @@ -358,6 +393,7 @@ AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::Node size_t enum_index = decoder.decodeEnum(); const auto & enum_symbol = symbols[enum_index]; column.insertData(enum_symbol.c_str(), enum_symbol.length()); + return true; }; } if (target.isEnum()) @@ -372,6 +408,7 @@ AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::Node { size_t enum_index = decoder.decodeEnum(); column.insert(symbol_mapping[enum_index]); + return true; }; } break; @@ -384,6 +421,7 @@ AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::Node { decoder.decodeFixed(tmp_fixed.size(), tmp_fixed); column.insertData(reinterpret_cast(tmp_fixed.data()), tmp_fixed.size()); + return true; }; } break; @@ -415,6 +453,7 @@ AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::Node auto nested_columns = column_tuple.getColumns(); for (const auto & [nested_deserializer, pos] : nested_deserializers) nested_deserializer(*nested_columns[pos], decoder); + return true; }; } break; @@ -449,6 +488,7 @@ AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::Node } } offsets.push_back(offsets.back() + total); + return true; }; } break; @@ -465,6 +505,7 @@ AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::Node ColumnNullable & col = assert_cast(column); nested_deserialize(col.getNestedColumn(), decoder); col.getNullMapData().push_back(0); + return true; }; } @@ -593,7 +634,6 @@ void AvroDeserializer::Action::deserializeNested(MutableColumns & columns, avro: ColumnArray & column_array = assert_cast(*columns[index]); arrays_offsets.push_back(&column_array.getOffsets()); nested_columns.push_back(&column_array.getData()); - ext.read_columns[index] = true; } size_t total = 0; @@ -603,7 +643,7 @@ void AvroDeserializer::Action::deserializeNested(MutableColumns & columns, avro: for (size_t i = 0; i < n; ++i) { for (size_t j = 0; j != nested_deserializers.size(); ++j) - nested_deserializers[j](*nested_columns[j], decoder); + ext.read_columns[nested_column_indexes[j]] = nested_deserializers[j](*nested_columns[j], decoder); } } @@ -742,7 +782,8 @@ void AvroDeserializer::deserializeRow(MutableColumns & columns, avro::Decoder & row_action.execute(columns, decoder, ext); for (size_t i = 0; i < ext.read_columns.size(); ++i) { - if (!ext.read_columns[i]) + /// Insert default in missing columns. + if (!column_found[i]) { columns[i]->insertDefault(); } @@ -759,7 +800,7 @@ void AvroRowInputFormat::readPrefix() { file_reader_ptr = std::make_unique(std::make_unique(*in)); deserializer_ptr = std::make_unique( - output.getHeader(), file_reader_ptr->dataSchema(), format_settings.avro.allow_missing_fields, format_settings.avro.null_as_default); + output.getHeader(), file_reader_ptr->dataSchema(), format_settings.avro.allow_missing_fields, format_settings.null_as_default); file_reader_ptr->init(); } @@ -950,7 +991,7 @@ const AvroDeserializer & AvroConfluentRowInputFormat::getOrCreateDeserializer(Sc { auto schema = schema_registry->getSchema(schema_id); AvroDeserializer deserializer( - output.getHeader(), schema, format_settings.avro.allow_missing_fields, format_settings.avro.null_as_default); + output.getHeader(), schema, format_settings.avro.allow_missing_fields, format_settings.null_as_default); it = deserializer_cache.emplace(schema_id, deserializer).first; } return it->second; diff --git a/src/Processors/Formats/Impl/AvroRowInputFormat.h b/src/Processors/Formats/Impl/AvroRowInputFormat.h index 96370b8c4c7..dcd51398032 100644 --- a/src/Processors/Formats/Impl/AvroRowInputFormat.h +++ b/src/Processors/Formats/Impl/AvroRowInputFormat.h @@ -35,8 +35,8 @@ public: void deserializeRow(MutableColumns & columns, avro::Decoder & decoder, RowReadExtension & ext) const; private: - using DeserializeFn = std::function; - using DeserializeNestedFn = std::function; + using DeserializeFn = std::function; + using DeserializeNestedFn = std::function; using SkipFn = std::function; DeserializeFn createDeserializeFn(avro::NodePtr root_node, DataTypePtr target_type); @@ -86,8 +86,7 @@ private: case Noop: break; case Deserialize: - deserialize_fn(*columns[target_column_idx], decoder); - ext.read_columns[target_column_idx] = true; + ext.read_columns[target_column_idx] = deserialize_fn(*columns[target_column_idx], decoder); break; case Skip: skip_fn(decoder); diff --git a/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp b/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp index f337eedbb05..488f4ff9a73 100644 --- a/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp @@ -45,11 +45,11 @@ namespace ErrorCodes extern const int UNEXPECTED_END_OF_FILE; } -MsgPackRowInputFormat::MsgPackRowInputFormat(const Block & header_, ReadBuffer & in_, Params params_) - : MsgPackRowInputFormat(header_, std::make_unique(in_), params_) {} +MsgPackRowInputFormat::MsgPackRowInputFormat(const Block & header_, ReadBuffer & in_, Params params_, const FormatSettings & settings) + : MsgPackRowInputFormat(header_, std::make_unique(in_), params_, settings) {} -MsgPackRowInputFormat::MsgPackRowInputFormat(const Block & header_, std::unique_ptr buf_, Params params_) - : IRowInputFormat(header_, *buf_, std::move(params_)), buf(std::move(buf_)), parser(visitor), data_types(header_.getDataTypes()) {} +MsgPackRowInputFormat::MsgPackRowInputFormat(const Block & header_, std::unique_ptr buf_, Params params_, const FormatSettings & settings) + : IRowInputFormat(header_, *buf_, std::move(params_)), buf(std::move(buf_)), visitor(settings.null_as_default), parser(visitor), data_types(header_.getDataTypes()) {} void MsgPackRowInputFormat::resetParser() { @@ -58,13 +58,13 @@ void MsgPackRowInputFormat::resetParser() visitor.reset(); } -void MsgPackVisitor::set_info(IColumn & column, DataTypePtr type) // NOLINT +void MsgPackVisitor::set_info(IColumn & column, DataTypePtr type, UInt8 & read) // NOLINT { while (!info_stack.empty()) { info_stack.pop(); } - info_stack.push(Info{column, type}); + info_stack.push(Info{column, type, &read}); } void MsgPackVisitor::reset() @@ -228,11 +228,11 @@ static void insertFloat64(IColumn & column, DataTypePtr type, Float64 value) // assert_cast(column).insertValue(value); } -static void insertNull(IColumn & column, DataTypePtr type) +static void insertNull(IColumn & column, DataTypePtr type, UInt8 * read, bool null_as_default) { auto insert_func = [&](IColumn & column_, DataTypePtr type_) { - insertNull(column_, type_); + insertNull(column_, type_, read, null_as_default); }; /// LowCardinality(Nullable(...)) @@ -240,7 +240,16 @@ static void insertNull(IColumn & column, DataTypePtr type) return; if (!type->isNullable()) - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Cannot insert MessagePack null into non-nullable column with type {}.", type->getName()); + { + if (!null_as_default) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, "Cannot insert MessagePack null into non-nullable column with type {}.", type->getName()); + column.insertDefault(); + /// In case of default on null column can have defined DEFAULT expression that should be used. + if (read) + *read = false; + return; + } assert_cast(column).insertDefault(); } @@ -316,7 +325,7 @@ bool MsgPackVisitor::start_array(size_t size) // NOLINT ColumnArray::Offsets & offsets = column_array.getOffsets(); IColumn & nested_column = column_array.getData(); offsets.push_back(offsets.back() + size); - info_stack.push(Info{nested_column, nested_type}); + info_stack.push(Info{nested_column, nested_type, nullptr}); return true; } @@ -340,7 +349,7 @@ bool MsgPackVisitor::start_map_key() // NOLINT { auto key_column = assert_cast(info_stack.top().column).getNestedData().getColumns()[0]; auto key_type = assert_cast(*info_stack.top().type).getKeyType(); - info_stack.push(Info{*key_column, key_type}); + info_stack.push(Info{*key_column, key_type, nullptr}); return true; } @@ -354,7 +363,7 @@ bool MsgPackVisitor::start_map_value() // NOLINT { auto value_column = assert_cast(info_stack.top().column).getNestedData().getColumns()[1]; auto value_type = assert_cast(*info_stack.top().type).getValueType(); - info_stack.push(Info{*value_column, value_type}); + info_stack.push(Info{*value_column, value_type, nullptr}); return true; } @@ -366,7 +375,7 @@ bool MsgPackVisitor::end_map_value() // NOLINT bool MsgPackVisitor::visit_nil() { - insertNull(info_stack.top().column, info_stack.top().type); + insertNull(info_stack.top().column, info_stack.top().type, info_stack.top().read, null_as_default); return true; } @@ -407,13 +416,14 @@ bool MsgPackRowInputFormat::readObject() return true; } -bool MsgPackRowInputFormat::readRow(MutableColumns & columns, RowReadExtension &) +bool MsgPackRowInputFormat::readRow(MutableColumns & columns, RowReadExtension & ext) { size_t column_index = 0; bool has_more_data = true; + ext.read_columns.resize(columns.size(), true); for (; column_index != columns.size(); ++column_index) { - visitor.set_info(*columns[column_index], data_types[column_index]); + visitor.set_info(*columns[column_index], data_types[column_index], ext.read_columns[column_index]); has_more_data = readObject(); if (!has_more_data) break; @@ -547,9 +557,9 @@ void registerInputFormatMsgPack(FormatFactory & factory) ReadBuffer & buf, const Block & sample, const RowInputFormatParams & params, - const FormatSettings &) + const FormatSettings & settings) { - return std::make_shared(sample, buf, params); + return std::make_shared(sample, buf, params, settings); }); factory.registerFileExtension("messagepack", "MsgPack"); } diff --git a/src/Processors/Formats/Impl/MsgPackRowInputFormat.h b/src/Processors/Formats/Impl/MsgPackRowInputFormat.h index 64bb8b569e0..5eaa3719d0c 100644 --- a/src/Processors/Formats/Impl/MsgPackRowInputFormat.h +++ b/src/Processors/Formats/Impl/MsgPackRowInputFormat.h @@ -19,10 +19,13 @@ class ReadBuffer; class MsgPackVisitor : public msgpack::null_visitor { public: + MsgPackVisitor(bool null_as_default_) : null_as_default(null_as_default_) {} + struct Info { IColumn & column; DataTypePtr type; + UInt8 * read; }; /// These functions are called when parser meets corresponding object in parsed data @@ -47,25 +50,26 @@ public: [[noreturn]] void parse_error(size_t parsed_offset, size_t error_offset); /// Update info_stack - void set_info(IColumn & column, DataTypePtr type); + void set_info(IColumn & column, DataTypePtr type, UInt8 & read); void reset(); private: /// Stack is needed to process arrays and maps std::stack info_stack; + bool null_as_default; }; class MsgPackRowInputFormat : public IRowInputFormat { public: - MsgPackRowInputFormat(const Block & header_, ReadBuffer & in_, Params params_); + MsgPackRowInputFormat(const Block & header_, ReadBuffer & in_, Params params_, const FormatSettings & settings); String getName() const override { return "MagPackRowInputFormat"; } void resetParser() override; void setReadBuffer(ReadBuffer & in_) override; private: - MsgPackRowInputFormat(const Block & header_, std::unique_ptr buf_, Params params_); + MsgPackRowInputFormat(const Block & header_, std::unique_ptr buf_, Params params_, const FormatSettings & settings); bool readRow(MutableColumns & columns, RowReadExtension & ext) override; diff --git a/src/Processors/Formats/Impl/NativeFormat.cpp b/src/Processors/Formats/Impl/NativeFormat.cpp index 959b86ec051..3c1a2bd5965 100644 --- a/src/Processors/Formats/Impl/NativeFormat.cpp +++ b/src/Processors/Formats/Impl/NativeFormat.cpp @@ -17,7 +17,13 @@ class NativeInputFormat final : public IInputFormat public: NativeInputFormat(ReadBuffer & buf, const Block & header_, const FormatSettings & settings) : IInputFormat(header_, buf) - , reader(std::make_unique(buf, header_, 0, settings.skip_unknown_fields)) + , reader(std::make_unique( + buf, + header_, + 0, + settings.skip_unknown_fields, + settings.null_as_default, + settings.defaults_for_omitted_fields ? &block_missing_values : nullptr)) , header(header_) {} String getName() const override { return "Native"; } @@ -30,6 +36,7 @@ public: Chunk generate() override { + block_missing_values.clear(); auto block = reader->read(); if (!block) return {}; @@ -47,9 +54,12 @@ public: IInputFormat::setReadBuffer(in_); } + const BlockMissingValues & getMissingValues() const override { return block_missing_values; } + private: std::unique_ptr reader; Block header; + BlockMissingValues block_missing_values; }; class NativeOutputFormat final : public IOutputFormat diff --git a/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp b/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp index 2e45d817506..03f056e22b3 100644 --- a/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp @@ -67,12 +67,10 @@ Chunk ORCBlockInputFormat::generate() ++stripe_current; Chunk res; - arrow_column_to_ch_column->arrowTableToCHChunk(res, table, num_rows); /// If defaults_for_omitted_fields is true, calculate the default values from default expression for omitted fields. /// Otherwise fill the missing columns with zero values of its type. - if (format_settings.defaults_for_omitted_fields) - for (const auto & column_idx : missing_columns) - block_missing_values.setBits(column_idx, res.getNumRows()); + BlockMissingValues * block_missing_values_ptr = format_settings.defaults_for_omitted_fields ? &block_missing_values : nullptr; + arrow_column_to_ch_column->arrowTableToCHChunk(res, table, num_rows, block_missing_values_ptr); return res; } @@ -128,8 +126,8 @@ void ORCBlockInputFormat::prepareReader() "ORC", format_settings.orc.import_nested, format_settings.orc.allow_missing_columns, + format_settings.null_as_default, format_settings.orc.case_insensitive_column_matching); - missing_columns = arrow_column_to_ch_column->getMissingColumns(*schema); ArrowFieldIndexUtil field_util( format_settings.orc.case_insensitive_column_matching, diff --git a/src/Processors/Formats/Impl/ORCBlockInputFormat.h b/src/Processors/Formats/Impl/ORCBlockInputFormat.h index bc2abe41cc1..3d8bc781278 100644 --- a/src/Processors/Formats/Impl/ORCBlockInputFormat.h +++ b/src/Processors/Formats/Impl/ORCBlockInputFormat.h @@ -49,7 +49,6 @@ private: // indices of columns to read from ORC file std::vector include_indices; - std::vector missing_columns; BlockMissingValues block_missing_values; const FormatSettings format_settings; diff --git a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp index d2ec3c02eed..fca097d8ea7 100644 --- a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp @@ -71,7 +71,10 @@ Chunk ParquetBlockInputFormat::generate() if (*batch) { auto tmp_table = arrow::Table::FromRecordBatches({*batch}); - arrow_column_to_ch_column->arrowTableToCHChunk(res, *tmp_table, (*tmp_table)->num_rows()); + /// If defaults_for_omitted_fields is true, calculate the default values from default expression for omitted fields. + /// Otherwise fill the missing columns with zero values of its type. + BlockMissingValues * block_missing_values_ptr = format_settings.defaults_for_omitted_fields ? &block_missing_values : nullptr; + arrow_column_to_ch_column->arrowTableToCHChunk(res, *tmp_table, (*tmp_table)->num_rows(), block_missing_values_ptr); } else { @@ -80,12 +83,6 @@ Chunk ParquetBlockInputFormat::generate() return {}; } - /// If defaults_for_omitted_fields is true, calculate the default values from default expression for omitted fields. - /// Otherwise fill the missing columns with zero values of its type. - if (format_settings.defaults_for_omitted_fields) - for (const auto & column_idx : missing_columns) - block_missing_values.setBits(column_idx, res.getNumRows()); - return res; } @@ -133,8 +130,8 @@ void ParquetBlockInputFormat::prepareReader() "Parquet", format_settings.parquet.import_nested, format_settings.parquet.allow_missing_columns, + format_settings.null_as_default, format_settings.parquet.case_insensitive_column_matching); - missing_columns = arrow_column_to_ch_column->getMissingColumns(*schema); ArrowFieldIndexUtil field_util( format_settings.parquet.case_insensitive_column_matching, diff --git a/src/Processors/Formats/Impl/ParquetBlockInputFormat.h b/src/Processors/Formats/Impl/ParquetBlockInputFormat.h index 37878a94dd9..afc46939c79 100644 --- a/src/Processors/Formats/Impl/ParquetBlockInputFormat.h +++ b/src/Processors/Formats/Impl/ParquetBlockInputFormat.h @@ -42,7 +42,6 @@ private: // indices of columns to read from Parquet file std::vector column_indices; std::unique_ptr arrow_column_to_ch_column; - std::vector missing_columns; BlockMissingValues block_missing_values; const FormatSettings format_settings; const std::unordered_set & skip_row_groups; diff --git a/tests/queries/0_stateless/02561_null_as_default_more_formats.reference b/tests/queries/0_stateless/02561_null_as_default_more_formats.reference new file mode 100644 index 00000000000..f5d4f41efe8 --- /dev/null +++ b/tests/queries/0_stateless/02561_null_as_default_more_formats.reference @@ -0,0 +1,36 @@ +Parquet +1 +0 0 0 +0 0 +0 0 0 +42 0 42 +Arrow +1 +0 0 0 +0 0 +0 0 0 +42 0 42 +ORC +1 +0 0 0 +0 0 +0 0 0 +42 0 42 +Avro +1 +0 0 0 +0 0 +0 0 0 +42 0 42 +MsgPack +1 +0 0 0 +0 0 +0 0 0 +42 0 42 +Native +1 +0 0 0 +0 0 +0 0 0 +42 0 42 diff --git a/tests/queries/0_stateless/02561_null_as_default_more_formats.sh b/tests/queries/0_stateless/02561_null_as_default_more_formats.sh new file mode 100755 index 00000000000..eacd8e964a6 --- /dev/null +++ b/tests/queries/0_stateless/02561_null_as_default_more_formats.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -q "drop table if exists test" +$CLICKHOUSE_CLIENT -q "create table test (x UInt64 default 42, y UInt64, z LowCardinality(String) default '42') engine=Memory"; +for format in Parquet Arrow ORC Avro MsgPack Native +do + echo $format + $CLICKHOUSE_CLIENT -q "select number % 2 ? NULL : number as x, x as y, CAST(number % 2 ? NULL : toString(number), 'LowCardinality(Nullable(String))') as z from numbers(2) format $format" | $CLICKHOUSE_CLIENT -q "insert into test settings input_format_null_as_default=0 format $format" 2>&1 | grep "Exception" -c + $CLICKHOUSE_CLIENT -q "select number % 2 ? NULL : number as x, x as y, CAST(number % 2 ? NULL : toString(number), 'LowCardinality(Nullable(String))') as z from numbers(2) format $format settings output_format_arrow_low_cardinality_as_dictionary=1" | $CLICKHOUSE_CLIENT -q "insert into test settings input_format_null_as_default=1, input_format_defaults_for_omitted_fields=0 format $format" + $CLICKHOUSE_CLIENT -q "select * from test" + $CLICKHOUSE_CLIENT -q "truncate table test" + $CLICKHOUSE_CLIENT -q "select number % 2 ? NULL : number as x, x as y, CAST(number % 2 ? NULL : toString(number), 'LowCardinality(Nullable(String))') as z from numbers(2) format $format settings output_format_arrow_low_cardinality_as_dictionary=1" | $CLICKHOUSE_CLIENT -q "insert into test settings input_format_null_as_default=1, input_format_defaults_for_omitted_fields=1 format $format" + $CLICKHOUSE_CLIENT -q "select * from test" + $CLICKHOUSE_CLIENT -q "truncate table test" +done + diff --git a/tests/queries/0_stateless/02562_native_null_on_missing_columns.reference b/tests/queries/0_stateless/02562_native_null_on_missing_columns.reference new file mode 100644 index 00000000000..e072efc3352 --- /dev/null +++ b/tests/queries/0_stateless/02562_native_null_on_missing_columns.reference @@ -0,0 +1,4 @@ +0 0 +1 0 +0 42 +1 42 diff --git a/tests/queries/0_stateless/02562_native_null_on_missing_columns.sh b/tests/queries/0_stateless/02562_native_null_on_missing_columns.sh new file mode 100755 index 00000000000..c3d174d77e8 --- /dev/null +++ b/tests/queries/0_stateless/02562_native_null_on_missing_columns.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -q "drop table if exists test" +$CLICKHOUSE_CLIENT -q "create table test (x UInt64, y UInt64 default 42) engine=Memory" + +$CLICKHOUSE_CLIENT -q "select number as x from numbers(2) format Native" | $CLICKHOUSE_CLIENT -q "insert into test settings input_format_defaults_for_omitted_fields=0 format Native" +$CLICKHOUSE_CLIENT -q "select * from test" +$CLICKHOUSE_CLIENT -q "truncate table test" + +$CLICKHOUSE_CLIENT -q "select number as x from numbers(2) format Native" | $CLICKHOUSE_CLIENT -q "insert into test settings input_format_defaults_for_omitted_fields=1 format Native" +$CLICKHOUSE_CLIENT -q "select * from test" +$CLICKHOUSE_CLIENT -q "truncate table test" From 04cf144edc9abba1f9e61ff4f4a949140f8f51e7 Mon Sep 17 00:00:00 2001 From: avogar Date: Fri, 10 Feb 2023 17:20:51 +0000 Subject: [PATCH 154/717] Fix TSKV, update docs --- .../operations/settings/settings-formats.md | 11 +++++--- .../Formats/Impl/TSKVRowInputFormat.cpp | 5 +++- .../02562_native_null_on_missing_columns.sh | 16 ------------ ...tskv_default_for_omitted_fields.reference} | 4 +-- ..._native_tskv_default_for_omitted_fields.sh | 25 +++++++++++++++++++ 5 files changed, 38 insertions(+), 23 deletions(-) delete mode 100755 tests/queries/0_stateless/02562_native_null_on_missing_columns.sh rename tests/queries/0_stateless/{02562_native_null_on_missing_columns.reference => 02562_native_tskv_default_for_omitted_fields.reference} (50%) create mode 100755 tests/queries/0_stateless/02562_native_tskv_default_for_omitted_fields.sh diff --git a/docs/en/operations/settings/settings-formats.md b/docs/en/operations/settings/settings-formats.md index fd727704710..dda97696de5 100644 --- a/docs/en/operations/settings/settings-formats.md +++ b/docs/en/operations/settings/settings-formats.md @@ -15,11 +15,12 @@ When writing data, ClickHouse throws an exception if input data contain columns Supported formats: -- [JSONEachRow](../../interfaces/formats.md/#jsoneachrow) +- [JSONEachRow](../../interfaces/formats.md/#jsoneachrow) (and other JSON formats) +- [BSONEachRow](../../interfaces/formats.md/#bsoneachrow) (and other JSON formats) - [TSKV](../../interfaces/formats.md/#tskv) - All formats with suffixes WithNames/WithNamesAndTypes -- [JSONColumns](../../interfaces/formats.md/#jsoncolumns) - [MySQLDump](../../interfaces/formats.md/#mysqldump) +- [Native](../../interfaces/formats.md/#native) Possible values: @@ -78,7 +79,7 @@ Default value: 1. ## input_format_defaults_for_omitted_fields {#input_format_defaults_for_omitted_fields} -When performing `INSERT` queries, replace omitted input column values with default values of the respective columns. This option only applies to [JSONEachRow](../../interfaces/formats.md/#jsoneachrow), [CSV](../../interfaces/formats.md/#csv), [TabSeparated](../../interfaces/formats.md/#tabseparated) formats and formats with `WithNames`/`WithNamesAndTypes` suffixes. +When performing `INSERT` queries, replace omitted input column values with default values of the respective columns. This option applies to [JSONEachRow](../../interfaces/formats.md/#jsoneachrow) (and other JSON formats), [CSV](../../interfaces/formats.md/#csv), [TabSeparated](../../interfaces/formats.md/#tabseparated), [TSKV](../../interfaces/formats.md/#tskv), [Parquet](../../interfaces/formats.md/#parquet), [Arrow](../../interfaces/formats.md/#arrow), [Avro](../../interfaces/formats.md/#avro), [ORC](../../interfaces/formats.md/#orc), [Native](../../interfaces/formats.md/#native) formats and formats with `WithNames`/`WithNamesAndTypes` suffixes. :::note When this option is enabled, extended table metadata are sent from server to client. It consumes additional computing resources on the server and can reduce performance. @@ -96,7 +97,9 @@ Default value: 1. Enables or disables the initialization of [NULL](../../sql-reference/syntax.md/#null-literal) fields with [default values](../../sql-reference/statements/create/table.md/#create-default-values), if data type of these fields is not [nullable](../../sql-reference/data-types/nullable.md/#data_type-nullable). If column type is not nullable and this setting is disabled, then inserting `NULL` causes an exception. If column type is nullable, then `NULL` values are inserted as is, regardless of this setting. -This setting is applicable to [INSERT ... VALUES](../../sql-reference/statements/insert-into.md) queries for text input formats. +This setting is applicable for most input formats. + +For complex default expressions `input_format_defaults_for_omitted_fields` must be enabled too. Possible values: diff --git a/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp b/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp index bf6d0ab88d2..23a8589bd0a 100644 --- a/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp @@ -193,7 +193,10 @@ bool TSKVRowInputFormat::readRow(MutableColumns & columns, RowReadExtension & ex header.getByPosition(i).type->insertDefaultInto(*columns[i]); /// return info about defaults set - ext.read_columns = read_columns; + if (format_settings.defaults_for_omitted_fields) + ext.read_columns = read_columns; + else + ext.read_columns.assign(num_columns, true); return true; } diff --git a/tests/queries/0_stateless/02562_native_null_on_missing_columns.sh b/tests/queries/0_stateless/02562_native_null_on_missing_columns.sh deleted file mode 100755 index c3d174d77e8..00000000000 --- a/tests/queries/0_stateless/02562_native_null_on_missing_columns.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# shellcheck source=../shell_config.sh -. "$CUR_DIR"/../shell_config.sh - -$CLICKHOUSE_CLIENT -q "drop table if exists test" -$CLICKHOUSE_CLIENT -q "create table test (x UInt64, y UInt64 default 42) engine=Memory" - -$CLICKHOUSE_CLIENT -q "select number as x from numbers(2) format Native" | $CLICKHOUSE_CLIENT -q "insert into test settings input_format_defaults_for_omitted_fields=0 format Native" -$CLICKHOUSE_CLIENT -q "select * from test" -$CLICKHOUSE_CLIENT -q "truncate table test" - -$CLICKHOUSE_CLIENT -q "select number as x from numbers(2) format Native" | $CLICKHOUSE_CLIENT -q "insert into test settings input_format_defaults_for_omitted_fields=1 format Native" -$CLICKHOUSE_CLIENT -q "select * from test" -$CLICKHOUSE_CLIENT -q "truncate table test" diff --git a/tests/queries/0_stateless/02562_native_null_on_missing_columns.reference b/tests/queries/0_stateless/02562_native_tskv_default_for_omitted_fields.reference similarity index 50% rename from tests/queries/0_stateless/02562_native_null_on_missing_columns.reference rename to tests/queries/0_stateless/02562_native_tskv_default_for_omitted_fields.reference index e072efc3352..17197fa3563 100644 --- a/tests/queries/0_stateless/02562_native_null_on_missing_columns.reference +++ b/tests/queries/0_stateless/02562_native_tskv_default_for_omitted_fields.reference @@ -1,4 +1,4 @@ -0 0 1 0 -0 42 +1 42 +1 0 1 42 diff --git a/tests/queries/0_stateless/02562_native_tskv_default_for_omitted_fields.sh b/tests/queries/0_stateless/02562_native_tskv_default_for_omitted_fields.sh new file mode 100755 index 00000000000..a08c948705d --- /dev/null +++ b/tests/queries/0_stateless/02562_native_tskv_default_for_omitted_fields.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -q "drop table if exists test" +$CLICKHOUSE_CLIENT -q "insert into function file(02562_data.native) select 1::UInt64 as x settings engine_file_truncate_on_insert=1" +$CLICKHOUSE_CLIENT -q "create table test (x UInt64, y UInt64 default 42) engine=File(Native, '02562_data.native') settings input_format_defaults_for_omitted_fields=0" +$CLICKHOUSE_CLIENT -q "select * from test" +$CLICKHOUSE_CLIENT -q "drop table test" + +$CLICKHOUSE_CLIENT -q "create table test (x UInt64, y UInt64 default 42) engine=File(Native, '02562_data.native') settings input_format_defaults_for_omitted_fields=1" +$CLICKHOUSE_CLIENT -q "select * from test" +$CLICKHOUSE_CLIENT -q "drop table test" + +$CLICKHOUSE_CLIENT -q "insert into function file(02562_data.tskv) select 1::UInt64 as x settings engine_file_truncate_on_insert=1" +$CLICKHOUSE_CLIENT -q "create table test (x UInt64, y UInt64 default 42) engine=File(TSKV, '02562_data.tskv') settings input_format_defaults_for_omitted_fields=0" +$CLICKHOUSE_CLIENT -q "select * from test" +$CLICKHOUSE_CLIENT -q "drop table test" + +$CLICKHOUSE_CLIENT -q "create table test (x UInt64, y UInt64 default 42) engine=File(TSKV, '02562_data.tskv') settings input_format_defaults_for_omitted_fields=1" +$CLICKHOUSE_CLIENT -q "select * from test" +$CLICKHOUSE_CLIENT -q "drop table test" + From c68ef743b0a073bce75ddd649f1b85cc7411cf95 Mon Sep 17 00:00:00 2001 From: alesapin Date: Sat, 11 Feb 2023 12:55:05 +0100 Subject: [PATCH 155/717] Fix flaky test --- tests/queries/0_stateless/02555_davengers_rename_chain.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/02555_davengers_rename_chain.sh b/tests/queries/0_stateless/02555_davengers_rename_chain.sh index f4af5e091ad..b23f8085fd7 100755 --- a/tests/queries/0_stateless/02555_davengers_rename_chain.sh +++ b/tests/queries/0_stateless/02555_davengers_rename_chain.sh @@ -27,8 +27,8 @@ counter=0 retries=60 I=0 while [[ $counter -lt $retries ]]; do I=$((I + 1)) - result=$($CLICKHOUSE_CLIENT --query "SELECT * FROM system.mutations WHERE table = 'wrong_metadata' AND database='${CLICKHOUSE_DATABASE}'") - if [[ $result == *"a TO a1"* ]]; then + result=$($CLICKHOUSE_CLIENT --query "SHOW CREATE TABLE wrong_metadata") + if [[ $result == *"\`a1\` UInt64"* ]]; then break; fi sleep 0.1 @@ -98,8 +98,8 @@ counter=0 retries=60 I=0 while [[ $counter -lt $retries ]]; do I=$((I + 1)) - result=$($CLICKHOUSE_CLIENT --query "SELECT * FROM system.mutations WHERE table = 'wrong_metadata_compact' AND database='${CLICKHOUSE_DATABASE}'") - if [[ $result == *"a TO a1"* ]]; then + result=$($CLICKHOUSE_CLIENT --query "SHOW CREATE TABLE wrong_metadata_compact") + if [[ $result == *"\`a1\` UInt64"* ]]; then break; fi sleep 0.1 From 3ae526d4ba9f1d78ce735f3156b7f48e704a082f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 12 Feb 2023 04:57:53 +0100 Subject: [PATCH 156/717] Enable retries for INSERT by default in case of ZooKeeper session loss --- src/Core/Settings.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 2e0a4f00c64..fd9eba94ec2 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -697,7 +697,7 @@ class IColumn; M(UInt64, grace_hash_join_max_buckets, 1024, "Limit on the number of grace hash join buckets", 0) \ M(Bool, optimize_distinct_in_order, true, "Enable DISTINCT optimization if some columns in DISTINCT form a prefix of sorting. For example, prefix of sorting key in merge tree or ORDER BY statement", 0) \ M(Bool, optimize_sorting_by_input_stream_properties, true, "Optimize sorting by sorting properties of input stream", 0) \ - M(UInt64, insert_keeper_max_retries, 0, "Max retries for keeper operations during insert", 0) \ + M(UInt64, insert_keeper_max_retries, 20, "Max retries for keeper operations during insert", 0) \ M(UInt64, insert_keeper_retry_initial_backoff_ms, 100, "Initial backoff timeout for keeper operations during insert", 0) \ M(UInt64, insert_keeper_retry_max_backoff_ms, 10000, "Max backoff timeout for keeper operations during insert", 0) \ M(Float, insert_keeper_fault_injection_probability, 0.0f, "Approximate probability of failure for a keeper request during insert. Valid value is in interval [0.0f, 1.0f]", 0) \ From 45ef2f6d60af170c070fde363bdfc9d0997bff48 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Tue, 7 Feb 2023 20:45:40 +0100 Subject: [PATCH 157/717] Split prewhere actions into separate conjuctive steps --- src/Interpreters/ActionsDAG.cpp | 23 +++++---- src/Interpreters/ActionsDAG.h | 14 ++++++ .../MergeTreeBaseSelectProcessor.cpp | 49 +++++++++++++++++++ .../MergeTree/MergeTreeBaseSelectProcessor.h | 2 + .../MergeTree/MergeTreeBlockReadUtils.cpp | 40 ++++++++++++++- .../MergeTree/MergeTreeWhereOptimizer.cpp | 4 +- 6 files changed, 118 insertions(+), 14 deletions(-) diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index 5f1398fed39..913a891c6bb 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -1017,6 +1017,9 @@ std::string ActionsDAG::dumpDAG() const out << ' ' << map[node]; out << '\n'; + out << "Project input: " << project_input << '\n'; + out << "Projected output: " << projected_output << '\n'; + return out.str(); } @@ -1660,20 +1663,20 @@ ActionsDAG::SplitResult ActionsDAG::splitActionsForFilter(const std::string & co return res; } -namespace -{ - -struct ConjunctionNodes -{ - ActionsDAG::NodeRawConstPtrs allowed; - ActionsDAG::NodeRawConstPtrs rejected; -}; +//namespace +//{ +// +//struct ConjunctionNodes +//{ +// ActionsDAG::NodeRawConstPtrs allowed; +// ActionsDAG::NodeRawConstPtrs rejected; +//}; /// Take a node which result is predicate. /// Assuming predicate is a conjunction (probably, trivial). /// Find separate conjunctions nodes. Split nodes into allowed and rejected sets. /// Allowed predicate is a predicate which can be calculated using only nodes from allowed_nodes set. -ConjunctionNodes getConjunctionNodes(ActionsDAG::Node * predicate, std::unordered_set allowed_nodes) +ConjunctionNodes getConjunctionNodes(const ActionsDAG::Node * predicate, std::unordered_set allowed_nodes) { ConjunctionNodes conjunction; std::unordered_set allowed; @@ -1795,7 +1798,7 @@ ColumnsWithTypeAndName prepareFunctionArguments(const ActionsDAG::NodeRawConstPt return arguments; } -} +//} /// Create actions which calculate conjunction of selected nodes. /// Assume conjunction nodes are predicates (and may be used as arguments of function AND). diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index 40bc76fe057..98c5d36c69a 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -363,6 +363,7 @@ private: void compileFunctions(size_t min_count_to_compile_expression, const std::unordered_set & lazy_executed_nodes = {}); #endif +public: static ActionsDAGPtr cloneActionsForConjunction(NodeRawConstPtrs conjunction, const ColumnsWithTypeAndName & all_inputs); }; @@ -372,4 +373,17 @@ struct ActionDAGNodes ActionsDAG::NodeRawConstPtrs nodes; }; +struct ConjunctionNodes +{ + ActionsDAG::NodeRawConstPtrs allowed; + ActionsDAG::NodeRawConstPtrs rejected; +}; + +/// Take a node which result is predicate. +/// Assuming predicate is a conjunction (probably, trivial). +/// Find separate conjunctions nodes. Split nodes into allowed and rejected sets. +/// Allowed predicate is a predicate which can be calculated using only nodes from allowed_nodes set. +ConjunctionNodes getConjunctionNodes(const ActionsDAG::Node * predicate, std::unordered_set allowed_nodes); + + } diff --git a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp index 49458be4232..072bf5c2cba 100644 --- a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp @@ -7,6 +7,7 @@ #include #include #include +#include "Core/Names.h" #include #include #include @@ -102,6 +103,53 @@ std::unique_ptr IMergeTreeSelectAlgorithm::getPrewhereActions( prewhere_actions->steps.emplace_back(std::move(row_level_filter_step)); } +#if 1 + auto conjunctions = getConjunctionNodes( + prewhere_info->prewhere_actions->tryFindInOutputs(prewhere_info->prewhere_column_name), + {}); + + auto original_outputs = prewhere_info->prewhere_actions->getOutputs(); + NameSet original_output_names; + for (const auto & output : original_outputs) + original_output_names.insert(output->result_name); + + auto inputs = prewhere_info->prewhere_actions->getInputs(); + ColumnsWithTypeAndName all_inputs; + for (const auto & input : inputs) + all_inputs.emplace_back(input->column, input->result_type, input->result_name); + + ActionsDAG::NodeRawConstPtrs all_conjunctions = std::move(conjunctions.allowed); + all_conjunctions.insert(all_conjunctions.end(), conjunctions.rejected.begin(), conjunctions.rejected.end()); + + for (const auto & conjunction : all_conjunctions) + { + auto step_dag = ActionsDAG::cloneActionsForConjunction({conjunction}, all_inputs); + + /// Return the condition columns + Names step_outputs{conjunction->result_name}; + /// Preserve all the original outputs computed at this step + for (const auto & output : original_output_names) + if (step_dag->tryRestoreColumn(output)) + step_outputs.emplace_back(output); + step_dag->removeUnusedActions(step_outputs, true, true); + + //std::cerr << conjunction->result_name << "\n"; + std::cerr << step_dag->dumpDAG() << "\n"; + + PrewhereExprStep prewhere_step + { + .actions = std::make_shared(step_dag, actions_settings), + .column_name = conjunction->result_name, + .remove_column = false, // TODO: properly set this depending on whether the column is used in the next step + .need_filter = false + }; + prewhere_actions->steps.emplace_back(std::move(prewhere_step)); + } + + //prewhere_actions->steps.back().remove_column = prewhere_info->remove_prewhere_column; + prewhere_actions->steps.back().need_filter = prewhere_info->need_filter; +#else + PrewhereExprStep prewhere_step { .actions = std::make_shared(prewhere_info->prewhere_actions, actions_settings), @@ -111,6 +159,7 @@ std::unique_ptr IMergeTreeSelectAlgorithm::getPrewhereActions( }; prewhere_actions->steps.emplace_back(std::move(prewhere_step)); +#endif } return prewhere_actions; diff --git a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.h b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.h index c6680676ce9..0dc6f2f9f9b 100644 --- a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.h +++ b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.h @@ -101,8 +101,10 @@ protected: static void injectVirtualColumns(Block & block, size_t row_count, MergeTreeReadTask * task, const DataTypePtr & partition_value_type, const Names & virtual_columns); +public: static std::unique_ptr getPrewhereActions(PrewhereInfoPtr prewhere_info, const ExpressionActionsSettings & actions_settings); +protected: static void initializeRangeReadersImpl( MergeTreeRangeReader & range_reader, std::deque & pre_range_readers, diff --git a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp index 1f69fcae8dc..d71e7b276ab 100644 --- a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp +++ b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp @@ -5,6 +5,7 @@ #include #include #include +#include "Storages/MergeTree/MergeTreeBaseSelectProcessor.h" #include #include #include @@ -291,7 +292,6 @@ MergeTreeReadTaskColumns getReadTaskColumns( bool with_subcolumns) { Names column_names = required_columns; - Names pre_column_names; /// Read system columns such as lightweight delete mask "_row_exists" if it is persisted in the part for (const auto & name : system_columns) @@ -313,6 +313,40 @@ MergeTreeReadTaskColumns getReadTaskColumns( if (prewhere_info) { + auto prewhere_actions = IMergeTreeSelectAlgorithm::getPrewhereActions(prewhere_info, {}); + + NameSet pre_name_set; + + for (const auto & step : prewhere_actions->steps) + { + Names step_column_names = step.actions->getActionsDAG().getRequiredColumnsNames(); + + injectRequiredColumns( + data_part_info_for_reader, storage_snapshot, with_subcolumns, step_column_names); + + Names new_step_column_names; + for (const auto & name : step_column_names) + { + if (pre_name_set.contains(name)) + continue; + new_step_column_names.push_back(name); + pre_name_set.insert(name); + } + + result.pre_columns.push_back(storage_snapshot->getColumnsByNames(options, new_step_column_names)); + } + + /// Remove prewhere columns from the list of columns to read + Names post_column_names; + for (const auto & name : column_names) + if (!pre_name_set.contains(name)) + post_column_names.push_back(name); + + column_names = post_column_names; + + + +#if 0 NameSet pre_name_set; /// Add column reading steps: @@ -346,9 +380,11 @@ MergeTreeReadTaskColumns getReadTaskColumns( post_column_names.push_back(name); column_names = post_column_names; +#endif + } - result.pre_columns.push_back(storage_snapshot->getColumnsByNames(options, pre_column_names)); +// result.pre_columns.push_back(storage_snapshot->getColumnsByNames(options, pre_column_names)); /// 3. Rest of the requested columns result.columns = storage_snapshot->getColumnsByNames(options, column_names); diff --git a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp index b3ff05a960a..ce73dad48f9 100644 --- a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp +++ b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp @@ -274,7 +274,7 @@ void MergeTreeWhereOptimizer::optimize(ASTSelectQuery & select) const if (!it->viable) break; - +#if 0 bool moved_enough = false; if (total_size_of_queried_columns > 0) { @@ -292,7 +292,7 @@ void MergeTreeWhereOptimizer::optimize(ASTSelectQuery & select) const if (moved_enough) break; - +#endif move_condition(it); } From 249e670c9fd0b0cc4a0a3b7ddbd58c4c93a09c57 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Wed, 8 Feb 2023 00:07:19 +0100 Subject: [PATCH 158/717] Properly preserve columns that are required by next steps --- .../MergeTreeBaseSelectProcessor.cpp | 44 +++++++++++++++---- .../MergeTree/MergeTreeBlockReadUtils.cpp | 2 - 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp index 072bf5c2cba..43c4666f1b9 100644 --- a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp @@ -8,6 +8,7 @@ #include #include #include "Core/Names.h" +#include "Interpreters/ActionsDAG.h" #include #include #include @@ -121,32 +122,57 @@ std::unique_ptr IMergeTreeSelectAlgorithm::getPrewhereActions( ActionsDAG::NodeRawConstPtrs all_conjunctions = std::move(conjunctions.allowed); all_conjunctions.insert(all_conjunctions.end(), conjunctions.rejected.begin(), conjunctions.rejected.end()); + struct Step + { + ActionsDAGPtr actions; + String column_name; + }; + std::vector steps; + for (const auto & conjunction : all_conjunctions) { auto step_dag = ActionsDAG::cloneActionsForConjunction({conjunction}, all_inputs); + step_dag->removeUnusedActions(Names{conjunction->result_name}, true, true); + steps.emplace_back(Step{step_dag, conjunction->result_name}); + } + + /// "Rename" the last step result to the combined prewhere column name, because in fact it will be AND of all step results + if (steps.back().column_name != prewhere_info->prewhere_column_name) + steps.back().actions->addAlias(steps.back().actions->findInOutputs(steps.back().column_name), prewhere_info->prewhere_column_name); + + prewhere_actions->steps.resize(steps.size()); + + for (ssize_t i = steps.size() - 1; i >= 0; --i) + { + const auto & step = steps[i]; /// Return the condition columns - Names step_outputs{conjunction->result_name}; + Names step_outputs{step.column_name}; + const bool remove_column = !original_output_names.contains(step.column_name); /// Preserve all the original outputs computed at this step for (const auto & output : original_output_names) - if (step_dag->tryRestoreColumn(output)) + if (step.actions->tryRestoreColumn(output)) step_outputs.emplace_back(output); - step_dag->removeUnusedActions(step_outputs, true, true); + step.actions->removeUnusedActions(step_outputs, true, true); + + /// Add current step columns as outputs for previous steps + for (const auto & input :step.actions->getInputs()) + original_output_names.insert(input->result_name); //std::cerr << conjunction->result_name << "\n"; - std::cerr << step_dag->dumpDAG() << "\n"; + //std::cerr << step.actions->dumpDAG() << "\n"; PrewhereExprStep prewhere_step { - .actions = std::make_shared(step_dag, actions_settings), - .column_name = conjunction->result_name, - .remove_column = false, // TODO: properly set this depending on whether the column is used in the next step + .actions = std::make_shared(step.actions, actions_settings), + .column_name = step.column_name, + .remove_column = remove_column, // TODO: properly set this depending on whether the column is used in the next step .need_filter = false }; - prewhere_actions->steps.emplace_back(std::move(prewhere_step)); + prewhere_actions->steps[i] = std::move(prewhere_step); } - //prewhere_actions->steps.back().remove_column = prewhere_info->remove_prewhere_column; + prewhere_actions->steps.back().remove_column = prewhere_info->remove_prewhere_column; prewhere_actions->steps.back().need_filter = prewhere_info->need_filter; #else diff --git a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp index d71e7b276ab..c58ab06f08f 100644 --- a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp +++ b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp @@ -344,8 +344,6 @@ MergeTreeReadTaskColumns getReadTaskColumns( column_names = post_column_names; - - #if 0 NameSet pre_name_set; From 1b207d9fa099f97cf7b1dcfd637af28430f2db35 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Wed, 8 Feb 2023 14:29:12 +0100 Subject: [PATCH 159/717] Fix for overwritten row level filter step --- src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp index 43c4666f1b9..56929809b42 100644 --- a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp @@ -140,7 +140,8 @@ std::unique_ptr IMergeTreeSelectAlgorithm::getPrewhereActions( if (steps.back().column_name != prewhere_info->prewhere_column_name) steps.back().actions->addAlias(steps.back().actions->findInOutputs(steps.back().column_name), prewhere_info->prewhere_column_name); - prewhere_actions->steps.resize(steps.size()); + const size_t steps_before_prewhere = prewhere_actions->steps.size(); + prewhere_actions->steps.resize(steps_before_prewhere + steps.size()); for (ssize_t i = steps.size() - 1; i >= 0; --i) { @@ -169,7 +170,7 @@ std::unique_ptr IMergeTreeSelectAlgorithm::getPrewhereActions( .remove_column = remove_column, // TODO: properly set this depending on whether the column is used in the next step .need_filter = false }; - prewhere_actions->steps[i] = std::move(prewhere_step); + prewhere_actions->steps[steps_before_prewhere + i] = std::move(prewhere_step); } prewhere_actions->steps.back().remove_column = prewhere_info->remove_prewhere_column; From 31f5dbe2a3e88e7c530c3284264df995a57a0f9c Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Wed, 8 Feb 2023 17:07:23 +0100 Subject: [PATCH 160/717] Cast intermediate step result to Bool if needed --- .../MergeTree/MergeTreeBaseSelectProcessor.cpp | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp index 56929809b42..2aa23243785 100644 --- a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp @@ -131,9 +131,18 @@ std::unique_ptr IMergeTreeSelectAlgorithm::getPrewhereActions( for (const auto & conjunction : all_conjunctions) { + auto result_name = conjunction->result_name; auto step_dag = ActionsDAG::cloneActionsForConjunction({conjunction}, all_inputs); - step_dag->removeUnusedActions(Names{conjunction->result_name}, true, true); - steps.emplace_back(Step{step_dag, conjunction->result_name}); + const auto & result_node = step_dag->findInOutputs(result_name); + /// Cast to UInt8 if needed + if (result_node.result_type->getTypeId() != TypeIndex::UInt8) + { + const auto & cast_node = step_dag->addCast(result_node, std::make_shared()); + step_dag->addOrReplaceInOutputs(cast_node); + result_name = cast_node.result_name; + } + step_dag->removeUnusedActions(Names{result_name}, true, true); + steps.emplace_back(Step{step_dag, result_name}); } /// "Rename" the last step result to the combined prewhere column name, because in fact it will be AND of all step results From 120b112743ed786cb1a89db22342b9c9b59794fa Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Thu, 9 Feb 2023 12:56:59 +0100 Subject: [PATCH 161/717] Cleanup --- .../MergeTree/MergeTreeBlockReadUtils.cpp | 73 +++++-------------- 1 file changed, 17 insertions(+), 56 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp index c58ab06f08f..17ef9c91e78 100644 --- a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp +++ b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp @@ -291,18 +291,18 @@ MergeTreeReadTaskColumns getReadTaskColumns( const PrewhereInfoPtr & prewhere_info, bool with_subcolumns) { - Names column_names = required_columns; + Names column_to_read_after_prewhere = required_columns; /// Read system columns such as lightweight delete mask "_row_exists" if it is persisted in the part for (const auto & name : system_columns) { if (data_part_info_for_reader.getColumns().contains(name)) - column_names.push_back(name); + column_to_read_after_prewhere.push_back(name); } - /// inject columns required for defaults evaluation + /// Inject columns required for defaults evaluation injectRequiredColumns( - data_part_info_for_reader, storage_snapshot, with_subcolumns, column_names); + data_part_info_for_reader, storage_snapshot, with_subcolumns, column_to_read_after_prewhere); MergeTreeReadTaskColumns result; auto options = GetColumnsOptions(GetColumnsOptions::All) @@ -313,9 +313,9 @@ MergeTreeReadTaskColumns getReadTaskColumns( if (prewhere_info) { - auto prewhere_actions = IMergeTreeSelectAlgorithm::getPrewhereActions(prewhere_info, {}); + auto prewhere_actions = IMergeTreeSelectAlgorithm::getPrewhereActions(prewhere_info, {}); // TODO: pass proper actions_settings - NameSet pre_name_set; + NameSet columns_from_previous_steps; for (const auto & step : prewhere_actions->steps) { @@ -324,68 +324,29 @@ MergeTreeReadTaskColumns getReadTaskColumns( injectRequiredColumns( data_part_info_for_reader, storage_snapshot, with_subcolumns, step_column_names); - Names new_step_column_names; + Names columns_to_read_in_step; for (const auto & name : step_column_names) { - if (pre_name_set.contains(name)) + if (columns_from_previous_steps.contains(name)) continue; - new_step_column_names.push_back(name); - pre_name_set.insert(name); + columns_to_read_in_step.push_back(name); + columns_from_previous_steps.insert(name); } - result.pre_columns.push_back(storage_snapshot->getColumnsByNames(options, new_step_column_names)); + result.pre_columns.push_back(storage_snapshot->getColumnsByNames(options, columns_to_read_in_step)); } - /// Remove prewhere columns from the list of columns to read + /// Remove columns read in prewehere from the list of columns to read Names post_column_names; - for (const auto & name : column_names) - if (!pre_name_set.contains(name)) + for (const auto & name : column_to_read_after_prewhere) + if (!columns_from_previous_steps.contains(name)) post_column_names.push_back(name); - column_names = post_column_names; - -#if 0 - NameSet pre_name_set; - - /// Add column reading steps: - /// 1. Columns for row level filter - if (prewhere_info->row_level_filter) - { - Names row_filter_column_names = prewhere_info->row_level_filter->getRequiredColumnsNames(); - injectRequiredColumns( - data_part_info_for_reader, storage_snapshot, with_subcolumns, row_filter_column_names); - result.pre_columns.push_back(storage_snapshot->getColumnsByNames(options, row_filter_column_names)); - pre_name_set.insert(row_filter_column_names.begin(), row_filter_column_names.end()); - } - - /// 2. Columns for prewhere - Names all_pre_column_names = prewhere_info->prewhere_actions->getRequiredColumnsNames(); - - injectRequiredColumns( - data_part_info_for_reader, storage_snapshot, with_subcolumns, all_pre_column_names); - - for (const auto & name : all_pre_column_names) - { - if (pre_name_set.contains(name)) - continue; - pre_column_names.push_back(name); - pre_name_set.insert(name); - } - - Names post_column_names; - for (const auto & name : column_names) - if (!pre_name_set.contains(name)) - post_column_names.push_back(name); - - column_names = post_column_names; -#endif - + column_to_read_after_prewhere = std::move(post_column_names); } -// result.pre_columns.push_back(storage_snapshot->getColumnsByNames(options, pre_column_names)); - - /// 3. Rest of the requested columns - result.columns = storage_snapshot->getColumnsByNames(options, column_names); + /// Rest of the requested columns + result.columns = storage_snapshot->getColumnsByNames(options, column_to_read_after_prewhere); return result; } From 7634b9f56effe0b716382f0af974677cda88b520 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Thu, 9 Feb 2023 15:37:06 +0100 Subject: [PATCH 162/717] Added settings to enable the new logic --- src/Core/Settings.h | 2 + .../QueryPlan/ReadFromMergeTree.cpp | 3 + .../MergeTreeBaseSelectProcessor.cpp | 172 +++++++++--------- .../MergeTree/MergeTreeBaseSelectProcessor.h | 2 +- .../MergeTree/MergeTreeBlockReadUtils.cpp | 3 +- .../MergeTree/MergeTreeBlockReadUtils.h | 2 + src/Storages/MergeTree/MergeTreeIOSettings.h | 2 + src/Storages/MergeTree/MergeTreeReadPool.cpp | 4 +- src/Storages/MergeTree/MergeTreeReadPool.h | 6 + .../MergeTree/MergeTreeSelectProcessor.cpp | 2 +- .../MergeTree/MergeTreeWhereOptimizer.cpp | 37 ++-- .../MergeTree/MergeTreeWhereOptimizer.h | 1 + 12 files changed, 132 insertions(+), 104 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 2e0a4f00c64..52cc5eb7ad0 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -114,6 +114,8 @@ class IColumn; \ M(Bool, optimize_move_to_prewhere, true, "Allows disabling WHERE to PREWHERE optimization in SELECT queries from MergeTree.", 0) \ M(Bool, optimize_move_to_prewhere_if_final, false, "If query has `FINAL`, the optimization `move_to_prewhere` is not always correct and it is enabled only if both settings `optimize_move_to_prewhere` and `optimize_move_to_prewhere_if_final` are turned on", 0) \ + M(Bool, move_all_conditions_to_prewhere, false, "Move all viable conditions from WHERE to PREWHERE", 0) \ + M(Bool, enable_multiple_prewhere_read_steps, false, "Move more conditions from WHERE to PREWHERE and do reads from disk and filtering in multiple steps if there are multiple conditions combined with AND", 0) \ \ M(UInt64, replication_alter_partitions_sync, 1, "Wait for actions to manipulate the partitions. 0 - do not wait, 1 - wait for execution only of itself, 2 - wait for everyone.", 0) \ M(Int64, replication_wait_for_inactive_replica_timeout, 120, "Wait for inactive replica to execute ALTER/OPTIMIZE. Time in seconds, 0 - do not wait, negative - wait for unlimited time.", 0) \ diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index ff0c5002e09..6ff230e9d78 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -73,6 +73,7 @@ static MergeTreeReaderSettings getMergeTreeReaderSettings( .read_in_order = query_info.input_order_info != nullptr, .use_asynchronous_read_from_pool = settings.allow_asynchronous_read_from_io_pool_for_merge_tree && (settings.max_streams_to_max_threads_ratio > 1 || settings.max_streams_for_merge_tree_reading > 1), + .enable_multiple_prewhere_read_steps = settings.enable_multiple_prewhere_read_steps, }; } @@ -225,6 +226,7 @@ Pipe ReadFromMergeTree::readFromPoolParallelReplicas( extension, parts_with_range, prewhere_info, + reader_settings, required_columns, virt_column_names, min_marks_for_concurrent_read @@ -302,6 +304,7 @@ Pipe ReadFromMergeTree::readFromPool( std::move(parts_with_range), storage_snapshot, prewhere_info, + reader_settings, required_columns, virt_column_names, backoff_settings, diff --git a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp index 2aa23243785..49022dce828 100644 --- a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp @@ -59,7 +59,7 @@ IMergeTreeSelectAlgorithm::IMergeTreeSelectAlgorithm( : storage(storage_) , storage_snapshot(storage_snapshot_) , prewhere_info(prewhere_info_) - , prewhere_actions(getPrewhereActions(prewhere_info, actions_settings)) + , prewhere_actions(getPrewhereActions(prewhere_info, actions_settings, reader_settings_.enable_multiple_prewhere_read_steps)) , max_block_size_rows(max_block_size_rows_) , preferred_block_size_bytes(preferred_block_size_bytes_) , preferred_max_column_in_block_size_bytes(preferred_max_column_in_block_size_bytes_) @@ -84,7 +84,7 @@ IMergeTreeSelectAlgorithm::IMergeTreeSelectAlgorithm( } -std::unique_ptr IMergeTreeSelectAlgorithm::getPrewhereActions(PrewhereInfoPtr prewhere_info, const ExpressionActionsSettings & actions_settings) +std::unique_ptr IMergeTreeSelectAlgorithm::getPrewhereActions(PrewhereInfoPtr prewhere_info, const ExpressionActionsSettings & actions_settings, bool enable_multiple_prewhere_read_steps) { std::unique_ptr prewhere_actions; if (prewhere_info) @@ -104,98 +104,102 @@ std::unique_ptr IMergeTreeSelectAlgorithm::getPrewhereActions( prewhere_actions->steps.emplace_back(std::move(row_level_filter_step)); } -#if 1 - auto conjunctions = getConjunctionNodes( - prewhere_info->prewhere_actions->tryFindInOutputs(prewhere_info->prewhere_column_name), - {}); - - auto original_outputs = prewhere_info->prewhere_actions->getOutputs(); - NameSet original_output_names; - for (const auto & output : original_outputs) - original_output_names.insert(output->result_name); - - auto inputs = prewhere_info->prewhere_actions->getInputs(); - ColumnsWithTypeAndName all_inputs; - for (const auto & input : inputs) - all_inputs.emplace_back(input->column, input->result_type, input->result_name); - - ActionsDAG::NodeRawConstPtrs all_conjunctions = std::move(conjunctions.allowed); - all_conjunctions.insert(all_conjunctions.end(), conjunctions.rejected.begin(), conjunctions.rejected.end()); - - struct Step + if (enable_multiple_prewhere_read_steps) { - ActionsDAGPtr actions; - String column_name; - }; - std::vector steps; + auto conjunctions = getConjunctionNodes( + prewhere_info->prewhere_actions->tryFindInOutputs(prewhere_info->prewhere_column_name), + {}); - for (const auto & conjunction : all_conjunctions) - { - auto result_name = conjunction->result_name; - auto step_dag = ActionsDAG::cloneActionsForConjunction({conjunction}, all_inputs); - const auto & result_node = step_dag->findInOutputs(result_name); - /// Cast to UInt8 if needed - if (result_node.result_type->getTypeId() != TypeIndex::UInt8) + auto original_outputs = prewhere_info->prewhere_actions->getOutputs(); + NameSet outputs_required_by_next_steps; + for (const auto & output : original_outputs) + outputs_required_by_next_steps.insert(output->result_name); + + auto inputs = prewhere_info->prewhere_actions->getInputs(); + ColumnsWithTypeAndName all_inputs; + for (const auto & input : inputs) + all_inputs.emplace_back(input->column, input->result_type, input->result_name); + + ActionsDAG::NodeRawConstPtrs all_conjunctions = std::move(conjunctions.allowed); + all_conjunctions.insert(all_conjunctions.end(), conjunctions.rejected.begin(), conjunctions.rejected.end()); + + struct Step { - const auto & cast_node = step_dag->addCast(result_node, std::make_shared()); - step_dag->addOrReplaceInOutputs(cast_node); - result_name = cast_node.result_name; + ActionsDAGPtr actions; + String column_name; + }; + std::vector steps; + + for (const auto & conjunction : all_conjunctions) + { + auto result_name = conjunction->result_name; + auto step_dag = ActionsDAG::cloneActionsForConjunction({conjunction}, all_inputs); + const auto & result_node = step_dag->findInOutputs(result_name); + /// Cast to UInt8 if needed + if (result_node.result_type->getTypeId() != TypeIndex::UInt8) + { + const auto & cast_node = step_dag->addCast(result_node, std::make_shared()); + step_dag->addOrReplaceInOutputs(cast_node); + result_name = cast_node.result_name; + } + step_dag->removeUnusedActions(Names{result_name}, true, true); + steps.emplace_back(Step{step_dag, result_name}); } - step_dag->removeUnusedActions(Names{result_name}, true, true); - steps.emplace_back(Step{step_dag, result_name}); + + /// "Rename" the last step result to the combined prewhere column name, because in fact it will be AND of all step results + if (steps.back().column_name != prewhere_info->prewhere_column_name) + steps.back().actions->addAlias(steps.back().actions->findInOutputs(steps.back().column_name), prewhere_info->prewhere_column_name); + + const size_t steps_before_prewhere = prewhere_actions->steps.size(); + prewhere_actions->steps.resize(steps_before_prewhere + steps.size()); + + /// Check the steps in the reverse order so that we can maintain the list of outputs used by the next steps + /// and preserve them in the current step. + for (ssize_t i = steps.size() - 1; i >= 0; --i) + { + const auto & step = steps[i]; + + /// Return the condition column + Names step_outputs{step.column_name}; + const bool remove_column = !outputs_required_by_next_steps.contains(step.column_name); + /// Preserve outputs computed at this step that are used by the next steps + for (const auto & output : outputs_required_by_next_steps) + if (step.actions->tryRestoreColumn(output)) + step_outputs.emplace_back(output); + step.actions->removeUnusedActions(step_outputs, true, true); + + /// Add current step columns as outputs that should be preserved from previous steps + for (const auto & input :step.actions->getInputs()) + outputs_required_by_next_steps.insert(input->result_name); + + //std::cerr << conjunction->result_name << "\n"; + //std::cerr << step.actions->dumpDAG() << "\n"; + + PrewhereExprStep prewhere_step + { + .actions = std::make_shared(step.actions, actions_settings), + .column_name = step.column_name, + .remove_column = remove_column, + .need_filter = false + }; + prewhere_actions->steps[steps_before_prewhere + i] = std::move(prewhere_step); + } + + prewhere_actions->steps.back().remove_column = prewhere_info->remove_prewhere_column; + prewhere_actions->steps.back().need_filter = prewhere_info->need_filter; } - - /// "Rename" the last step result to the combined prewhere column name, because in fact it will be AND of all step results - if (steps.back().column_name != prewhere_info->prewhere_column_name) - steps.back().actions->addAlias(steps.back().actions->findInOutputs(steps.back().column_name), prewhere_info->prewhere_column_name); - - const size_t steps_before_prewhere = prewhere_actions->steps.size(); - prewhere_actions->steps.resize(steps_before_prewhere + steps.size()); - - for (ssize_t i = steps.size() - 1; i >= 0; --i) + else { - const auto & step = steps[i]; - - /// Return the condition columns - Names step_outputs{step.column_name}; - const bool remove_column = !original_output_names.contains(step.column_name); - /// Preserve all the original outputs computed at this step - for (const auto & output : original_output_names) - if (step.actions->tryRestoreColumn(output)) - step_outputs.emplace_back(output); - step.actions->removeUnusedActions(step_outputs, true, true); - - /// Add current step columns as outputs for previous steps - for (const auto & input :step.actions->getInputs()) - original_output_names.insert(input->result_name); - - //std::cerr << conjunction->result_name << "\n"; - //std::cerr << step.actions->dumpDAG() << "\n"; - PrewhereExprStep prewhere_step { - .actions = std::make_shared(step.actions, actions_settings), - .column_name = step.column_name, - .remove_column = remove_column, // TODO: properly set this depending on whether the column is used in the next step - .need_filter = false + .actions = std::make_shared(prewhere_info->prewhere_actions, actions_settings), + .column_name = prewhere_info->prewhere_column_name, + .remove_column = prewhere_info->remove_prewhere_column, + .need_filter = prewhere_info->need_filter }; - prewhere_actions->steps[steps_before_prewhere + i] = std::move(prewhere_step); + + prewhere_actions->steps.emplace_back(std::move(prewhere_step)); } - - prewhere_actions->steps.back().remove_column = prewhere_info->remove_prewhere_column; - prewhere_actions->steps.back().need_filter = prewhere_info->need_filter; -#else - - PrewhereExprStep prewhere_step - { - .actions = std::make_shared(prewhere_info->prewhere_actions, actions_settings), - .column_name = prewhere_info->prewhere_column_name, - .remove_column = prewhere_info->remove_prewhere_column, - .need_filter = prewhere_info->need_filter - }; - - prewhere_actions->steps.emplace_back(std::move(prewhere_step)); -#endif } return prewhere_actions; diff --git a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.h b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.h index 0dc6f2f9f9b..e66e3d8a741 100644 --- a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.h +++ b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.h @@ -102,7 +102,7 @@ protected: injectVirtualColumns(Block & block, size_t row_count, MergeTreeReadTask * task, const DataTypePtr & partition_value_type, const Names & virtual_columns); public: - static std::unique_ptr getPrewhereActions(PrewhereInfoPtr prewhere_info, const ExpressionActionsSettings & actions_settings); + static std::unique_ptr getPrewhereActions(PrewhereInfoPtr prewhere_info, const ExpressionActionsSettings & actions_settings, bool enable_multiple_prewhere_read_steps = true); protected: static void initializeRangeReadersImpl( diff --git a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp index 17ef9c91e78..7fd063826e7 100644 --- a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp +++ b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp @@ -289,6 +289,7 @@ MergeTreeReadTaskColumns getReadTaskColumns( const Names & required_columns, const Names & system_columns, const PrewhereInfoPtr & prewhere_info, + const MergeTreeReaderSettings & reader_settings, bool with_subcolumns) { Names column_to_read_after_prewhere = required_columns; @@ -313,7 +314,7 @@ MergeTreeReadTaskColumns getReadTaskColumns( if (prewhere_info) { - auto prewhere_actions = IMergeTreeSelectAlgorithm::getPrewhereActions(prewhere_info, {}); // TODO: pass proper actions_settings + auto prewhere_actions = IMergeTreeSelectAlgorithm::getPrewhereActions(prewhere_info, {}, reader_settings.enable_multiple_prewhere_read_steps); // TODO: pass proper actions_settings NameSet columns_from_previous_steps; diff --git a/src/Storages/MergeTree/MergeTreeBlockReadUtils.h b/src/Storages/MergeTree/MergeTreeBlockReadUtils.h index 91c895c197e..7b654a05283 100644 --- a/src/Storages/MergeTree/MergeTreeBlockReadUtils.h +++ b/src/Storages/MergeTree/MergeTreeBlockReadUtils.h @@ -13,6 +13,7 @@ namespace DB class MergeTreeData; struct MergeTreeReadTask; +struct MergeTreeReaderSettings; struct MergeTreeBlockSizePredictor; class IMergeTreeDataPartInfoForReader; @@ -100,6 +101,7 @@ MergeTreeReadTaskColumns getReadTaskColumns( const Names & required_columns, const Names & system_columns, const PrewhereInfoPtr & prewhere_info, + const MergeTreeReaderSettings & reader_settings, bool with_subcolumns); struct MergeTreeBlockSizePredictor diff --git a/src/Storages/MergeTree/MergeTreeIOSettings.h b/src/Storages/MergeTree/MergeTreeIOSettings.h index 65a8efebb8e..dbe5d893444 100644 --- a/src/Storages/MergeTree/MergeTreeIOSettings.h +++ b/src/Storages/MergeTree/MergeTreeIOSettings.h @@ -29,6 +29,8 @@ struct MergeTreeReaderSettings bool apply_deleted_mask = true; /// Put reading task in a common I/O pool, return Async state on prepare() bool use_asynchronous_read_from_pool = false; + /// If PREWHERE has multiple conditions combined with AND, execute them in separate read/filtering steps. + bool enable_multiple_prewhere_read_steps = false; }; struct MergeTreeWriterSettings diff --git a/src/Storages/MergeTree/MergeTreeReadPool.cpp b/src/Storages/MergeTree/MergeTreeReadPool.cpp index ebb87be0c41..54cc44b6503 100644 --- a/src/Storages/MergeTree/MergeTreeReadPool.cpp +++ b/src/Storages/MergeTree/MergeTreeReadPool.cpp @@ -46,7 +46,7 @@ std::vector IMergeTreeReadPool::fillPerPartInfo(const RangesInDataParts auto task_columns = getReadTaskColumns( LoadedMergeTreeDataPartInfoForReader(part.data_part), storage_snapshot, - column_names, virtual_column_names, prewhere_info, /*with_subcolumns=*/ true); + column_names, virtual_column_names, prewhere_info, reader_settings, /*with_subcolumns=*/ true); auto size_predictor = !predict_block_size_bytes ? nullptr : IMergeTreeSelectAlgorithm::getSizePredictor(part.data_part, task_columns, sample_block); @@ -72,6 +72,7 @@ MergeTreeReadPool::MergeTreeReadPool( RangesInDataParts && parts_, const StorageSnapshotPtr & storage_snapshot_, const PrewhereInfoPtr & prewhere_info_, + const MergeTreeReaderSettings & reader_settings_, const Names & column_names_, const Names & virtual_column_names_, const BackoffSettings & backoff_settings_, @@ -83,6 +84,7 @@ MergeTreeReadPool::MergeTreeReadPool( virtual_column_names_, min_marks_for_concurrent_read_, prewhere_info_, + reader_settings_, std::move(parts_), (preferred_block_size_bytes_ > 0), do_not_steal_tasks_) diff --git a/src/Storages/MergeTree/MergeTreeReadPool.h b/src/Storages/MergeTree/MergeTreeReadPool.h index ad9d8b2e225..857aa37f783 100644 --- a/src/Storages/MergeTree/MergeTreeReadPool.h +++ b/src/Storages/MergeTree/MergeTreeReadPool.h @@ -27,6 +27,7 @@ public: Names virtual_column_names_, size_t min_marks_for_concurrent_read_, PrewhereInfoPtr prewhere_info_, + const MergeTreeReaderSettings & reader_settings_, RangesInDataParts parts_ranges_, bool predict_block_size_bytes_, bool do_not_steal_tasks_) @@ -35,6 +36,7 @@ public: , virtual_column_names(virtual_column_names_) , min_marks_for_concurrent_read(min_marks_for_concurrent_read_) , prewhere_info(prewhere_info_) + , reader_settings(reader_settings_) , parts_ranges(parts_ranges_) , predict_block_size_bytes(predict_block_size_bytes_) , do_not_steal_tasks(do_not_steal_tasks_) @@ -55,6 +57,7 @@ protected: const Names virtual_column_names; size_t min_marks_for_concurrent_read{0}; PrewhereInfoPtr prewhere_info; + MergeTreeReaderSettings reader_settings; RangesInDataParts parts_ranges; bool predict_block_size_bytes; bool do_not_steal_tasks; @@ -123,6 +126,7 @@ public: RangesInDataParts && parts_, const StorageSnapshotPtr & storage_snapshot_, const PrewhereInfoPtr & prewhere_info_, + const MergeTreeReaderSettings & reader_settings_, const Names & column_names_, const Names & virtual_column_names_, const BackoffSettings & backoff_settings_, @@ -197,6 +201,7 @@ public: ParallelReadingExtension extension_, const RangesInDataParts & parts_, const PrewhereInfoPtr & prewhere_info_, + const MergeTreeReaderSettings & reader_settings_, const Names & column_names_, const Names & virtual_column_names_, size_t min_marks_for_concurrent_read_ @@ -207,6 +212,7 @@ public: virtual_column_names_, min_marks_for_concurrent_read_, prewhere_info_, + reader_settings_, parts_, /*predict_block_size*/false, /*do_not_steal_tasks*/false) diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp index df2b0ea9e3d..7daccdb95d3 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp @@ -46,7 +46,7 @@ void MergeTreeSelectAlgorithm::initializeReaders() { task_columns = getReadTaskColumns( LoadedMergeTreeDataPartInfoForReader(data_part), storage_snapshot, - required_columns, virt_column_names, prewhere_info, /*with_subcolumns=*/ true); + required_columns, virt_column_names, prewhere_info, reader_settings, /*with_subcolumns=*/ true); /// Will be used to distinguish between PREWHERE and WHERE columns when applying filter const auto & column_names = task_columns.columns.getNames(); diff --git a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp index ce73dad48f9..5be1e514faa 100644 --- a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp +++ b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp @@ -11,6 +11,7 @@ #include #include #include +#include "Interpreters/Context_fwd.h" #include #include @@ -41,6 +42,7 @@ MergeTreeWhereOptimizer::MergeTreeWhereOptimizer( , block_with_constants{KeyCondition::getBlockWithConstants(query_info.query->clone(), query_info.syntax_analyzer_result, context)} , log{log_} , column_sizes{std::move(column_sizes_)} + , move_all_conditions_to_prewhere(context->getSettingsRef().move_all_conditions_to_prewhere) { const auto & primary_key = metadata_snapshot->getPrimaryKey(); if (!primary_key.column_names.empty()) @@ -274,25 +276,28 @@ void MergeTreeWhereOptimizer::optimize(ASTSelectQuery & select) const if (!it->viable) break; -#if 0 - bool moved_enough = false; - if (total_size_of_queried_columns > 0) + + if (!move_all_conditions_to_prewhere) { - /// If we know size of queried columns use it as threshold. 10% ratio is just a guess. - moved_enough = total_size_of_moved_conditions > 0 - && (total_size_of_moved_conditions + it->columns_size) * 10 > total_size_of_queried_columns; - } - else - { - /// Otherwise, use number of moved columns as a fallback. - /// It can happen, if table has only compact parts. 25% ratio is just a guess. - moved_enough = total_number_of_moved_columns > 0 - && (total_number_of_moved_columns + it->identifiers.size()) * 4 > queried_columns.size(); + bool moved_enough = false; + if (total_size_of_queried_columns > 0) + { + /// If we know size of queried columns use it as threshold. 10% ratio is just a guess. + moved_enough = total_size_of_moved_conditions > 0 + && (total_size_of_moved_conditions + it->columns_size) * 10 > total_size_of_queried_columns; + } + else + { + /// Otherwise, use number of moved columns as a fallback. + /// It can happen, if table has only compact parts. 25% ratio is just a guess. + moved_enough = total_number_of_moved_columns > 0 + && (total_number_of_moved_columns + it->identifiers.size()) * 4 > queried_columns.size(); + } + + if (moved_enough) + break; } - if (moved_enough) - break; -#endif move_condition(it); } diff --git a/src/Storages/MergeTree/MergeTreeWhereOptimizer.h b/src/Storages/MergeTree/MergeTreeWhereOptimizer.h index fa14fea94d1..b0aa35ea266 100644 --- a/src/Storages/MergeTree/MergeTreeWhereOptimizer.h +++ b/src/Storages/MergeTree/MergeTreeWhereOptimizer.h @@ -114,6 +114,7 @@ private: std::unordered_map column_sizes; UInt64 total_size_of_queried_columns = 0; NameSet array_joined_names; + const bool move_all_conditions_to_prewhere = false; }; From 9e656b5c0e130333b8a818e0689a89c40bcfba4d Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Thu, 9 Feb 2023 19:40:15 +0100 Subject: [PATCH 163/717] Added a test --- ..._multiple_read_steps_in_prewhere.reference | 31 +++++++++++++++++++ .../02559_multiple_read_steps_in_prewhere.sql | 21 +++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.reference create mode 100644 tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.sql diff --git a/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.reference b/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.reference new file mode 100644 index 00000000000..fbd143e9b55 --- /dev/null +++ b/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.reference @@ -0,0 +1,31 @@ +SELECT cast(id as UInt16) AS id16 FROM test_02559 PREWHERE id16 and (id % 40000) LIMIT 10; +1 +2 +3 +4 +5 +6 +7 +8 +9 +SELECT cast(id as UInt16) AS id16, (id % 40000) AS id40000, (id16 AND id40000) AS cond FROM test_02559 PREWHERE cond LIMIT 10; +1 1 1 +2 2 1 +3 3 1 +4 4 1 +5 5 1 +6 6 1 +7 7 1 +8 8 1 +9 9 1 +SELECT cast(id as UInt16) AS id16 FROM test_02559 PREWHERE id16 LIMIT 10; +1 +2 +3 +4 +5 +6 +7 +8 +9 +SELECT count() FROM test_02559 PREWHERE 1 OR ignore(id) WHERE ignore(id)=0; diff --git a/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.sql b/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.sql new file mode 100644 index 00000000000..68c61dda6dd --- /dev/null +++ b/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.sql @@ -0,0 +1,21 @@ +DROP TABLE IF EXISTS test_02559; + +CREATE TABLE test_02559 (id UInt64) ENGINE=MergeTree ORDER BY id; + +INSERT INTO test_02559 SELECT number FROM numbers(10); + +SET enable_multiple_prewhere_read_steps=true, move_all_conditions_to_prewhere=true; + +-- { echoOn } + +SELECT cast(id as UInt16) AS id16 FROM test_02559 PREWHERE id16 and (id % 40000) LIMIT 10; + +SELECT cast(id as UInt16) AS id16, (id % 40000) AS id40000, (id16 AND id40000) AS cond FROM test_02559 PREWHERE cond LIMIT 10; + +SELECT cast(id as UInt16) AS id16 FROM test_02559 PREWHERE id16 LIMIT 10; + +SELECT count() FROM test_02559 PREWHERE 1 OR ignore(id) WHERE ignore(id)=0; + +-- { echoOff } + +DROP TABLE test_02559; From e460d5361911cfc78442932034892030f34e0a2d Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Fri, 10 Feb 2023 15:08:15 +0100 Subject: [PATCH 164/717] Use different name when filter column is CASTed to UInt8 --- .../MergeTreeBaseSelectProcessor.cpp | 64 +++++++++++++++---- 1 file changed, 52 insertions(+), 12 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp index 49022dce828..c13cda6466a 100644 --- a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp @@ -7,13 +7,18 @@ #include #include #include -#include "Core/Names.h" -#include "Interpreters/ActionsDAG.h" #include #include #include #include #include + +#include + +/// For CAST to bool +#include +#include + #include namespace ProfileEvents @@ -104,17 +109,22 @@ std::unique_ptr IMergeTreeSelectAlgorithm::getPrewhereActions( prewhere_actions->steps.emplace_back(std::move(row_level_filter_step)); } + //std::cerr << "ORIGINAL PREWHERE:\n" << prewhere_info->prewhere_actions->dumpDAG() << std::endl; + if (enable_multiple_prewhere_read_steps) { + /// Find all conjunctions in prewhere expression. auto conjunctions = getConjunctionNodes( prewhere_info->prewhere_actions->tryFindInOutputs(prewhere_info->prewhere_column_name), {}); + /// Save the list of outputs from the original prewhere expression. auto original_outputs = prewhere_info->prewhere_actions->getOutputs(); - NameSet outputs_required_by_next_steps; + std::unordered_map outputs_required_by_next_steps; for (const auto & output : original_outputs) - outputs_required_by_next_steps.insert(output->result_name); + outputs_required_by_next_steps[output->result_name] = output->result_type; + /// Save the list of inputs to the original prewhere expression. auto inputs = prewhere_info->prewhere_actions->getInputs(); ColumnsWithTypeAndName all_inputs; for (const auto & input : inputs) @@ -130,15 +140,36 @@ std::unique_ptr IMergeTreeSelectAlgorithm::getPrewhereActions( }; std::vector steps; + /// Adds a CAST node with the regular name ("CAST(...)") or with the provided name. + /// This is different from ActionsDAG::addCast() becuase it set the name equal to the original name effectively hiding the value before cast, + /// but it might be required for further steps with its original uncasted type. + auto add_cast = [] (ActionsDAGPtr dag, const ActionsDAG::Node & node_to_cast, const String & type_name, const String & new_name = {}) -> const ActionsDAG::Node & + { + Field cast_type_constant_value(type_name); + + ColumnWithTypeAndName column; + column.name = calculateConstantActionNodeName(cast_type_constant_value); + column.column = DataTypeString().createColumnConst(0, cast_type_constant_value); + column.type = std::make_shared(); + + const auto * cast_type_constant_node = &dag->addColumn(std::move(column)); + ActionsDAG::NodeRawConstPtrs children = {&node_to_cast, cast_type_constant_node}; + FunctionOverloadResolverPtr func_builder_cast = CastInternalOverloadResolver::createImpl(); + + return dag->addFunction(func_builder_cast, std::move(children), new_name); + }; + + /// Make separate DAG for each step for (const auto & conjunction : all_conjunctions) { auto result_name = conjunction->result_name; auto step_dag = ActionsDAG::cloneActionsForConjunction({conjunction}, all_inputs); const auto & result_node = step_dag->findInOutputs(result_name); - /// Cast to UInt8 if needed + /// Cast result to UInt8 if needed if (result_node.result_type->getTypeId() != TypeIndex::UInt8) { - const auto & cast_node = step_dag->addCast(result_node, std::make_shared()); + const auto & cast_node = add_cast(step_dag, result_node, "UInt8"); + step_dag->addOrReplaceInOutputs(cast_node); result_name = cast_node.result_name; } @@ -147,8 +178,17 @@ std::unique_ptr IMergeTreeSelectAlgorithm::getPrewhereActions( } /// "Rename" the last step result to the combined prewhere column name, because in fact it will be AND of all step results - if (steps.back().column_name != prewhere_info->prewhere_column_name) - steps.back().actions->addAlias(steps.back().actions->findInOutputs(steps.back().column_name), prewhere_info->prewhere_column_name); + if (steps.back().column_name != prewhere_info->prewhere_column_name && + outputs_required_by_next_steps.contains(prewhere_info->prewhere_column_name)) + { + const auto & prewhere_result_node = add_cast( + steps.back().actions, + steps.back().actions->findInOutputs(steps.back().column_name), + outputs_required_by_next_steps[prewhere_info->prewhere_column_name]->getName(), + prewhere_info->prewhere_column_name); + + steps.back().actions->addOrReplaceInOutputs(prewhere_result_node); + } const size_t steps_before_prewhere = prewhere_actions->steps.size(); prewhere_actions->steps.resize(steps_before_prewhere + steps.size()); @@ -164,16 +204,16 @@ std::unique_ptr IMergeTreeSelectAlgorithm::getPrewhereActions( const bool remove_column = !outputs_required_by_next_steps.contains(step.column_name); /// Preserve outputs computed at this step that are used by the next steps for (const auto & output : outputs_required_by_next_steps) - if (step.actions->tryRestoreColumn(output)) - step_outputs.emplace_back(output); + if (step.actions->tryRestoreColumn(output.first)) + step_outputs.emplace_back(output.first); step.actions->removeUnusedActions(step_outputs, true, true); /// Add current step columns as outputs that should be preserved from previous steps for (const auto & input :step.actions->getInputs()) - outputs_required_by_next_steps.insert(input->result_name); + outputs_required_by_next_steps[input->result_name] = input->result_type; //std::cerr << conjunction->result_name << "\n"; - //std::cerr << step.actions->dumpDAG() << "\n"; + //std::cerr << "STEP " << i << ":\n" << step.actions->dumpDAG() << "\n"; PrewhereExprStep prewhere_step { From 8fc08c74f0dd60fa7778200da9bcb7e37a2f1197 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Fri, 10 Feb 2023 16:50:47 +0100 Subject: [PATCH 165/717] Pass action_settings to getReadTaskColumns() --- src/Processors/QueryPlan/ReadFromMergeTree.cpp | 2 ++ src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp | 5 +++-- src/Storages/MergeTree/MergeTreeBaseSelectProcessor.h | 3 ++- src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp | 4 +++- src/Storages/MergeTree/MergeTreeBlockReadUtils.h | 1 + src/Storages/MergeTree/MergeTreeReadPool.cpp | 4 +++- src/Storages/MergeTree/MergeTreeReadPool.h | 6 ++++++ src/Storages/MergeTree/MergeTreeSelectProcessor.cpp | 6 +++--- src/Storages/MergeTree/MergeTreeSelectProcessor.h | 2 +- src/Storages/MergeTree/MergeTreeThreadSelectProcessor.cpp | 4 ++-- src/Storages/MergeTree/MergeTreeThreadSelectProcessor.h | 2 +- 11 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 6ff230e9d78..734bab4624b 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -226,6 +226,7 @@ Pipe ReadFromMergeTree::readFromPoolParallelReplicas( extension, parts_with_range, prewhere_info, + actions_settings, reader_settings, required_columns, virt_column_names, @@ -304,6 +305,7 @@ Pipe ReadFromMergeTree::readFromPool( std::move(parts_with_range), storage_snapshot, prewhere_info, + actions_settings, reader_settings, required_columns, virt_column_names, diff --git a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp index c13cda6466a..00e1eed37e1 100644 --- a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp @@ -54,7 +54,7 @@ IMergeTreeSelectAlgorithm::IMergeTreeSelectAlgorithm( const MergeTreeData & storage_, const StorageSnapshotPtr & storage_snapshot_, const PrewhereInfoPtr & prewhere_info_, - ExpressionActionsSettings actions_settings, + const ExpressionActionsSettings & actions_settings_, UInt64 max_block_size_rows_, UInt64 preferred_block_size_bytes_, UInt64 preferred_max_column_in_block_size_bytes_, @@ -64,6 +64,7 @@ IMergeTreeSelectAlgorithm::IMergeTreeSelectAlgorithm( : storage(storage_) , storage_snapshot(storage_snapshot_) , prewhere_info(prewhere_info_) + , actions_settings(actions_settings_) , prewhere_actions(getPrewhereActions(prewhere_info, actions_settings, reader_settings_.enable_multiple_prewhere_read_steps)) , max_block_size_rows(max_block_size_rows_) , preferred_block_size_bytes(preferred_block_size_bytes_) @@ -141,7 +142,7 @@ std::unique_ptr IMergeTreeSelectAlgorithm::getPrewhereActions( std::vector steps; /// Adds a CAST node with the regular name ("CAST(...)") or with the provided name. - /// This is different from ActionsDAG::addCast() becuase it set the name equal to the original name effectively hiding the value before cast, + /// This is different from ActionsDAG::addCast() because it set the name equal to the original name effectively hiding the value before cast, /// but it might be required for further steps with its original uncasted type. auto add_cast = [] (ActionsDAGPtr dag, const ActionsDAG::Node & node_to_cast, const String & type_name, const String & new_name = {}) -> const ActionsDAG::Node & { diff --git a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.h b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.h index e66e3d8a741..b27a7114122 100644 --- a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.h +++ b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.h @@ -43,7 +43,7 @@ public: const MergeTreeData & storage_, const StorageSnapshotPtr & storage_snapshot_, const PrewhereInfoPtr & prewhere_info_, - ExpressionActionsSettings actions_settings, + const ExpressionActionsSettings & actions_settings, UInt64 max_block_size_rows_, UInt64 preferred_block_size_bytes_, UInt64 preferred_max_column_in_block_size_bytes_, @@ -140,6 +140,7 @@ protected: /// This step is added when the part has lightweight delete mask const PrewhereExprStep lightweight_delete_filter_step { nullptr, LightweightDeleteDescription::FILTER_COLUMN.name, true, true }; PrewhereInfoPtr prewhere_info; + ExpressionActionsSettings actions_settings; std::unique_ptr prewhere_actions; UInt64 max_block_size_rows; diff --git a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp index 7fd063826e7..95c76c81665 100644 --- a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp +++ b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp @@ -289,6 +289,7 @@ MergeTreeReadTaskColumns getReadTaskColumns( const Names & required_columns, const Names & system_columns, const PrewhereInfoPtr & prewhere_info, + const ExpressionActionsSettings & actions_settings, const MergeTreeReaderSettings & reader_settings, bool with_subcolumns) { @@ -314,7 +315,8 @@ MergeTreeReadTaskColumns getReadTaskColumns( if (prewhere_info) { - auto prewhere_actions = IMergeTreeSelectAlgorithm::getPrewhereActions(prewhere_info, {}, reader_settings.enable_multiple_prewhere_read_steps); // TODO: pass proper actions_settings + auto prewhere_actions = IMergeTreeSelectAlgorithm::getPrewhereActions( + prewhere_info, actions_settings, reader_settings.enable_multiple_prewhere_read_steps); NameSet columns_from_previous_steps; diff --git a/src/Storages/MergeTree/MergeTreeBlockReadUtils.h b/src/Storages/MergeTree/MergeTreeBlockReadUtils.h index 7b654a05283..8a8c1ca6a40 100644 --- a/src/Storages/MergeTree/MergeTreeBlockReadUtils.h +++ b/src/Storages/MergeTree/MergeTreeBlockReadUtils.h @@ -101,6 +101,7 @@ MergeTreeReadTaskColumns getReadTaskColumns( const Names & required_columns, const Names & system_columns, const PrewhereInfoPtr & prewhere_info, + const ExpressionActionsSettings & actions_settings, const MergeTreeReaderSettings & reader_settings, bool with_subcolumns); diff --git a/src/Storages/MergeTree/MergeTreeReadPool.cpp b/src/Storages/MergeTree/MergeTreeReadPool.cpp index 54cc44b6503..f5de75c7471 100644 --- a/src/Storages/MergeTree/MergeTreeReadPool.cpp +++ b/src/Storages/MergeTree/MergeTreeReadPool.cpp @@ -46,7 +46,7 @@ std::vector IMergeTreeReadPool::fillPerPartInfo(const RangesInDataParts auto task_columns = getReadTaskColumns( LoadedMergeTreeDataPartInfoForReader(part.data_part), storage_snapshot, - column_names, virtual_column_names, prewhere_info, reader_settings, /*with_subcolumns=*/ true); + column_names, virtual_column_names, prewhere_info, actions_settings, reader_settings, /*with_subcolumns=*/ true); auto size_predictor = !predict_block_size_bytes ? nullptr : IMergeTreeSelectAlgorithm::getSizePredictor(part.data_part, task_columns, sample_block); @@ -72,6 +72,7 @@ MergeTreeReadPool::MergeTreeReadPool( RangesInDataParts && parts_, const StorageSnapshotPtr & storage_snapshot_, const PrewhereInfoPtr & prewhere_info_, + const ExpressionActionsSettings & actions_settings_, const MergeTreeReaderSettings & reader_settings_, const Names & column_names_, const Names & virtual_column_names_, @@ -84,6 +85,7 @@ MergeTreeReadPool::MergeTreeReadPool( virtual_column_names_, min_marks_for_concurrent_read_, prewhere_info_, + actions_settings_, reader_settings_, std::move(parts_), (preferred_block_size_bytes_ > 0), diff --git a/src/Storages/MergeTree/MergeTreeReadPool.h b/src/Storages/MergeTree/MergeTreeReadPool.h index 857aa37f783..988dabb47f0 100644 --- a/src/Storages/MergeTree/MergeTreeReadPool.h +++ b/src/Storages/MergeTree/MergeTreeReadPool.h @@ -27,6 +27,7 @@ public: Names virtual_column_names_, size_t min_marks_for_concurrent_read_, PrewhereInfoPtr prewhere_info_, + const ExpressionActionsSettings & actions_settings_, const MergeTreeReaderSettings & reader_settings_, RangesInDataParts parts_ranges_, bool predict_block_size_bytes_, @@ -36,6 +37,7 @@ public: , virtual_column_names(virtual_column_names_) , min_marks_for_concurrent_read(min_marks_for_concurrent_read_) , prewhere_info(prewhere_info_) + , actions_settings(actions_settings_) , reader_settings(reader_settings_) , parts_ranges(parts_ranges_) , predict_block_size_bytes(predict_block_size_bytes_) @@ -57,6 +59,7 @@ protected: const Names virtual_column_names; size_t min_marks_for_concurrent_read{0}; PrewhereInfoPtr prewhere_info; + ExpressionActionsSettings actions_settings; MergeTreeReaderSettings reader_settings; RangesInDataParts parts_ranges; bool predict_block_size_bytes; @@ -126,6 +129,7 @@ public: RangesInDataParts && parts_, const StorageSnapshotPtr & storage_snapshot_, const PrewhereInfoPtr & prewhere_info_, + const ExpressionActionsSettings & actions_settings_, const MergeTreeReaderSettings & reader_settings_, const Names & column_names_, const Names & virtual_column_names_, @@ -201,6 +205,7 @@ public: ParallelReadingExtension extension_, const RangesInDataParts & parts_, const PrewhereInfoPtr & prewhere_info_, + const ExpressionActionsSettings & actions_settings_, const MergeTreeReaderSettings & reader_settings_, const Names & column_names_, const Names & virtual_column_names_, @@ -212,6 +217,7 @@ public: virtual_column_names_, min_marks_for_concurrent_read_, prewhere_info_, + actions_settings_, reader_settings_, parts_, /*predict_block_size*/false, diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp index 7daccdb95d3..83e9cc90547 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp @@ -19,7 +19,7 @@ MergeTreeSelectAlgorithm::MergeTreeSelectAlgorithm( MarkRanges mark_ranges_, bool use_uncompressed_cache_, const PrewhereInfoPtr & prewhere_info_, - ExpressionActionsSettings actions_settings, + const ExpressionActionsSettings & actions_settings_, const MergeTreeReaderSettings & reader_settings_, MergeTreeInOrderReadPoolParallelReplicasPtr pool_, const Names & virt_column_names_, @@ -27,7 +27,7 @@ MergeTreeSelectAlgorithm::MergeTreeSelectAlgorithm( bool has_limit_below_one_block_) : IMergeTreeSelectAlgorithm{ storage_snapshot_->getSampleBlockForColumns(required_columns_), - storage_, storage_snapshot_, prewhere_info_, std::move(actions_settings), max_block_size_rows_, + storage_, storage_snapshot_, prewhere_info_, actions_settings_, max_block_size_rows_, preferred_block_size_bytes_, preferred_max_column_in_block_size_bytes_, reader_settings_, use_uncompressed_cache_, virt_column_names_}, required_columns{std::move(required_columns_)}, @@ -46,7 +46,7 @@ void MergeTreeSelectAlgorithm::initializeReaders() { task_columns = getReadTaskColumns( LoadedMergeTreeDataPartInfoForReader(data_part), storage_snapshot, - required_columns, virt_column_names, prewhere_info, reader_settings, /*with_subcolumns=*/ true); + required_columns, virt_column_names, prewhere_info, actions_settings, reader_settings, /*with_subcolumns=*/ true); /// Will be used to distinguish between PREWHERE and WHERE columns when applying filter const auto & column_names = task_columns.columns.getNames(); diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.h b/src/Storages/MergeTree/MergeTreeSelectProcessor.h index 76c8d81dd0b..981c42574e0 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.h +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.h @@ -28,7 +28,7 @@ public: MarkRanges mark_ranges, bool use_uncompressed_cache, const PrewhereInfoPtr & prewhere_info, - ExpressionActionsSettings actions_settings, + const ExpressionActionsSettings & actions_settings_, const MergeTreeReaderSettings & reader_settings, MergeTreeInOrderReadPoolParallelReplicasPtr pool_, const Names & virt_column_names = {}, diff --git a/src/Storages/MergeTree/MergeTreeThreadSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeThreadSelectProcessor.cpp index 0c7f90b2349..d84bda63421 100644 --- a/src/Storages/MergeTree/MergeTreeThreadSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeThreadSelectProcessor.cpp @@ -18,12 +18,12 @@ MergeTreeThreadSelectAlgorithm::MergeTreeThreadSelectAlgorithm( const StorageSnapshotPtr & storage_snapshot_, bool use_uncompressed_cache_, const PrewhereInfoPtr & prewhere_info_, - ExpressionActionsSettings actions_settings, + const ExpressionActionsSettings & actions_settings_, const MergeTreeReaderSettings & reader_settings_, const Names & virt_column_names_) : IMergeTreeSelectAlgorithm{ - pool_->getHeader(), storage_, storage_snapshot_, prewhere_info_, std::move(actions_settings), max_block_size_rows_, + pool_->getHeader(), storage_, storage_snapshot_, prewhere_info_, actions_settings_, max_block_size_rows_, preferred_block_size_bytes_, preferred_max_column_in_block_size_bytes_, reader_settings_, use_uncompressed_cache_, virt_column_names_}, thread{thread_}, diff --git a/src/Storages/MergeTree/MergeTreeThreadSelectProcessor.h b/src/Storages/MergeTree/MergeTreeThreadSelectProcessor.h index 37c9375a581..4d9c9c92daf 100644 --- a/src/Storages/MergeTree/MergeTreeThreadSelectProcessor.h +++ b/src/Storages/MergeTree/MergeTreeThreadSelectProcessor.h @@ -25,7 +25,7 @@ public: const StorageSnapshotPtr & storage_snapshot_, bool use_uncompressed_cache_, const PrewhereInfoPtr & prewhere_info_, - ExpressionActionsSettings actions_settings, + const ExpressionActionsSettings & actions_settings_, const MergeTreeReaderSettings & reader_settings_, const Names & virt_column_names_); From 7ee7afe75062ba2fb13d646aa262d30e9e671f94 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Fri, 10 Feb 2023 22:09:07 +0100 Subject: [PATCH 166/717] More test cases --- ..._multiple_read_steps_in_prewhere.reference | 33 ++++++++++++------- .../02559_multiple_read_steps_in_prewhere.sql | 10 ++++-- 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.reference b/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.reference index fbd143e9b55..8e4056ec660 100644 --- a/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.reference +++ b/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.reference @@ -1,3 +1,7 @@ +--SET enable_multiple_prewhere_read_steps=true, move_all_conditions_to_prewhere=true; + +-- { echoOn } + SELECT cast(id as UInt16) AS id16 FROM test_02559 PREWHERE id16 and (id % 40000) LIMIT 10; 1 2 @@ -8,7 +12,7 @@ SELECT cast(id as UInt16) AS id16 FROM test_02559 PREWHERE id16 and (id % 40000) 7 8 9 -SELECT cast(id as UInt16) AS id16, (id % 40000) AS id40000, (id16 AND id40000) AS cond FROM test_02559 PREWHERE cond LIMIT 10; +SELECT cast(id as UInt16) AS cond1, (id % 40000) AS cond2, (cond1 AND cond2) AS cond FROM test_02559 PREWHERE cond LIMIT 10; 1 1 1 2 2 1 3 3 1 @@ -18,14 +22,21 @@ SELECT cast(id as UInt16) AS id16, (id % 40000) AS id40000, (id16 AND id40000) A 7 7 1 8 8 1 9 9 1 -SELECT cast(id as UInt16) AS id16 FROM test_02559 PREWHERE id16 LIMIT 10; -1 -2 -3 -4 -5 -6 -7 -8 -9 +SELECT cast(id as UInt16) AS cond1, (id % 40000) AS cond2, (cond1 AND cond2) AS cond FROM test_02559 PREWHERE cond AND id > 4 LIMIT 10; +5 5 1 +6 6 1 +7 7 1 +8 8 1 +9 9 1 +SELECT cast(id as UInt16) AS cond1, (id % 40000) AS cond2, (cond1 AND cond2) AS cond FROM test_02559 PREWHERE id > 5 AND cond LIMIT 10; +6 6 1 +7 7 1 +8 8 1 +9 9 1 +SELECT cast(id as UInt16) AS cond1, (id % 40000) AS cond2, (cond1 AND cond2) AS cond FROM test_02559 PREWHERE cond1 AND id > 6 AND cond2 LIMIT 10; +7 7 1 +8 8 1 +9 9 1 +SELECT cast(id as UInt16) AS cond1 FROM test_02559 PREWHERE cond1 LIMIT 10; -- { serverError ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER } SELECT count() FROM test_02559 PREWHERE 1 OR ignore(id) WHERE ignore(id)=0; +10 diff --git a/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.sql b/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.sql index 68c61dda6dd..5dd09457313 100644 --- a/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.sql +++ b/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.sql @@ -10,9 +10,15 @@ SET enable_multiple_prewhere_read_steps=true, move_all_conditions_to_prewhere=tr SELECT cast(id as UInt16) AS id16 FROM test_02559 PREWHERE id16 and (id % 40000) LIMIT 10; -SELECT cast(id as UInt16) AS id16, (id % 40000) AS id40000, (id16 AND id40000) AS cond FROM test_02559 PREWHERE cond LIMIT 10; +SELECT cast(id as UInt16) AS cond1, (id % 40000) AS cond2, (cond1 AND cond2) AS cond FROM test_02559 PREWHERE cond LIMIT 10; -SELECT cast(id as UInt16) AS id16 FROM test_02559 PREWHERE id16 LIMIT 10; +SELECT cast(id as UInt16) AS cond1, (id % 40000) AS cond2, (cond1 AND cond2) AS cond FROM test_02559 PREWHERE cond AND id > 4 LIMIT 10; + +SELECT cast(id as UInt16) AS cond1, (id % 40000) AS cond2, (cond1 AND cond2) AS cond FROM test_02559 PREWHERE id > 5 AND cond LIMIT 10; + +SELECT cast(id as UInt16) AS cond1, (id % 40000) AS cond2, (cond1 AND cond2) AS cond FROM test_02559 PREWHERE cond1 AND id > 6 AND cond2 LIMIT 10; + +SELECT cast(id as UInt16) AS cond1 FROM test_02559 PREWHERE cond1 LIMIT 10; -- { serverError ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER } SELECT count() FROM test_02559 PREWHERE 1 OR ignore(id) WHERE ignore(id)=0; From 12e576cddb18083021b4a5984939265d4d544537 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Fri, 10 Feb 2023 22:15:27 +0100 Subject: [PATCH 167/717] Slight refactoring --- .../MergeTreeBaseSelectProcessor.cpp | 70 ++++++++++--------- 1 file changed, 36 insertions(+), 34 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp index 00e1eed37e1..df7f77261af 100644 --- a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp @@ -89,6 +89,24 @@ IMergeTreeSelectAlgorithm::IMergeTreeSelectAlgorithm( LOG_TEST(log, "PREWHERE actions: {}", (prewhere_actions ? prewhere_actions->dump() : std::string(""))); } +/// Adds a CAST node with the regular name ("CAST(...)") or with the provided name. +/// This is different from ActionsDAG::addCast() because it set the name equal to the original name effectively hiding the value before cast, +/// but it might be required for further steps with its original uncasted type. +static const ActionsDAG::Node & addCast(ActionsDAGPtr dag, const ActionsDAG::Node & node_to_cast, const String & type_name, const String & new_name = {}) +{ + Field cast_type_constant_value(type_name); + + ColumnWithTypeAndName column; + column.name = calculateConstantActionNodeName(cast_type_constant_value); + column.column = DataTypeString().createColumnConst(0, cast_type_constant_value); + column.type = std::make_shared(); + + const auto * cast_type_constant_node = &dag->addColumn(std::move(column)); + ActionsDAG::NodeRawConstPtrs children = {&node_to_cast, cast_type_constant_node}; + FunctionOverloadResolverPtr func_builder_cast = CastInternalOverloadResolver::createImpl(); + + return dag->addFunction(func_builder_cast, std::move(children), new_name); +}; std::unique_ptr IMergeTreeSelectAlgorithm::getPrewhereActions(PrewhereInfoPtr prewhere_info, const ExpressionActionsSettings & actions_settings, bool enable_multiple_prewhere_read_steps) { @@ -112,6 +130,13 @@ std::unique_ptr IMergeTreeSelectAlgorithm::getPrewhereActions( //std::cerr << "ORIGINAL PREWHERE:\n" << prewhere_info->prewhere_actions->dumpDAG() << std::endl; + struct Step + { + ActionsDAGPtr actions; + String column_name; + }; + std::vector steps; + if (enable_multiple_prewhere_read_steps) { /// Find all conjunctions in prewhere expression. @@ -119,12 +144,6 @@ std::unique_ptr IMergeTreeSelectAlgorithm::getPrewhereActions( prewhere_info->prewhere_actions->tryFindInOutputs(prewhere_info->prewhere_column_name), {}); - /// Save the list of outputs from the original prewhere expression. - auto original_outputs = prewhere_info->prewhere_actions->getOutputs(); - std::unordered_map outputs_required_by_next_steps; - for (const auto & output : original_outputs) - outputs_required_by_next_steps[output->result_name] = output->result_type; - /// Save the list of inputs to the original prewhere expression. auto inputs = prewhere_info->prewhere_actions->getInputs(); ColumnsWithTypeAndName all_inputs; @@ -134,32 +153,6 @@ std::unique_ptr IMergeTreeSelectAlgorithm::getPrewhereActions( ActionsDAG::NodeRawConstPtrs all_conjunctions = std::move(conjunctions.allowed); all_conjunctions.insert(all_conjunctions.end(), conjunctions.rejected.begin(), conjunctions.rejected.end()); - struct Step - { - ActionsDAGPtr actions; - String column_name; - }; - std::vector steps; - - /// Adds a CAST node with the regular name ("CAST(...)") or with the provided name. - /// This is different from ActionsDAG::addCast() because it set the name equal to the original name effectively hiding the value before cast, - /// but it might be required for further steps with its original uncasted type. - auto add_cast = [] (ActionsDAGPtr dag, const ActionsDAG::Node & node_to_cast, const String & type_name, const String & new_name = {}) -> const ActionsDAG::Node & - { - Field cast_type_constant_value(type_name); - - ColumnWithTypeAndName column; - column.name = calculateConstantActionNodeName(cast_type_constant_value); - column.column = DataTypeString().createColumnConst(0, cast_type_constant_value); - column.type = std::make_shared(); - - const auto * cast_type_constant_node = &dag->addColumn(std::move(column)); - ActionsDAG::NodeRawConstPtrs children = {&node_to_cast, cast_type_constant_node}; - FunctionOverloadResolverPtr func_builder_cast = CastInternalOverloadResolver::createImpl(); - - return dag->addFunction(func_builder_cast, std::move(children), new_name); - }; - /// Make separate DAG for each step for (const auto & conjunction : all_conjunctions) { @@ -169,7 +162,7 @@ std::unique_ptr IMergeTreeSelectAlgorithm::getPrewhereActions( /// Cast result to UInt8 if needed if (result_node.result_type->getTypeId() != TypeIndex::UInt8) { - const auto & cast_node = add_cast(step_dag, result_node, "UInt8"); + const auto & cast_node = addCast(step_dag, result_node, "UInt8"); step_dag->addOrReplaceInOutputs(cast_node); result_name = cast_node.result_name; @@ -177,12 +170,21 @@ std::unique_ptr IMergeTreeSelectAlgorithm::getPrewhereActions( step_dag->removeUnusedActions(Names{result_name}, true, true); steps.emplace_back(Step{step_dag, result_name}); } + } + + if (steps.size() > 1) + { + /// Save the list of outputs from the original prewhere expression. + auto original_outputs = prewhere_info->prewhere_actions->getOutputs(); + std::unordered_map outputs_required_by_next_steps; + for (const auto & output : original_outputs) + outputs_required_by_next_steps[output->result_name] = output->result_type; /// "Rename" the last step result to the combined prewhere column name, because in fact it will be AND of all step results if (steps.back().column_name != prewhere_info->prewhere_column_name && outputs_required_by_next_steps.contains(prewhere_info->prewhere_column_name)) { - const auto & prewhere_result_node = add_cast( + const auto & prewhere_result_node = addCast( steps.back().actions, steps.back().actions->findInOutputs(steps.back().column_name), outputs_required_by_next_steps[prewhere_info->prewhere_column_name]->getName(), From 943f2ea212db183306429ba99fa0e6508651f581 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Sun, 12 Feb 2023 21:41:13 +0100 Subject: [PATCH 168/717] Use 2 columns in test to have 2 read steps --- ...9_multiple_read_steps_in_prewhere.reference | 16 +++++++--------- .../02559_multiple_read_steps_in_prewhere.sql | 18 +++++++++--------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.reference b/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.reference index 8e4056ec660..0c46a90257d 100644 --- a/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.reference +++ b/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.reference @@ -1,8 +1,6 @@ ---SET enable_multiple_prewhere_read_steps=true, move_all_conditions_to_prewhere=true; - -- { echoOn } -SELECT cast(id as UInt16) AS id16 FROM test_02559 PREWHERE id16 and (id % 40000) LIMIT 10; +SELECT cast(id1 as UInt16) AS id16 FROM test_02559 PREWHERE id16 and (id2 % 40000) LIMIT 10; 1 2 3 @@ -12,7 +10,7 @@ SELECT cast(id as UInt16) AS id16 FROM test_02559 PREWHERE id16 and (id % 40000) 7 8 9 -SELECT cast(id as UInt16) AS cond1, (id % 40000) AS cond2, (cond1 AND cond2) AS cond FROM test_02559 PREWHERE cond LIMIT 10; +SELECT cast(id1 as UInt16) AS cond1, (id2 % 40000) AS cond2, (cond1 AND cond2) AS cond FROM test_02559 PREWHERE cond LIMIT 10; 1 1 1 2 2 1 3 3 1 @@ -22,21 +20,21 @@ SELECT cast(id as UInt16) AS cond1, (id % 40000) AS cond2, (cond1 AND cond2) AS 7 7 1 8 8 1 9 9 1 -SELECT cast(id as UInt16) AS cond1, (id % 40000) AS cond2, (cond1 AND cond2) AS cond FROM test_02559 PREWHERE cond AND id > 4 LIMIT 10; +SELECT cast(id1 as UInt16) AS cond1, (id2 % 40000) AS cond2, (cond1 AND cond2) AS cond FROM test_02559 PREWHERE cond AND id2 > 4 LIMIT 10; 5 5 1 6 6 1 7 7 1 8 8 1 9 9 1 -SELECT cast(id as UInt16) AS cond1, (id % 40000) AS cond2, (cond1 AND cond2) AS cond FROM test_02559 PREWHERE id > 5 AND cond LIMIT 10; +SELECT cast(id1 as UInt16) AS cond1, (id2 % 40000) AS cond2, (cond1 AND cond2) AS cond FROM test_02559 PREWHERE id2 > 5 AND cond LIMIT 10; 6 6 1 7 7 1 8 8 1 9 9 1 -SELECT cast(id as UInt16) AS cond1, (id % 40000) AS cond2, (cond1 AND cond2) AS cond FROM test_02559 PREWHERE cond1 AND id > 6 AND cond2 LIMIT 10; +SELECT cast(id1 as UInt16) AS cond1, (id2 % 40000) AS cond2, (cond1 AND cond2) AS cond FROM test_02559 PREWHERE cond1 AND id2 > 6 AND cond2 LIMIT 10; 7 7 1 8 8 1 9 9 1 -SELECT cast(id as UInt16) AS cond1 FROM test_02559 PREWHERE cond1 LIMIT 10; -- { serverError ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER } -SELECT count() FROM test_02559 PREWHERE 1 OR ignore(id) WHERE ignore(id)=0; +SELECT cast(id1 as UInt16) AS cond1 FROM test_02559 PREWHERE cond1 LIMIT 10; -- { serverError ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER } +SELECT count() FROM test_02559 PREWHERE id2>=0 AND (1 OR ignore(id1)) WHERE ignore(id1)=0; 10 diff --git a/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.sql b/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.sql index 5dd09457313..e39f8ac846a 100644 --- a/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.sql +++ b/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.sql @@ -1,26 +1,26 @@ DROP TABLE IF EXISTS test_02559; -CREATE TABLE test_02559 (id UInt64) ENGINE=MergeTree ORDER BY id; +CREATE TABLE test_02559 (id1 UInt64, id2 UInt64) ENGINE=MergeTree ORDER BY id1; -INSERT INTO test_02559 SELECT number FROM numbers(10); +INSERT INTO test_02559 SELECT number, number FROM numbers(10); SET enable_multiple_prewhere_read_steps=true, move_all_conditions_to_prewhere=true; -- { echoOn } -SELECT cast(id as UInt16) AS id16 FROM test_02559 PREWHERE id16 and (id % 40000) LIMIT 10; +SELECT cast(id1 as UInt16) AS id16 FROM test_02559 PREWHERE id16 and (id2 % 40000) LIMIT 10; -SELECT cast(id as UInt16) AS cond1, (id % 40000) AS cond2, (cond1 AND cond2) AS cond FROM test_02559 PREWHERE cond LIMIT 10; +SELECT cast(id1 as UInt16) AS cond1, (id2 % 40000) AS cond2, (cond1 AND cond2) AS cond FROM test_02559 PREWHERE cond LIMIT 10; -SELECT cast(id as UInt16) AS cond1, (id % 40000) AS cond2, (cond1 AND cond2) AS cond FROM test_02559 PREWHERE cond AND id > 4 LIMIT 10; +SELECT cast(id1 as UInt16) AS cond1, (id2 % 40000) AS cond2, (cond1 AND cond2) AS cond FROM test_02559 PREWHERE cond AND id2 > 4 LIMIT 10; -SELECT cast(id as UInt16) AS cond1, (id % 40000) AS cond2, (cond1 AND cond2) AS cond FROM test_02559 PREWHERE id > 5 AND cond LIMIT 10; +SELECT cast(id1 as UInt16) AS cond1, (id2 % 40000) AS cond2, (cond1 AND cond2) AS cond FROM test_02559 PREWHERE id2 > 5 AND cond LIMIT 10; -SELECT cast(id as UInt16) AS cond1, (id % 40000) AS cond2, (cond1 AND cond2) AS cond FROM test_02559 PREWHERE cond1 AND id > 6 AND cond2 LIMIT 10; +SELECT cast(id1 as UInt16) AS cond1, (id2 % 40000) AS cond2, (cond1 AND cond2) AS cond FROM test_02559 PREWHERE cond1 AND id2 > 6 AND cond2 LIMIT 10; -SELECT cast(id as UInt16) AS cond1 FROM test_02559 PREWHERE cond1 LIMIT 10; -- { serverError ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER } +SELECT cast(id1 as UInt16) AS cond1 FROM test_02559 PREWHERE cond1 LIMIT 10; -- { serverError ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER } -SELECT count() FROM test_02559 PREWHERE 1 OR ignore(id) WHERE ignore(id)=0; +SELECT count() FROM test_02559 PREWHERE id2>=0 AND (1 OR ignore(id1)) WHERE ignore(id1)=0; -- { echoOff } From 3ad0683de93691bc8c9745f622a373b8f958db3a Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Sun, 12 Feb 2023 21:42:24 +0100 Subject: [PATCH 169/717] Refactor DAG splitting into steps --- .../MergeTreeBaseSelectProcessor.cpp | 269 +++++++++++++++++- 1 file changed, 268 insertions(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp index df7f77261af..5f40950c3b2 100644 --- a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp @@ -17,6 +17,7 @@ /// For CAST to bool #include +#include #include #include @@ -108,6 +109,255 @@ static const ActionsDAG::Node & addCast(ActionsDAGPtr dag, const ActionsDAG::Nod return dag->addFunction(func_builder_cast, std::move(children), new_name); }; +struct NodeInfo +{ + NameSet required_columns; +}; + +void fillRequiredColumns(const ActionsDAG::Node * node, std::unordered_map & nodes_info) +{ + if (nodes_info.contains(node)) + return; + + auto & node_info = nodes_info[node]; + + if (node->type == ActionsDAG::ActionType::INPUT) + { + node_info.required_columns.insert(node->result_name); + return; + } + + for (const auto & child : node->children) + { + fillRequiredColumns(child, nodes_info); + const auto & child_info = nodes_info[child]; + node_info.required_columns.insert(child_info.required_columns.begin(), child_info.required_columns.end()); + } +} + +struct DAGNodeRef +{ + ActionsDAGPtr dag; + const ActionsDAG::Node * node; +}; + +using OriginalToNewNodeMap = std::unordered_map; + +const ActionsDAG::Node & addClonedDAGToDAG(const ActionsDAG::Node * original_dag_node, ActionsDAGPtr new_dag, OriginalToNewNodeMap & node_remap) +{ + if (node_remap.contains(original_dag_node)) + { + /// If the node is already in the new DAG, return it + const auto & node_ref = node_remap.at(original_dag_node); + if (node_ref.dag == new_dag) + return *node_ref.node; + + /// If the node is known from the previous steps, add it as an input + node_ref.dag->addOrReplaceInOutputs(*node_ref.node); + const auto & new_node = new_dag->addInput(node_ref.node->result_name, node_ref.node->result_type); + node_remap[original_dag_node] = {new_dag, &new_node}; /// TODO: here we update the node reference. Is ti always correct? + return new_node; + } + + /// If the node is an input, add it as an input + if (original_dag_node->type == ActionsDAG::ActionType::INPUT) + { + const auto & new_node = new_dag->addInput(original_dag_node->result_name, original_dag_node->result_type); + node_remap[original_dag_node] = {new_dag, &new_node}; + return new_node; + } + + /// If the node is a column, add it as an input + if (original_dag_node->type == ActionsDAG::ActionType::COLUMN) + { + const auto & new_node = new_dag->addColumn( + ColumnWithTypeAndName(original_dag_node->column, original_dag_node->result_type, original_dag_node->result_name)); + node_remap[original_dag_node] = {new_dag, &new_node}; + return new_node; + } + + /// TODO: Alias node? + + /// If the node is a function, add it as a function and add its children + if (original_dag_node->type == ActionsDAG::ActionType::FUNCTION) + { + ActionsDAG::NodeRawConstPtrs new_children; + for (const auto & child : original_dag_node->children) + { + const auto & new_child = addClonedDAGToDAG(child, new_dag, node_remap); + new_children.push_back(&new_child); + } + + const auto & new_node = new_dag->addFunction(original_dag_node->function_base, new_children, original_dag_node->result_name); + node_remap[original_dag_node] = {new_dag, &new_node}; + return new_node; + } + + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected node type in PREWHERE actions: {}", original_dag_node->type); +} + +bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionActionsSettings & actions_settings, PrewhereExprInfo & prewhere) +{ + /// We want to build a sequence of steps that will compute parts of the prewhere condition. + /// Each step reads some new columns and computes some new expressions and a filter condition. + /// The last step computes the final filter condition and the remaining expressions that are required for the main query. + /// The steps are built in the following way: + /// 1. List all condition nodes that are combined with AND into PREWHERE condition + /// 2. Collect the set of columns that are used in the condition + /// 3. Sort condition nodes by the number of columns used in them and the overall size of those columns + /// 4. Group conditions with the same set of columns into a single read/compute step + /// 5. Build DAGs for each step: + /// - DFS from the condition root node: + /// - If the node was not computed yet, add it to the DAG and traverse its children + /// - If the node was already computed by one of the previous steps, add it as output for that step and as input for the current step + /// - If the node was already computed by the current step just stop traversing + /// 6. Find all outputs of the original DAG + /// 7. Find all outputs that were computed in the already built DAGs, mark these nodes as outputs in the steps where they were computed + /// 8. Add computation of the remaining outputs to the last step with the procedure similar to 4 + + if (!prewhere_info || !prewhere_info->prewhere_actions) + return true; + + /// 1. List all condition nodes that are combined with AND into PREWHERE condition + const auto & condition_root = prewhere_info->prewhere_actions->findInOutputs(prewhere_info->prewhere_column_name); + const bool is_conjunction = (condition_root.type == ActionsDAG::ActionType::FUNCTION && condition_root.function_base->getName() == "and"); + if (!is_conjunction) + return false; + auto condition_nodes = condition_root.children; + + /// 2. Collect the set of columns that are used in the condition + std::unordered_map nodes_info; + for (const auto & node : condition_nodes) + { + fillRequiredColumns(node, nodes_info); + } + + /// 3. Sort condition nodes by the number of columns used in them and the overall size of those columns + /// TODO: not sorting for now because the conditions are already sorted by Where Optimizer + + /// 4. Group conditions with the same set of columns into a single read/compute step + std::vector> condition_groups; + for (const auto & node : condition_nodes) + { + const auto & node_info = nodes_info[node]; + if (!condition_groups.empty() && nodes_info[condition_groups.back().back()].required_columns == node_info.required_columns) + condition_groups.back().push_back(node); /// Add to the last group + else + condition_groups.push_back({node}); /// Start new group + } + + /// 5. Build DAGs for each step + struct Step + { + ActionsDAGPtr actions; + String column_name; + }; + std::vector steps; + + OriginalToNewNodeMap node_remap; + + for (const auto & condition_group : condition_groups) + { +// std::cerr +// << "Conditions: ["; +// +// for (const auto & condition : condition_group) +// std::cerr << " \"" << condition->result_name; +// +// std::cerr << "\" ] Columns: " << boost::algorithm::join(nodes_info[condition_group.front()].required_columns, " ") +// << std::endl; + + ActionsDAGPtr step_dag = std::make_shared(); + String result_name; + + std::vector new_condition_nodes; + for (const auto * node : condition_group) + { + const auto & node_in_new_dag = addClonedDAGToDAG(node, step_dag, node_remap); + new_condition_nodes.push_back(&node_in_new_dag); + } + + if (new_condition_nodes.size() > 1) + { + /// Add AND function to combine the conditions + FunctionOverloadResolverPtr func_builder_and = std::make_unique(std::make_shared()); + const auto & and_function_node = step_dag->addFunction(func_builder_and, new_condition_nodes, ""); + step_dag->addOrReplaceInOutputs(and_function_node); + result_name = and_function_node.result_name; + } + else + { + const auto & result_node = *new_condition_nodes.front(); + /// Add cast to UInt8 if needed + if (result_node.result_type->getTypeId() == TypeIndex::UInt8) + { + step_dag->addOrReplaceInOutputs(result_node); + result_name = result_node.result_name; + } + else + { + const auto & cast_node = addCast(step_dag, result_node, "UInt8"); + step_dag->addOrReplaceInOutputs(cast_node); + result_name = cast_node.result_name; + } + } + +// std::cerr << "Step DAG:\n" << step_dag->dumpDAG() << std::endl; + + steps.push_back({step_dag, result_name}); + } + + /// 6. Find all outputs of the original DAG + auto original_outputs = prewhere_info->prewhere_actions->getOutputs(); + /// 7. Find all outputs that were computed in the already built DAGs, mark these nodes as outputs in the steps where they were computed + /// 8. Add computation of the remaining outputs to the last step with the procedure similar to 4 + for (const auto * output : original_outputs) + { +// std::cerr << "Original output: " << output->result_name << std::endl; + if (node_remap.contains(output)) + { + const auto & new_node_info = node_remap[output]; + new_node_info.dag->addOrReplaceInOutputs(*new_node_info.node); + } + else if (output->result_name == prewhere_info->prewhere_column_name) + { + /// Special case for final PREWHERE column: + /// "Rename" the last step result to the combined PREWHERE column name, because in fact it will be AND of all step results + const auto & prewhere_result_node = addCast( + steps.back().actions, + steps.back().actions->findInOutputs(steps.back().column_name), + output->result_type->getName(), + prewhere_info->prewhere_column_name); + + steps.back().actions->addOrReplaceInOutputs(prewhere_result_node); + } + else + { + const auto & node_in_new_dag = addClonedDAGToDAG(output, steps.back().actions, node_remap); + steps.back().actions->addOrReplaceInOutputs(node_in_new_dag); + } + } + + /// 9. Build PrewhereExprInfo + { + for (const auto & step : steps) + { +// std::cerr << "Step DAG:\n" << step.actions->dumpDAG() << std::endl; + prewhere.steps.push_back( + { + .actions = std::make_shared(step.actions, actions_settings), + .column_name = step.column_name, + .remove_column = true, + .need_filter = false, + }); + } + prewhere.steps.back().remove_column = prewhere_info->remove_prewhere_column; + prewhere.steps.back().need_filter = prewhere_info->need_filter; + } + + return true; +} + std::unique_ptr IMergeTreeSelectAlgorithm::getPrewhereActions(PrewhereInfoPtr prewhere_info, const ExpressionActionsSettings & actions_settings, bool enable_multiple_prewhere_read_steps) { std::unique_ptr prewhere_actions; @@ -128,7 +378,23 @@ std::unique_ptr IMergeTreeSelectAlgorithm::getPrewhereActions( prewhere_actions->steps.emplace_back(std::move(row_level_filter_step)); } - //std::cerr << "ORIGINAL PREWHERE:\n" << prewhere_info->prewhere_actions->dumpDAG() << std::endl; +// std::cerr << "ORIGINAL PREWHERE:\n" << prewhere_info->prewhere_actions->dumpDAG() << std::endl; + +#if 1 + if (!enable_multiple_prewhere_read_steps || !tryBuildPrewhereSteps(prewhere_info, actions_settings, *prewhere_actions)) + { + PrewhereExprStep prewhere_step + { + .actions = std::make_shared(prewhere_info->prewhere_actions, actions_settings), + .column_name = prewhere_info->prewhere_column_name, + .remove_column = prewhere_info->remove_prewhere_column, + .need_filter = prewhere_info->need_filter + }; + + prewhere_actions->steps.emplace_back(std::move(prewhere_step)); + } + +#else struct Step { @@ -243,6 +509,7 @@ std::unique_ptr IMergeTreeSelectAlgorithm::getPrewhereActions( prewhere_actions->steps.emplace_back(std::move(prewhere_step)); } +#endif } return prewhere_actions; From 9817c5601bbb3330b53479ea956e457065b0b5a3 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Sun, 12 Feb 2023 22:25:33 +0100 Subject: [PATCH 170/717] Fixed clang tidy build by updating parameter name to common_backups_path - Use cluster state data to check concurrent backup/restore --- src/Backups/BackupCoordinationRemote.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Backups/BackupCoordinationRemote.cpp b/src/Backups/BackupCoordinationRemote.cpp index f613eb7d198..29514b8fe1f 100644 --- a/src/Backups/BackupCoordinationRemote.cpp +++ b/src/Backups/BackupCoordinationRemote.cpp @@ -595,23 +595,23 @@ Strings BackupCoordinationRemote::getAllArchiveSuffixes() const return node_names; } -bool BackupCoordinationRemote::hasConcurrentBackups(const String & backup_id, const String & common_backup_path, const std::atomic &) const +bool BackupCoordinationRemote::hasConcurrentBackups(const String & backup_id, const String & common_backups_path, const std::atomic &) const { /// If its internal concurrency will be checked for the base backup if (is_internal) return false; auto zk = getZooKeeper(); - std::string backup_stage_path = common_backup_path + "/backup-" + toString(backup_id) +"/stage"; + std::string backup_stage_path = common_backups_path + "/backup-" + toString(backup_id) +"/stage"; - if (!zk->exists(common_backup_path)) - zk->createAncestors(common_backup_path); + if (!zk->exists(common_backups_path)) + zk->createAncestors(common_backups_path); for (size_t attempt = 0; attempt < MAX_ZOOKEEPER_ATTEMPTS; ++attempt) { Coordination::Stat stat; - zk->get(common_backup_path, &stat); - Strings existing_backup_paths = zk->getChildren(common_backup_path); + zk->get(common_backups_path, &stat); + Strings existing_backup_paths = zk->getChildren(common_backups_path); for (const auto & existing_backup_path : existing_backup_paths) { @@ -624,7 +624,7 @@ bool BackupCoordinationRemote::hasConcurrentBackups(const String & backup_id, co if (existing_backup_id == toString(backup_id)) continue; - const auto status = zk->get(common_backup_path + "/" + existing_backup_path + "/stage"); + const auto status = zk->get(common_backups_path + "/" + existing_backup_path + "/stage"); if (status != Stage::COMPLETED) return true; } From a89465d1cb3512be4a440fb6e657e0294e5d5dbf Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Sun, 12 Feb 2023 22:48:28 +0100 Subject: [PATCH 171/717] Fix build after merge --- src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp | 6 +++++- src/Storages/MergeTree/MergeTreePrefetchedReadPool.h | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp b/src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp index d7cbf18c115..73de5685085 100644 --- a/src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp +++ b/src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp @@ -31,6 +31,7 @@ MergeTreePrefetchedReadPool::MergeTreePrefetchedReadPool( RangesInDataParts && parts_, const StorageSnapshotPtr & storage_snapshot_, const PrewhereInfoPtr & prewhere_info_, + const ExpressionActionsSettings & actions_settings_, const Names & column_names_, const Names & virtual_column_names_, size_t preferred_block_size_bytes_, @@ -45,6 +46,8 @@ MergeTreePrefetchedReadPool::MergeTreePrefetchedReadPool( virtual_column_names_, min_marks_for_concurrent_read_, prewhere_info_, + actions_settings_, + reader_settings_, parts_, (preferred_block_size_bytes_ > 0), /*do_not_steal_tasks_*/false) @@ -53,7 +56,6 @@ MergeTreePrefetchedReadPool::MergeTreePrefetchedReadPool( , header(storage_snapshot_->getSampleBlockForColumns(column_names_)) , mark_cache(context_->getGlobalContext()->getMarkCache().get()) , uncompressed_cache(use_uncompressed_cache_ ? context_->getGlobalContext()->getUncompressedCache().get() : nullptr) - , reader_settings(reader_settings_) , profile_callback([this](ReadBufferFromFileBase::ProfileInfo info_) { profileFeedback(info_); }) , index_granularity_bytes(storage_settings_.index_granularity_bytes) , fixed_index_granularity(storage_settings_.index_granularity) @@ -326,6 +328,8 @@ MergeTreePrefetchedReadPool::PartsInfos MergeTreePrefetchedReadPool::getPartsInf column_names, virtual_column_names, prewhere_info, + actions_settings, + reader_settings, /* with_subcolumns */true); part_info->size_predictor = !predict_block_size_bytes diff --git a/src/Storages/MergeTree/MergeTreePrefetchedReadPool.h b/src/Storages/MergeTree/MergeTreePrefetchedReadPool.h index ca12739ef6b..c3ac6c7c3e7 100644 --- a/src/Storages/MergeTree/MergeTreePrefetchedReadPool.h +++ b/src/Storages/MergeTree/MergeTreePrefetchedReadPool.h @@ -25,6 +25,7 @@ public: RangesInDataParts && parts_, const StorageSnapshotPtr & storage_snapshot_, const PrewhereInfoPtr & prewhere_info_, + const ExpressionActionsSettings & actions_settings_, const Names & column_names_, const Names & virtual_column_names_, size_t preferred_block_size_bytes_, @@ -80,7 +81,6 @@ private: Block header; MarkCache * mark_cache; UncompressedCache * uncompressed_cache; - MergeTreeReaderSettings reader_settings; ReadBufferFromFileBase::ProfileCallback profile_callback; size_t index_granularity_bytes; size_t fixed_index_granularity; From d3dd9421dac96b9da413212c66ea626137573f1d Mon Sep 17 00:00:00 2001 From: flynn Date: Mon, 13 Feb 2023 08:29:22 +0000 Subject: [PATCH 172/717] refactor and get rid of s3 --- src/Storages/StorageIceberg.cpp | 85 ++++++++++------ src/Storages/StorageIceberg.h | 16 ++- src/Storages/StorageS3.h | 4 +- src/TableFunctions/ITableFunctionDataLake.h | 99 +++---------------- src/TableFunctions/TableFunctionDeltaLake.cpp | 2 +- src/TableFunctions/TableFunctionHudi.cpp | 2 +- src/TableFunctions/TableFunctionIceberg.cpp | 2 +- src/TableFunctions/TableFunctionS3.cpp | 10 +- src/TableFunctions/TableFunctionS3.h | 7 +- src/TableFunctions/TableFunctionS3Cluster.cpp | 2 +- 10 files changed, 96 insertions(+), 133 deletions(-) diff --git a/src/Storages/StorageIceberg.cpp b/src/Storages/StorageIceberg.cpp index 93aa16cd29a..ec5284adf94 100644 --- a/src/Storages/StorageIceberg.cpp +++ b/src/Storages/StorageIceberg.cpp @@ -46,12 +46,35 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; } -IcebergMetaParser::IcebergMetaParser(const StorageS3::Configuration & configuration_, ContextPtr context_) +std::shared_ptr +S3MetaReadHelper::createReadBuffer(const String & key, ContextPtr context, const StorageS3::Configuration & base_configuration) +{ + S3Settings::RequestSettings request_settings; + request_settings.max_single_read_retries = context->getSettingsRef().s3_max_single_read_retries; + return std::make_shared( + base_configuration.client, + base_configuration.url.bucket, + key, + base_configuration.url.version_id, + request_settings, + context->getReadSettings()); +} +std::vector +S3MetaReadHelper::listFilesMatchSuffix(const StorageS3::Configuration & base_configuration, const String & directory, const String & suffix) +{ + const auto & table_path = base_configuration.url.key; + return S3::listFiles( + *base_configuration.client, base_configuration.url.bucket, table_path, std::filesystem::path(table_path) / directory, suffix); +} + +template +IcebergMetaParser::IcebergMetaParser(const Configuration & configuration_, ContextPtr context_) : base_configuration(configuration_), context(context_) { } -std::vector IcebergMetaParser::getFiles() const +template +std::vector IcebergMetaParser::getFiles() const { auto metadata = getNewestMetaFile(); auto manifest_list = getManiFestList(metadata); @@ -66,29 +89,27 @@ std::vector IcebergMetaParser::getFiles() const return getFilesForRead(manifest_files); } -String IcebergMetaParser::getNewestMetaFile() const +template +String IcebergMetaParser::getNewestMetaFile() const { /// Iceberg stores all the metadata.json in metadata directory, and the /// newest version has the max version name, so we should list all of them /// then find the newest metadata. - const auto & table_path = base_configuration.url.key; - std::vector metadata_files = S3::listFiles( - *base_configuration.client, - base_configuration.url.bucket, - table_path, - std::filesystem::path(table_path) / metadata_directory, - ".json"); + constexpr auto meta_file_suffix = ".json"; + auto metadata_files = MetaReadHelper::listFilesMatchSuffix(base_configuration, metadata_directory, meta_file_suffix); if (metadata_files.empty()) - throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "The metadata file for Iceberg table with path {} doesn't exist", table_path); + throw Exception( + ErrorCodes::FILE_DOESNT_EXIST, "The metadata file for Iceberg table with path {} doesn't exist", base_configuration.url.key); auto it = std::max_element(metadata_files.begin(), metadata_files.end()); return *it; } -String IcebergMetaParser::getManiFestList(const String & metadata_name) const +template +String IcebergMetaParser::getManiFestList(const String & metadata_name) const { - auto buffer = createS3ReadBuffer(metadata_name); + auto buffer = MetaReadHelper::createReadBuffer(metadata_name, context, base_configuration); String json_str; readJSONObjectPossiblyInvalid(json_str, *buffer); @@ -132,9 +153,10 @@ parseAvro(const std::unique_ptr & file_reader, const D return columns; } -std::vector IcebergMetaParser::getManifestFiles(const String & manifest_list) const +template +std::vector IcebergMetaParser::getManifestFiles(const String & manifest_list) const { - auto buffer = createS3ReadBuffer(manifest_list); + auto buffer = MetaReadHelper::createReadBuffer(manifest_list, context, base_configuration); auto file_reader = std::make_unique(std::make_unique(*buffer)); @@ -168,12 +190,13 @@ std::vector IcebergMetaParser::getManifestFiles(const String & manifest_ col->getFamilyName()); } -std::vector IcebergMetaParser::getFilesForRead(const std::vector & manifest_files) const +template +std::vector IcebergMetaParser::getFilesForRead(const std::vector & manifest_files) const { std::vector keys; for (const auto & manifest_file : manifest_files) { - auto buffer = createS3ReadBuffer(manifest_file); + auto buffer = MetaReadHelper::createReadBuffer(manifest_file, context, base_configuration); auto file_reader = std::make_unique(std::make_unique(*buffer)); @@ -222,27 +245,27 @@ std::vector IcebergMetaParser::getFilesForRead(const std::vector return keys; } -std::shared_ptr IcebergMetaParser::createS3ReadBuffer(const String & key) const -{ - S3Settings::RequestSettings request_settings; - request_settings.max_single_read_retries = context->getSettingsRef().s3_max_single_read_retries; - return std::make_shared( - base_configuration.client, - base_configuration.url.bucket, - key, - base_configuration.url.version_id, - request_settings, - context->getReadSettings()); -} - // generateQueryFromKeys constructs query from all parquet filenames // for underlying StorageS3 engine -String IcebergMetaParser::generateQueryFromKeys(const std::vector & keys, const String &) +template +String IcebergMetaParser::generateQueryFromKeys(const std::vector & keys, const String &) { std::string new_query = fmt::format("{{{}}}", fmt::join(keys, ",")); return new_query; } +template IcebergMetaParser::IcebergMetaParser( + const StorageS3::Configuration & configuration_, ContextPtr context_); +template std::vector IcebergMetaParser::getFiles() const; +template String IcebergMetaParser::generateQueryFromKeys( + const std::vector & keys, const String & format); +template String IcebergMetaParser::getNewestMetaFile() const; +template String IcebergMetaParser::getManiFestList(const String & metadata_name) const; +template std::vector +IcebergMetaParser::getManifestFiles(const String & manifest_list) const; +template std::vector +IcebergMetaParser::getFilesForRead(const std::vector & manifest_files) const; + void registerStorageIceberg(StorageFactory & factory) { factory.registerStorage( diff --git a/src/Storages/StorageIceberg.h b/src/Storages/StorageIceberg.h index d351f073c8b..8a535643fce 100644 --- a/src/Storages/StorageIceberg.h +++ b/src/Storages/StorageIceberg.h @@ -12,16 +12,26 @@ namespace DB { +struct S3MetaReadHelper +{ + static std::shared_ptr + createReadBuffer(const String & key, ContextPtr context, const StorageS3::Configuration & base_configuration); + + static std::vector + listFilesMatchSuffix(const StorageS3::Configuration & base_configuration, const String & directory, const String & suffix); +}; + // Class to parse iceberg metadata and find files needed for query in table // Iceberg table directory outlook: // table/ // data/ // metadata/ // The metadata has three layers: metadata -> manifest list -> manifest files +template class IcebergMetaParser { public: - IcebergMetaParser(const StorageS3::Configuration & configuration_, ContextPtr context_); + IcebergMetaParser(const Configuration & configuration_, ContextPtr context_); std::vector getFiles() const; @@ -29,7 +39,7 @@ public: private: static constexpr auto metadata_directory = "metadata"; - StorageS3::Configuration base_configuration; + Configuration base_configuration; ContextPtr context; /// Just get file name @@ -47,7 +57,7 @@ struct StorageIcebergName static constexpr auto data_directory_prefix = "data"; }; -using StorageIceberg = IStorageDataLake; +using StorageIceberg = IStorageDataLake>; } #endif diff --git a/src/Storages/StorageS3.h b/src/Storages/StorageS3.h index f9e5b35dd60..40ebb6456fe 100644 --- a/src/Storages/StorageS3.h +++ b/src/Storages/StorageS3.h @@ -39,6 +39,8 @@ template class IStorageDataLake; struct StorageIcebergName; +struct S3MetaReadHelper; +template class IcebergMetaParser; struct StorageDeltaLakeName; @@ -317,7 +319,7 @@ private: friend class TableFunctionS3Cluster; friend class IStorageDataLake; friend class IStorageDataLake; - friend class IStorageDataLake; + friend class IStorageDataLake>; Configuration s3_configuration; std::vector keys; diff --git a/src/TableFunctions/ITableFunctionDataLake.h b/src/TableFunctions/ITableFunctionDataLake.h index e1ad1f2f665..0ce8cd1f8df 100644 --- a/src/TableFunctions/ITableFunctionDataLake.h +++ b/src/TableFunctions/ITableFunctionDataLake.h @@ -4,9 +4,7 @@ #if USE_AWS_S3 -#include -#include - +# include # include # include # include @@ -15,21 +13,19 @@ # include # include # include +# include # include # include +# include namespace DB { - namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } -class Context; -class TableFunctionS3Cluster; - -template +template class ITableFunctionDataLake : public ITableFunction { public: @@ -44,8 +40,6 @@ protected: executeImpl(const ASTPtr & /*ast_function*/, ContextPtr context, const std::string & table_name, ColumnsDescription /*cached_columns*/) const override { - S3::URI s3_uri(configuration.url); - ColumnsDescription columns; if (configuration.structure != "auto") columns = parseColumnsListFromString(configuration.structure, context); @@ -58,7 +52,7 @@ protected: return storage; } - const char * getStorageTypeName() const override { return name; } + const char * getStorageTypeName() const override { return Storage::name; } ColumnsDescription getActualTableStructure(ContextPtr context) const override { @@ -93,85 +87,14 @@ protected: auto & args = args_func.at(0)->children; - parseArgumentsImpl(message, args, context, configuration); + TableFunctionS3::parseArgumentsImpl(message, args, context, configuration); + + if (configuration.format == "auto") + configuration.format = "Parquet"; } - - static void - parseArgumentsImpl(const String & error_message, ASTs & args, ContextPtr context, StorageS3::Configuration & base_configuration) - { - if (args.empty() || args.size() > 6) - throw Exception::createDeprecated(error_message, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - - auto * header_it = StorageURL::collectHeaders(args, base_configuration.headers_from_ast, context); - if (header_it != args.end()) - args.erase(header_it); - - for (auto & arg : args) - arg = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context); - - /// Size -> argument indexes - static auto size_to_args = std::map>{ - {1, {{}}}, - {2, {{"format", 1}}}, - {5, {{"access_key_id", 1}, {"secret_access_key", 2}, {"format", 3}, {"structure", 4}}}, - {6, {{"access_key_id", 1}, {"secret_access_key", 2}, {"format", 3}, {"structure", 4}, {"compression_method", 5}}}}; - - std::map args_to_idx; - /// For 4 arguments we support 2 possible variants: - /// dataLake(source, format, structure, compression_method) and iceberg(source, access_key_id, access_key_id, format) - /// We can distinguish them by looking at the 2-nd argument: check if it's a format name or not. - if (args.size() == 4) - { - auto second_arg = checkAndGetLiteralArgument(args[1], "format/access_key_id"); - if (second_arg == "auto" || FormatFactory::instance().getAllFormats().contains(second_arg)) - args_to_idx = {{"format", 1}, {"structure", 2}, {"compression_method", 3}}; - - else - args_to_idx = {{"access_key_id", 1}, {"secret_access_key", 2}, {"format", 3}}; - } - /// For 3 arguments we support 2 possible variants: - /// dataLake(source, format, structure) and iceberg(source, access_key_id, access_key_id) - /// We can distinguish them by looking at the 2-nd argument: check if it's a format name or not. - else if (args.size() == 3) - { - auto second_arg = checkAndGetLiteralArgument(args[1], "format/access_key_id"); - if (second_arg == "auto" || FormatFactory::instance().getAllFormats().contains(second_arg)) - args_to_idx = {{"format", 1}, {"structure", 2}}; - else - args_to_idx = {{"access_key_id", 1}, {"secret_access_key", 2}}; - } - else - { - args_to_idx = size_to_args[args.size()]; - } - - /// This argument is always the first - base_configuration.url = S3::URI(checkAndGetLiteralArgument(args[0], "url")); - - if (args_to_idx.contains("format")) - base_configuration.format = checkAndGetLiteralArgument(args[args_to_idx["format"]], "format"); - else - base_configuration.format = "Parquet"; - - if (args_to_idx.contains("structure")) - base_configuration.structure = checkAndGetLiteralArgument(args[args_to_idx["structure"]], "structure"); - - if (args_to_idx.contains("compression_method")) - base_configuration.compression_method - = checkAndGetLiteralArgument(args[args_to_idx["compression_method"]], "compression_method"); - - if (args_to_idx.contains("access_key_id")) - base_configuration.auth_settings.access_key_id - = checkAndGetLiteralArgument(args[args_to_idx["access_key_id"]], "access_key_id"); - - if (args_to_idx.contains("secret_access_key")) - base_configuration.auth_settings.secret_access_key - = checkAndGetLiteralArgument(args[args_to_idx["secret_access_key"]], "secret_access_key"); - } - - mutable StorageS3::Configuration configuration; - }; + mutable Configuration configuration; +}; } #endif diff --git a/src/TableFunctions/TableFunctionDeltaLake.cpp b/src/TableFunctions/TableFunctionDeltaLake.cpp index d968c58bf94..a5602814abc 100644 --- a/src/TableFunctions/TableFunctionDeltaLake.cpp +++ b/src/TableFunctions/TableFunctionDeltaLake.cpp @@ -15,7 +15,7 @@ struct TableFunctionDeltaLakeName static constexpr auto name = "deltaLake"; }; -using TableFunctionDeltaLake = ITableFunctionDataLake; +using TableFunctionDeltaLake = ITableFunctionDataLake; void registerTableFunctionDeltaLake(TableFunctionFactory & factory) { diff --git a/src/TableFunctions/TableFunctionHudi.cpp b/src/TableFunctions/TableFunctionHudi.cpp index 1139b4d4a2e..d7ddc49c1de 100644 --- a/src/TableFunctions/TableFunctionHudi.cpp +++ b/src/TableFunctions/TableFunctionHudi.cpp @@ -14,7 +14,7 @@ struct TableFunctionHudiName { static constexpr auto name = "hudi"; }; -using TableFunctionHudi = ITableFunctionDataLake; +using TableFunctionHudi = ITableFunctionDataLake; void registerTableFunctionHudi(TableFunctionFactory & factory) { diff --git a/src/TableFunctions/TableFunctionIceberg.cpp b/src/TableFunctions/TableFunctionIceberg.cpp index 2555b197d30..eb25a115055 100644 --- a/src/TableFunctions/TableFunctionIceberg.cpp +++ b/src/TableFunctions/TableFunctionIceberg.cpp @@ -16,7 +16,7 @@ struct TableFunctionIcebergName static constexpr auto name = "iceberg"; }; -using TableFunctionIceberg = ITableFunctionDataLake; +using TableFunctionIceberg = ITableFunctionDataLake; void registerTableFunctionIceberg(TableFunctionFactory & factory) { diff --git a/src/TableFunctions/TableFunctionS3.cpp b/src/TableFunctions/TableFunctionS3.cpp index 0abddfef3e4..8f54b04f550 100644 --- a/src/TableFunctions/TableFunctionS3.cpp +++ b/src/TableFunctions/TableFunctionS3.cpp @@ -29,7 +29,9 @@ namespace ErrorCodes /// This is needed to avoid copy-pase. Because s3Cluster arguments only differ in additional argument (first) - cluster name -void TableFunctionS3::parseArgumentsImpl(const String & error_message, ASTs & args, ContextPtr context, StorageS3::Configuration & s3_configuration) +template +void TableFunctionS3::parseArgumentsImpl( + const String & error_message, ASTs & args, ContextPtr context, StorageS3::Configuration & s3_configuration) { if (auto named_collection = tryGetNamedCollectionWithOverrides(args)) { @@ -105,10 +107,14 @@ void TableFunctionS3::parseArgumentsImpl(const String & error_message, ASTs & ar s3_configuration.auth_settings.secret_access_key = checkAndGetLiteralArgument(args[args_to_idx["secret_access_key"]], "secret_access_key"); } - if (s3_configuration.format == "auto") + /// For DataLake table functions, we should specify default format. + if (s3_configuration.format == "auto" && get_format_from_file) s3_configuration.format = FormatFactory::instance().getFormatFromFileName(s3_configuration.url.uri.getPath(), true); } +template void TableFunctionS3::parseArgumentsImpl( + const String & error_message, ASTs & args, ContextPtr context, StorageS3::Configuration & s3_configuration); + void TableFunctionS3::parseArguments(const ASTPtr & ast_function, ContextPtr context) { /// Parse args diff --git a/src/TableFunctions/TableFunctionS3.h b/src/TableFunctions/TableFunctionS3.h index 2adddef538c..5f197a6e058 100644 --- a/src/TableFunctions/TableFunctionS3.h +++ b/src/TableFunctions/TableFunctionS3.h @@ -12,7 +12,6 @@ namespace DB { class Context; -class TableFunctionS3Cluster; /* s3(source, [access_key_id, secret_access_key,] format, structure[, compression]) - creates a temporary storage for a file in S3. */ @@ -37,8 +36,10 @@ public: return {"_path", "_file"}; } + template + static void parseArgumentsImpl(const String & error_message, ASTs & args, ContextPtr context, StorageS3::Configuration & configuration); + protected: - friend class TableFunctionS3Cluster; StoragePtr executeImpl( const ASTPtr & ast_function, @@ -51,8 +52,6 @@ protected: ColumnsDescription getActualTableStructure(ContextPtr context) const override; void parseArguments(const ASTPtr & ast_function, ContextPtr context) override; - static void parseArgumentsImpl(const String & error_message, ASTs & args, ContextPtr context, StorageS3::Configuration & configuration); - mutable StorageS3::Configuration configuration; ColumnsDescription structure_hint; }; diff --git a/src/TableFunctions/TableFunctionS3Cluster.cpp b/src/TableFunctions/TableFunctionS3Cluster.cpp index 6efe686ca5d..9b18bac536a 100644 --- a/src/TableFunctions/TableFunctionS3Cluster.cpp +++ b/src/TableFunctions/TableFunctionS3Cluster.cpp @@ -73,7 +73,7 @@ void TableFunctionS3Cluster::parseArguments(const ASTPtr & ast_function, Context std::copy(args.begin() + 1, args.end(), std::back_inserter(clipped_args)); /// StorageS3ClusterConfiguration inherints from StorageS3::Configuration, so it is safe to upcast it. - TableFunctionS3::parseArgumentsImpl(message.text, clipped_args, context, static_cast(configuration)); + TableFunctionS3::parseArgumentsImpl(message.text, clipped_args, context, static_cast(configuration)); } From 3060bee18a321a2cda50de866206abfad827a588 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Mon, 13 Feb 2023 12:58:33 +0100 Subject: [PATCH 173/717] Properly handle constants in multiple steps --- .../MergeTree/MergeTreeBaseSelectProcessor.cpp | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp index 5f40950c3b2..c7f098dcb1c 100644 --- a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp @@ -145,6 +145,7 @@ using OriginalToNewNodeMap = std::unordered_mapaddOrReplaceInOutputs(*node_ref.node); - const auto & new_node = new_dag->addInput(node_ref.node->result_name, node_ref.node->result_type); - node_remap[original_dag_node] = {new_dag, &new_node}; /// TODO: here we update the node reference. Is ti always correct? - return new_node; + /// If the node is known from the previous steps, add it as an input, except for constants + if (original_dag_node->type != ActionsDAG::ActionType::COLUMN) + { + node_ref.dag->addOrReplaceInOutputs(*node_ref.node); + const auto & new_node = new_dag->addInput(node_ref.node->result_name, node_ref.node->result_type); + node_remap[original_dag_node] = {new_dag, &new_node}; /// TODO: here we update the node reference. Is ti always correct? + return new_node; + } } /// If the node is an input, add it as an input From d5f413304dfe2b9807c6bbef530f803c14e86bdf Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Mon, 13 Feb 2023 13:28:09 +0100 Subject: [PATCH 174/717] Store null map size into a variable --- src/Columns/ColumnNullable.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Columns/ColumnNullable.cpp b/src/Columns/ColumnNullable.cpp index 99d377f10eb..139323f9770 100644 --- a/src/Columns/ColumnNullable.cpp +++ b/src/Columns/ColumnNullable.cpp @@ -786,16 +786,17 @@ ColumnPtr ColumnNullable::getNestedColumnWithDefaultOnNull() const auto res = nested_column->cloneEmpty(); const auto & null_map_data = getNullMapData(); size_t start = 0; + size_t end = null_map->size(); while (start < nested_column->size()) { size_t next_null_index = start; - while (next_null_index < null_map->size() && !null_map_data[next_null_index]) + while (next_null_index < end && !null_map_data[next_null_index]) ++next_null_index; if (next_null_index != start) res->insertRangeFrom(*nested_column, start, next_null_index - start); - if (next_null_index < null_map->size()) + if (next_null_index < end) res->insertDefault(); start = next_null_index + 1; From d67e7e47f5025744e4fdc291952303e646634684 Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Mon, 13 Feb 2023 13:28:46 +0100 Subject: [PATCH 175/717] Update src/Core/Settings.h --- src/Core/Settings.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 481929d915f..7fec350f208 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -751,7 +751,7 @@ class IColumn; M(Bool, input_format_csv_empty_as_default, true, "Treat empty fields in CSV input as default values.", 0) \ M(Bool, input_format_tsv_empty_as_default, false, "Treat empty fields in TSV input as default values.", 0) \ M(Bool, input_format_tsv_enum_as_number, false, "Treat inserted enum values in TSV formats as enum indices.", 0) \ - M(Bool, input_format_null_as_default, true, "For most input formats initialize null fields with default values if data type of this field is not nullable", 0) \ + M(Bool, input_format_null_as_default, true, "Initialize null fields with default values if the data type of this field is not nullable and it is supported by the input format", 0) \ M(Bool, input_format_arrow_import_nested, false, "Allow to insert array of structs into Nested table in Arrow input format.", 0) \ M(Bool, input_format_arrow_case_insensitive_column_matching, false, "Ignore case when matching Arrow columns with CH columns.", 0) \ M(Bool, input_format_orc_import_nested, false, "Allow to insert array of structs into Nested table in ORC input format.", 0) \ From c49a293a9f08d825c3c43744439f0d2edb373e6f Mon Sep 17 00:00:00 2001 From: flynn Date: Mon, 13 Feb 2023 12:39:54 +0000 Subject: [PATCH 176/717] refactor and get rid of s3 --- src/Storages/S3DataLakeMetaReadHelper.cpp | 71 +++++++++++++++++++++++ src/Storages/S3DataLakeMetaReadHelper.h | 36 ++++++++++++ src/Storages/StorageDeltaLake.cpp | 49 ++++++++-------- src/Storages/StorageDeltaLake.h | 10 ++-- src/Storages/StorageHudi.cpp | 54 +++++------------ src/Storages/StorageHudi.h | 9 ++- src/Storages/StorageIceberg.cpp | 37 +++--------- src/Storages/StorageIceberg.h | 16 +---- src/Storages/StorageS3.h | 11 ++-- 9 files changed, 171 insertions(+), 122 deletions(-) create mode 100644 src/Storages/S3DataLakeMetaReadHelper.cpp create mode 100644 src/Storages/S3DataLakeMetaReadHelper.h diff --git a/src/Storages/S3DataLakeMetaReadHelper.cpp b/src/Storages/S3DataLakeMetaReadHelper.cpp new file mode 100644 index 00000000000..fe5cd704132 --- /dev/null +++ b/src/Storages/S3DataLakeMetaReadHelper.cpp @@ -0,0 +1,71 @@ +#include + +#if USE_AWS_S3 + +# include + +namespace DB +{ +std::shared_ptr +S3DataLakeMetaReadHelper::createReadBuffer(const String & key, ContextPtr context, const StorageS3::Configuration & base_configuration) +{ + S3Settings::RequestSettings request_settings; + request_settings.max_single_read_retries = context->getSettingsRef().s3_max_single_read_retries; + return std::make_shared( + base_configuration.client, + base_configuration.url.bucket, + key, + base_configuration.url.version_id, + request_settings, + context->getReadSettings()); +} +std::vector S3DataLakeMetaReadHelper::listFilesMatchSuffix( + const StorageS3::Configuration & base_configuration, const String & directory, const String & suffix) +{ + const auto & table_path = base_configuration.url.key; + return S3::listFiles( + *base_configuration.client, base_configuration.url.bucket, table_path, std::filesystem::path(table_path) / directory, suffix); +} + +std::vector S3DataLakeMetaReadHelper::listFiles(const StorageS3::Configuration & configuration) +{ + const auto & client = configuration.client; + const auto & table_path = configuration.url.key; + const auto & bucket = configuration.url.bucket; + + std::vector keys; + S3::ListObjectsV2Request request; + Aws::S3::Model::ListObjectsV2Outcome outcome; + + bool is_finished{false}; + + request.SetBucket(bucket); + request.SetPrefix(table_path); + + while (!is_finished) + { + outcome = client->ListObjectsV2(request); + if (!outcome.IsSuccess()) + throw Exception( + ErrorCodes::S3_ERROR, + "Could not list objects in bucket {} with key {}, S3 exception: {}, message: {}", + quoteString(bucket), + quoteString(table_path), + backQuote(outcome.GetError().GetExceptionName()), + quoteString(outcome.GetError().GetMessage())); + + const auto & result_batch = outcome.GetResult().GetContents(); + for (const auto & obj : result_batch) + { + const auto & filename = obj.GetKey().substr(table_path.size()); /// Object name without tablepath prefix. + keys.push_back(filename); + } + + request.SetContinuationToken(outcome.GetResult().GetNextContinuationToken()); + is_finished = !outcome.GetResult().GetIsTruncated(); + } + + return keys; +} +} +#endif diff --git a/src/Storages/S3DataLakeMetaReadHelper.h b/src/Storages/S3DataLakeMetaReadHelper.h new file mode 100644 index 00000000000..5db2d51a2ec --- /dev/null +++ b/src/Storages/S3DataLakeMetaReadHelper.h @@ -0,0 +1,36 @@ +#pragma once + +#include + +#if USE_AWS_S3 + +# include +# include +# include + +# include + +# include +# include +# include +# include + + +class ReadBuffer; + +namespace DB +{ + +struct S3DataLakeMetaReadHelper +{ + static std::shared_ptr + createReadBuffer(const String & key, ContextPtr context, const StorageS3::Configuration & base_configuration); + + static std::vector + listFilesMatchSuffix(const StorageS3::Configuration & base_configuration, const String & directory, const String & suffix); + + static std::vector listFiles(const StorageS3::Configuration & configuration); +}; +} + +#endif diff --git a/src/Storages/StorageDeltaLake.cpp b/src/Storages/StorageDeltaLake.cpp index 6bcdb6b58b8..5bca8dc2138 100644 --- a/src/Storages/StorageDeltaLake.cpp +++ b/src/Storages/StorageDeltaLake.cpp @@ -56,20 +56,22 @@ std::vector DeltaLakeMetadata::listCurrentFiles() && return keys; } -DeltaLakeMetaParser::DeltaLakeMetaParser(const StorageS3::Configuration & configuration_, ContextPtr context) +template +DeltaLakeMetaParser::DeltaLakeMetaParser(const Configuration & configuration_, ContextPtr context) : base_configuration(configuration_) { init(context); } -void DeltaLakeMetaParser::init(ContextPtr context) +template +void DeltaLakeMetaParser::init(ContextPtr context) { auto keys = getJsonLogFiles(); // read data from every json log file for (const String & key : keys) { - auto buf = createS3ReadBuffer(key, context); + auto buf = MetaReadHelper::createReadBuffer(key, context, base_configuration); char c; while (!buf->eof()) @@ -93,35 +95,19 @@ void DeltaLakeMetaParser::init(ContextPtr context) } } -std::vector DeltaLakeMetaParser::getJsonLogFiles() const +template +std::vector DeltaLakeMetaParser::getJsonLogFiles() const { - const auto table_path = base_configuration.url.key; /// DeltaLake format stores all metadata json files in _delta_log directory static constexpr auto deltalake_metadata_directory = "_delta_log"; + static constexpr auto meta_file_suffix = ".json"; - return S3::listFiles( - *base_configuration.client, - base_configuration.url.bucket, - table_path, - std::filesystem::path(table_path) / deltalake_metadata_directory, - ".json"); + return MetaReadHelper::listFilesMatchSuffix(base_configuration, deltalake_metadata_directory, meta_file_suffix); } -std::shared_ptr DeltaLakeMetaParser::createS3ReadBuffer(const String & key, ContextPtr context) -{ - S3Settings::RequestSettings request_settings; - request_settings.max_single_read_retries = context->getSettingsRef().s3_max_single_read_retries; - return std::make_shared( - base_configuration.client, - base_configuration.url.bucket, - key, - base_configuration.url.version_id, - request_settings, - context->getReadSettings()); -} - -void DeltaLakeMetaParser::handleJSON(const JSON & json) +template +void DeltaLakeMetaParser::handleJSON(const JSON & json) { if (json.has("add")) { @@ -139,16 +125,27 @@ void DeltaLakeMetaParser::handleJSON(const JSON & json) } } + // DeltaLake stores data in parts in different files // keys is vector of parts with latest version // generateQueryFromKeys constructs query from parts filenames for // underlying StorageS3 engine -String DeltaLakeMetaParser::generateQueryFromKeys(const std::vector & keys, const String &) +template +String DeltaLakeMetaParser::generateQueryFromKeys(const std::vector & keys, const String &) { std::string new_query = fmt::format("{{{}}}", fmt::join(keys, ",")); return new_query; } +template DeltaLakeMetaParser::DeltaLakeMetaParser( + const StorageS3::Configuration & configuration_, ContextPtr context); +template std::vector DeltaLakeMetaParser::getFiles(); +template String DeltaLakeMetaParser::generateQueryFromKeys( + const std::vector & keys, const String & format); +template void DeltaLakeMetaParser::init(ContextPtr context); +template std::vector DeltaLakeMetaParser::getJsonLogFiles() const; +template void DeltaLakeMetaParser::handleJSON(const JSON & json); + void registerStorageDeltaLake(StorageFactory & factory) { factory.registerStorage( diff --git a/src/Storages/StorageDeltaLake.h b/src/Storages/StorageDeltaLake.h index c5b0d58a08e..e14725dd165 100644 --- a/src/Storages/StorageDeltaLake.h +++ b/src/Storages/StorageDeltaLake.h @@ -5,6 +5,7 @@ #if USE_AWS_S3 # include +# include # include # include @@ -28,10 +29,11 @@ private: }; // class to get deltalake log json files and read json from them +template class DeltaLakeMetaParser { public: - DeltaLakeMetaParser(const StorageS3::Configuration & configuration_, ContextPtr context); + DeltaLakeMetaParser(const Configuration & configuration_, ContextPtr context); std::vector getFiles() { return std::move(metadata).listCurrentFiles(); } @@ -42,11 +44,9 @@ private: std::vector getJsonLogFiles() const; - std::shared_ptr createS3ReadBuffer(const String & key, ContextPtr context); - void handleJSON(const JSON & json); - StorageS3::Configuration base_configuration; + Configuration base_configuration; DeltaLakeMetadata metadata; }; @@ -56,7 +56,7 @@ struct StorageDeltaLakeName static constexpr auto data_directory_prefix = ""; }; -using StorageDeltaLake = IStorageDataLake; +using StorageDeltaLake = IStorageDataLake>; } #endif diff --git a/src/Storages/StorageHudi.cpp b/src/Storages/StorageHudi.cpp index 68a65d3b757..62b0edb3c9a 100644 --- a/src/Storages/StorageHudi.cpp +++ b/src/Storages/StorageHudi.cpp @@ -26,7 +26,8 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -HudiMetaParser::HudiMetaParser(const StorageS3::Configuration & configuration_, ContextPtr context_) +template +HudiMetaParser::HudiMetaParser(const Configuration & configuration_, ContextPtr context_) : configuration(configuration_), context(context_), log(&Poco::Logger::get("StorageHudi")) { } @@ -36,7 +37,8 @@ HudiMetaParser::HudiMetaParser(const StorageS3::Configuration & configuration_, /// Every partition(directory) in Apache Hudi has different versions of part. /// To find needed parts we need to find out latest part file for every partition. /// Part format is usually parquet, but can differ. -String HudiMetaParser::generateQueryFromKeys(const std::vector & keys, const String & format) +template +String HudiMetaParser::generateQueryFromKeys(const std::vector & keys, const String & format) { /// For each partition path take only latest file. struct FileInfo @@ -87,48 +89,18 @@ String HudiMetaParser::generateQueryFromKeys(const std::vector & ke return "{" + list_of_keys + "}"; } -std::vector HudiMetaParser::getFiles() const +template +std::vector HudiMetaParser::getFiles() const { - const auto & client = configuration.client; - const auto & table_path = configuration.url.key; - const auto & bucket = configuration.url.bucket; - - std::vector keys; - S3::ListObjectsV2Request request; - Aws::S3::Model::ListObjectsV2Outcome outcome; - - bool is_finished{false}; - - request.SetBucket(bucket); - request.SetPrefix(table_path); - - while (!is_finished) - { - outcome = client->ListObjectsV2(request); - if (!outcome.IsSuccess()) - throw Exception( - ErrorCodes::S3_ERROR, - "Could not list objects in bucket {} with key {}, S3 exception: {}, message: {}", - quoteString(bucket), - quoteString(table_path), - backQuote(outcome.GetError().GetExceptionName()), - quoteString(outcome.GetError().GetMessage())); - - const auto & result_batch = outcome.GetResult().GetContents(); - for (const auto & obj : result_batch) - { - const auto & filename = obj.GetKey().substr(table_path.size()); /// Object name without tablepath prefix. - keys.push_back(filename); - LOG_DEBUG(log, "Found file: {}", filename); - } - - request.SetContinuationToken(outcome.GetResult().GetNextContinuationToken()); - is_finished = !outcome.GetResult().GetIsTruncated(); - } - - return keys; + return MetaReadHelper::listFiles(configuration); } +template HudiMetaParser::HudiMetaParser( + const StorageS3::Configuration & configuration_, ContextPtr context_); +template std::vector HudiMetaParser::getFiles() const; +template String HudiMetaParser::generateQueryFromKeys( + const std::vector & keys, const String & format); + void registerStorageHudi(StorageFactory & factory) { factory.registerStorage( diff --git a/src/Storages/StorageHudi.h b/src/Storages/StorageHudi.h index 9a0c98e249a..88e4956bca1 100644 --- a/src/Storages/StorageHudi.h +++ b/src/Storages/StorageHudi.h @@ -6,22 +6,25 @@ # include # include +# include # include namespace DB { + +template class HudiMetaParser { public: - HudiMetaParser(const StorageS3::Configuration & configuration_, ContextPtr context_); + HudiMetaParser(const Configuration & configuration_, ContextPtr context_); std::vector getFiles() const; static String generateQueryFromKeys(const std::vector & keys, const String & format); private: - StorageS3::Configuration configuration; + Configuration configuration; ContextPtr context; Poco::Logger * log; }; @@ -32,7 +35,7 @@ struct StorageHudiName static constexpr auto data_directory_prefix = ""; }; -using StorageHudi = IStorageDataLake; +using StorageHudi = IStorageDataLake>; } #endif diff --git a/src/Storages/StorageIceberg.cpp b/src/Storages/StorageIceberg.cpp index ec5284adf94..b9e23cb9fe6 100644 --- a/src/Storages/StorageIceberg.cpp +++ b/src/Storages/StorageIceberg.cpp @@ -46,27 +46,6 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; } -std::shared_ptr -S3MetaReadHelper::createReadBuffer(const String & key, ContextPtr context, const StorageS3::Configuration & base_configuration) -{ - S3Settings::RequestSettings request_settings; - request_settings.max_single_read_retries = context->getSettingsRef().s3_max_single_read_retries; - return std::make_shared( - base_configuration.client, - base_configuration.url.bucket, - key, - base_configuration.url.version_id, - request_settings, - context->getReadSettings()); -} -std::vector -S3MetaReadHelper::listFilesMatchSuffix(const StorageS3::Configuration & base_configuration, const String & directory, const String & suffix) -{ - const auto & table_path = base_configuration.url.key; - return S3::listFiles( - *base_configuration.client, base_configuration.url.bucket, table_path, std::filesystem::path(table_path) / directory, suffix); -} - template IcebergMetaParser::IcebergMetaParser(const Configuration & configuration_, ContextPtr context_) : base_configuration(configuration_), context(context_) @@ -95,7 +74,7 @@ String IcebergMetaParser::getNewestMetaFile() con /// Iceberg stores all the metadata.json in metadata directory, and the /// newest version has the max version name, so we should list all of them /// then find the newest metadata. - constexpr auto meta_file_suffix = ".json"; + static constexpr auto meta_file_suffix = ".json"; auto metadata_files = MetaReadHelper::listFilesMatchSuffix(base_configuration, metadata_directory, meta_file_suffix); if (metadata_files.empty()) @@ -254,17 +233,17 @@ String IcebergMetaParser::generateQueryFromKeys(c return new_query; } -template IcebergMetaParser::IcebergMetaParser( +template IcebergMetaParser::IcebergMetaParser( const StorageS3::Configuration & configuration_, ContextPtr context_); -template std::vector IcebergMetaParser::getFiles() const; -template String IcebergMetaParser::generateQueryFromKeys( +template std::vector IcebergMetaParser::getFiles() const; +template String IcebergMetaParser::generateQueryFromKeys( const std::vector & keys, const String & format); -template String IcebergMetaParser::getNewestMetaFile() const; -template String IcebergMetaParser::getManiFestList(const String & metadata_name) const; +template String IcebergMetaParser::getNewestMetaFile() const; +template String IcebergMetaParser::getManiFestList(const String & metadata_name) const; template std::vector -IcebergMetaParser::getManifestFiles(const String & manifest_list) const; +IcebergMetaParser::getManifestFiles(const String & manifest_list) const; template std::vector -IcebergMetaParser::getFilesForRead(const std::vector & manifest_files) const; +IcebergMetaParser::getFilesForRead(const std::vector & manifest_files) const; void registerStorageIceberg(StorageFactory & factory) { diff --git a/src/Storages/StorageIceberg.h b/src/Storages/StorageIceberg.h index 8a535643fce..a32858d1375 100644 --- a/src/Storages/StorageIceberg.h +++ b/src/Storages/StorageIceberg.h @@ -5,22 +5,11 @@ #if USE_AWS_S3 # include - -# include -# include +# include namespace DB { -struct S3MetaReadHelper -{ - static std::shared_ptr - createReadBuffer(const String & key, ContextPtr context, const StorageS3::Configuration & base_configuration); - - static std::vector - listFilesMatchSuffix(const StorageS3::Configuration & base_configuration, const String & directory, const String & suffix); -}; - // Class to parse iceberg metadata and find files needed for query in table // Iceberg table directory outlook: // table/ @@ -42,7 +31,6 @@ private: Configuration base_configuration; ContextPtr context; - /// Just get file name String getNewestMetaFile() const; String getManiFestList(const String & metadata_name) const; std::vector getManifestFiles(const String & manifest_list) const; @@ -57,7 +45,7 @@ struct StorageIcebergName static constexpr auto data_directory_prefix = "data"; }; -using StorageIceberg = IStorageDataLake>; +using StorageIceberg = IStorageDataLake>; } #endif diff --git a/src/Storages/StorageS3.h b/src/Storages/StorageS3.h index 40ebb6456fe..8194feaf754 100644 --- a/src/Storages/StorageS3.h +++ b/src/Storages/StorageS3.h @@ -38,15 +38,18 @@ class NamedCollection; template class IStorageDataLake; +struct S3DataLakeMetaReadHelper; + struct StorageIcebergName; -struct S3MetaReadHelper; template class IcebergMetaParser; struct StorageDeltaLakeName; +template class DeltaLakeMetaParser; struct StorageHudiName; +template class HudiMetaParser; class StorageS3Source : public ISource, WithContext @@ -317,9 +320,9 @@ public: private: friend class StorageS3Cluster; friend class TableFunctionS3Cluster; - friend class IStorageDataLake; - friend class IStorageDataLake; - friend class IStorageDataLake>; + friend class IStorageDataLake>; + friend class IStorageDataLake>; + friend class IStorageDataLake>; Configuration s3_configuration; std::vector keys; From c39cfee9548145f7436f62f51cd91a1ec33eb99e Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Mon, 13 Feb 2023 14:11:18 +0100 Subject: [PATCH 177/717] Properly set "remove" flag for step prewhere_columns when they are used in futher steps --- .../MergeTreeBaseSelectProcessor.cpp | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp index c7f098dcb1c..fa8b5d40ff2 100644 --- a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp @@ -315,9 +315,11 @@ bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionAction auto original_outputs = prewhere_info->prewhere_actions->getOutputs(); /// 7. Find all outputs that were computed in the already built DAGs, mark these nodes as outputs in the steps where they were computed /// 8. Add computation of the remaining outputs to the last step with the procedure similar to 4 + NameSet all_output_names; for (const auto * output : original_outputs) { // std::cerr << "Original output: " << output->result_name << std::endl; + all_output_names.insert(output->result_name); if (node_remap.contains(output)) { const auto & new_node_info = node_remap[output]; @@ -325,14 +327,12 @@ bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionAction } else if (output->result_name == prewhere_info->prewhere_column_name) { - /// Special case for final PREWHERE column: - /// "Rename" the last step result to the combined PREWHERE column name, because in fact it will be AND of all step results - const auto & prewhere_result_node = addCast( - steps.back().actions, - steps.back().actions->findInOutputs(steps.back().column_name), - output->result_type->getName(), - prewhere_info->prewhere_column_name); - + /// Special case for final PREWHERE column: it is an AND combination of all conditions, + /// but we have only the condition for the last step here. + /// However we know that the ultimate result after filtering is constant 1 for the PREWHERE column. + auto const_true = output->result_type->createColumnConst(0, Field{1}); + const auto & prewhere_result_node = + steps.back().actions->addColumn(ColumnWithTypeAndName(const_true, output->result_type, output->result_name)); steps.back().actions->addOrReplaceInOutputs(prewhere_result_node); } else @@ -351,11 +351,10 @@ bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionAction { .actions = std::make_shared(step.actions, actions_settings), .column_name = step.column_name, - .remove_column = true, + .remove_column = !all_output_names.contains(step.column_name), /// Don't remove if it's in the list of original outputs .need_filter = false, }); } - prewhere.steps.back().remove_column = prewhere_info->remove_prewhere_column; prewhere.steps.back().need_filter = prewhere_info->need_filter; } @@ -385,7 +384,8 @@ std::unique_ptr IMergeTreeSelectAlgorithm::getPrewhereActions( // std::cerr << "ORIGINAL PREWHERE:\n" << prewhere_info->prewhere_actions->dumpDAG() << std::endl; #if 1 - if (!enable_multiple_prewhere_read_steps || !tryBuildPrewhereSteps(prewhere_info, actions_settings, *prewhere_actions)) + if (!enable_multiple_prewhere_read_steps || + !tryBuildPrewhereSteps(prewhere_info, actions_settings, *prewhere_actions)) { PrewhereExprStep prewhere_step { From 39e4ca9850fd488ba21994960daaccb85d695558 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Mon, 13 Feb 2023 14:40:54 +0100 Subject: [PATCH 178/717] Cleanup --- src/Interpreters/ActionsDAG.cpp | 20 +-- src/Interpreters/ActionsDAG.h | 14 -- .../MergeTreeBaseSelectProcessor.cpp | 121 ------------------ 3 files changed, 10 insertions(+), 145 deletions(-) diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index 913a891c6bb..160772d38ea 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -1663,20 +1663,20 @@ ActionsDAG::SplitResult ActionsDAG::splitActionsForFilter(const std::string & co return res; } -//namespace -//{ -// -//struct ConjunctionNodes -//{ -// ActionsDAG::NodeRawConstPtrs allowed; -// ActionsDAG::NodeRawConstPtrs rejected; -//}; +namespace +{ + +struct ConjunctionNodes +{ + ActionsDAG::NodeRawConstPtrs allowed; + ActionsDAG::NodeRawConstPtrs rejected; +}; /// Take a node which result is predicate. /// Assuming predicate is a conjunction (probably, trivial). /// Find separate conjunctions nodes. Split nodes into allowed and rejected sets. /// Allowed predicate is a predicate which can be calculated using only nodes from allowed_nodes set. -ConjunctionNodes getConjunctionNodes(const ActionsDAG::Node * predicate, std::unordered_set allowed_nodes) +ConjunctionNodes getConjunctionNodes(ActionsDAG::Node * predicate, std::unordered_set allowed_nodes) { ConjunctionNodes conjunction; std::unordered_set allowed; @@ -1798,7 +1798,7 @@ ColumnsWithTypeAndName prepareFunctionArguments(const ActionsDAG::NodeRawConstPt return arguments; } -//} +} /// Create actions which calculate conjunction of selected nodes. /// Assume conjunction nodes are predicates (and may be used as arguments of function AND). diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index 98c5d36c69a..40bc76fe057 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -363,7 +363,6 @@ private: void compileFunctions(size_t min_count_to_compile_expression, const std::unordered_set & lazy_executed_nodes = {}); #endif -public: static ActionsDAGPtr cloneActionsForConjunction(NodeRawConstPtrs conjunction, const ColumnsWithTypeAndName & all_inputs); }; @@ -373,17 +372,4 @@ struct ActionDAGNodes ActionsDAG::NodeRawConstPtrs nodes; }; -struct ConjunctionNodes -{ - ActionsDAG::NodeRawConstPtrs allowed; - ActionsDAG::NodeRawConstPtrs rejected; -}; - -/// Take a node which result is predicate. -/// Assuming predicate is a conjunction (probably, trivial). -/// Find separate conjunctions nodes. Split nodes into allowed and rejected sets. -/// Allowed predicate is a predicate which can be calculated using only nodes from allowed_nodes set. -ConjunctionNodes getConjunctionNodes(const ActionsDAG::Node * predicate, std::unordered_set allowed_nodes); - - } diff --git a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp index fa8b5d40ff2..3280ec6e487 100644 --- a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp @@ -13,9 +13,6 @@ #include #include -#include - -/// For CAST to bool #include #include #include @@ -383,7 +380,6 @@ std::unique_ptr IMergeTreeSelectAlgorithm::getPrewhereActions( // std::cerr << "ORIGINAL PREWHERE:\n" << prewhere_info->prewhere_actions->dumpDAG() << std::endl; -#if 1 if (!enable_multiple_prewhere_read_steps || !tryBuildPrewhereSteps(prewhere_info, actions_settings, *prewhere_actions)) { @@ -397,123 +393,6 @@ std::unique_ptr IMergeTreeSelectAlgorithm::getPrewhereActions( prewhere_actions->steps.emplace_back(std::move(prewhere_step)); } - -#else - - struct Step - { - ActionsDAGPtr actions; - String column_name; - }; - std::vector steps; - - if (enable_multiple_prewhere_read_steps) - { - /// Find all conjunctions in prewhere expression. - auto conjunctions = getConjunctionNodes( - prewhere_info->prewhere_actions->tryFindInOutputs(prewhere_info->prewhere_column_name), - {}); - - /// Save the list of inputs to the original prewhere expression. - auto inputs = prewhere_info->prewhere_actions->getInputs(); - ColumnsWithTypeAndName all_inputs; - for (const auto & input : inputs) - all_inputs.emplace_back(input->column, input->result_type, input->result_name); - - ActionsDAG::NodeRawConstPtrs all_conjunctions = std::move(conjunctions.allowed); - all_conjunctions.insert(all_conjunctions.end(), conjunctions.rejected.begin(), conjunctions.rejected.end()); - - /// Make separate DAG for each step - for (const auto & conjunction : all_conjunctions) - { - auto result_name = conjunction->result_name; - auto step_dag = ActionsDAG::cloneActionsForConjunction({conjunction}, all_inputs); - const auto & result_node = step_dag->findInOutputs(result_name); - /// Cast result to UInt8 if needed - if (result_node.result_type->getTypeId() != TypeIndex::UInt8) - { - const auto & cast_node = addCast(step_dag, result_node, "UInt8"); - - step_dag->addOrReplaceInOutputs(cast_node); - result_name = cast_node.result_name; - } - step_dag->removeUnusedActions(Names{result_name}, true, true); - steps.emplace_back(Step{step_dag, result_name}); - } - } - - if (steps.size() > 1) - { - /// Save the list of outputs from the original prewhere expression. - auto original_outputs = prewhere_info->prewhere_actions->getOutputs(); - std::unordered_map outputs_required_by_next_steps; - for (const auto & output : original_outputs) - outputs_required_by_next_steps[output->result_name] = output->result_type; - - /// "Rename" the last step result to the combined prewhere column name, because in fact it will be AND of all step results - if (steps.back().column_name != prewhere_info->prewhere_column_name && - outputs_required_by_next_steps.contains(prewhere_info->prewhere_column_name)) - { - const auto & prewhere_result_node = addCast( - steps.back().actions, - steps.back().actions->findInOutputs(steps.back().column_name), - outputs_required_by_next_steps[prewhere_info->prewhere_column_name]->getName(), - prewhere_info->prewhere_column_name); - - steps.back().actions->addOrReplaceInOutputs(prewhere_result_node); - } - - const size_t steps_before_prewhere = prewhere_actions->steps.size(); - prewhere_actions->steps.resize(steps_before_prewhere + steps.size()); - - /// Check the steps in the reverse order so that we can maintain the list of outputs used by the next steps - /// and preserve them in the current step. - for (ssize_t i = steps.size() - 1; i >= 0; --i) - { - const auto & step = steps[i]; - - /// Return the condition column - Names step_outputs{step.column_name}; - const bool remove_column = !outputs_required_by_next_steps.contains(step.column_name); - /// Preserve outputs computed at this step that are used by the next steps - for (const auto & output : outputs_required_by_next_steps) - if (step.actions->tryRestoreColumn(output.first)) - step_outputs.emplace_back(output.first); - step.actions->removeUnusedActions(step_outputs, true, true); - - /// Add current step columns as outputs that should be preserved from previous steps - for (const auto & input :step.actions->getInputs()) - outputs_required_by_next_steps[input->result_name] = input->result_type; - - //std::cerr << conjunction->result_name << "\n"; - //std::cerr << "STEP " << i << ":\n" << step.actions->dumpDAG() << "\n"; - - PrewhereExprStep prewhere_step - { - .actions = std::make_shared(step.actions, actions_settings), - .column_name = step.column_name, - .remove_column = remove_column, - .need_filter = false - }; - prewhere_actions->steps[steps_before_prewhere + i] = std::move(prewhere_step); - } - - prewhere_actions->steps.back().remove_column = prewhere_info->remove_prewhere_column; - prewhere_actions->steps.back().need_filter = prewhere_info->need_filter; - } - else - { - PrewhereExprStep prewhere_step - { - .actions = std::make_shared(prewhere_info->prewhere_actions, actions_settings), - .column_name = prewhere_info->prewhere_column_name, - .remove_column = prewhere_info->remove_prewhere_column, - .need_filter = prewhere_info->need_filter - }; - - prewhere_actions->steps.emplace_back(std::move(prewhere_step)); - } -#endif } return prewhere_actions; From f0011862a984384c80de005f4dae1c2a554965ad Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Mon, 13 Feb 2023 15:28:54 +0100 Subject: [PATCH 179/717] Move prewhere splitting logic into a separate file --- .../MergeTreeBaseSelectProcessor.cpp | 278 +--------------- .../MergeTreeSplitPrewhereIntoReadSteps.cpp | 302 ++++++++++++++++++ 2 files changed, 303 insertions(+), 277 deletions(-) create mode 100644 src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp diff --git a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp index 3280ec6e487..fe649808cbe 100644 --- a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp @@ -12,11 +12,6 @@ #include #include #include - -#include -#include -#include - #include namespace ProfileEvents @@ -87,276 +82,7 @@ IMergeTreeSelectAlgorithm::IMergeTreeSelectAlgorithm( LOG_TEST(log, "PREWHERE actions: {}", (prewhere_actions ? prewhere_actions->dump() : std::string(""))); } -/// Adds a CAST node with the regular name ("CAST(...)") or with the provided name. -/// This is different from ActionsDAG::addCast() because it set the name equal to the original name effectively hiding the value before cast, -/// but it might be required for further steps with its original uncasted type. -static const ActionsDAG::Node & addCast(ActionsDAGPtr dag, const ActionsDAG::Node & node_to_cast, const String & type_name, const String & new_name = {}) -{ - Field cast_type_constant_value(type_name); - - ColumnWithTypeAndName column; - column.name = calculateConstantActionNodeName(cast_type_constant_value); - column.column = DataTypeString().createColumnConst(0, cast_type_constant_value); - column.type = std::make_shared(); - - const auto * cast_type_constant_node = &dag->addColumn(std::move(column)); - ActionsDAG::NodeRawConstPtrs children = {&node_to_cast, cast_type_constant_node}; - FunctionOverloadResolverPtr func_builder_cast = CastInternalOverloadResolver::createImpl(); - - return dag->addFunction(func_builder_cast, std::move(children), new_name); -}; - -struct NodeInfo -{ - NameSet required_columns; -}; - -void fillRequiredColumns(const ActionsDAG::Node * node, std::unordered_map & nodes_info) -{ - if (nodes_info.contains(node)) - return; - - auto & node_info = nodes_info[node]; - - if (node->type == ActionsDAG::ActionType::INPUT) - { - node_info.required_columns.insert(node->result_name); - return; - } - - for (const auto & child : node->children) - { - fillRequiredColumns(child, nodes_info); - const auto & child_info = nodes_info[child]; - node_info.required_columns.insert(child_info.required_columns.begin(), child_info.required_columns.end()); - } -} - -struct DAGNodeRef -{ - ActionsDAGPtr dag; - const ActionsDAG::Node * node; -}; - -using OriginalToNewNodeMap = std::unordered_map; - -const ActionsDAG::Node & addClonedDAGToDAG(const ActionsDAG::Node * original_dag_node, ActionsDAGPtr new_dag, OriginalToNewNodeMap & node_remap) -{ - /// Look for the node in the map of already known nodes - if (node_remap.contains(original_dag_node)) - { - /// If the node is already in the new DAG, return it - const auto & node_ref = node_remap.at(original_dag_node); - if (node_ref.dag == new_dag) - return *node_ref.node; - - /// If the node is known from the previous steps, add it as an input, except for constants - if (original_dag_node->type != ActionsDAG::ActionType::COLUMN) - { - node_ref.dag->addOrReplaceInOutputs(*node_ref.node); - const auto & new_node = new_dag->addInput(node_ref.node->result_name, node_ref.node->result_type); - node_remap[original_dag_node] = {new_dag, &new_node}; /// TODO: here we update the node reference. Is ti always correct? - return new_node; - } - } - - /// If the node is an input, add it as an input - if (original_dag_node->type == ActionsDAG::ActionType::INPUT) - { - const auto & new_node = new_dag->addInput(original_dag_node->result_name, original_dag_node->result_type); - node_remap[original_dag_node] = {new_dag, &new_node}; - return new_node; - } - - /// If the node is a column, add it as an input - if (original_dag_node->type == ActionsDAG::ActionType::COLUMN) - { - const auto & new_node = new_dag->addColumn( - ColumnWithTypeAndName(original_dag_node->column, original_dag_node->result_type, original_dag_node->result_name)); - node_remap[original_dag_node] = {new_dag, &new_node}; - return new_node; - } - - /// TODO: Alias node? - - /// If the node is a function, add it as a function and add its children - if (original_dag_node->type == ActionsDAG::ActionType::FUNCTION) - { - ActionsDAG::NodeRawConstPtrs new_children; - for (const auto & child : original_dag_node->children) - { - const auto & new_child = addClonedDAGToDAG(child, new_dag, node_remap); - new_children.push_back(&new_child); - } - - const auto & new_node = new_dag->addFunction(original_dag_node->function_base, new_children, original_dag_node->result_name); - node_remap[original_dag_node] = {new_dag, &new_node}; - return new_node; - } - - throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected node type in PREWHERE actions: {}", original_dag_node->type); -} - -bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionActionsSettings & actions_settings, PrewhereExprInfo & prewhere) -{ - /// We want to build a sequence of steps that will compute parts of the prewhere condition. - /// Each step reads some new columns and computes some new expressions and a filter condition. - /// The last step computes the final filter condition and the remaining expressions that are required for the main query. - /// The steps are built in the following way: - /// 1. List all condition nodes that are combined with AND into PREWHERE condition - /// 2. Collect the set of columns that are used in the condition - /// 3. Sort condition nodes by the number of columns used in them and the overall size of those columns - /// 4. Group conditions with the same set of columns into a single read/compute step - /// 5. Build DAGs for each step: - /// - DFS from the condition root node: - /// - If the node was not computed yet, add it to the DAG and traverse its children - /// - If the node was already computed by one of the previous steps, add it as output for that step and as input for the current step - /// - If the node was already computed by the current step just stop traversing - /// 6. Find all outputs of the original DAG - /// 7. Find all outputs that were computed in the already built DAGs, mark these nodes as outputs in the steps where they were computed - /// 8. Add computation of the remaining outputs to the last step with the procedure similar to 4 - - if (!prewhere_info || !prewhere_info->prewhere_actions) - return true; - - /// 1. List all condition nodes that are combined with AND into PREWHERE condition - const auto & condition_root = prewhere_info->prewhere_actions->findInOutputs(prewhere_info->prewhere_column_name); - const bool is_conjunction = (condition_root.type == ActionsDAG::ActionType::FUNCTION && condition_root.function_base->getName() == "and"); - if (!is_conjunction) - return false; - auto condition_nodes = condition_root.children; - - /// 2. Collect the set of columns that are used in the condition - std::unordered_map nodes_info; - for (const auto & node : condition_nodes) - { - fillRequiredColumns(node, nodes_info); - } - - /// 3. Sort condition nodes by the number of columns used in them and the overall size of those columns - /// TODO: not sorting for now because the conditions are already sorted by Where Optimizer - - /// 4. Group conditions with the same set of columns into a single read/compute step - std::vector> condition_groups; - for (const auto & node : condition_nodes) - { - const auto & node_info = nodes_info[node]; - if (!condition_groups.empty() && nodes_info[condition_groups.back().back()].required_columns == node_info.required_columns) - condition_groups.back().push_back(node); /// Add to the last group - else - condition_groups.push_back({node}); /// Start new group - } - - /// 5. Build DAGs for each step - struct Step - { - ActionsDAGPtr actions; - String column_name; - }; - std::vector steps; - - OriginalToNewNodeMap node_remap; - - for (const auto & condition_group : condition_groups) - { -// std::cerr -// << "Conditions: ["; -// -// for (const auto & condition : condition_group) -// std::cerr << " \"" << condition->result_name; -// -// std::cerr << "\" ] Columns: " << boost::algorithm::join(nodes_info[condition_group.front()].required_columns, " ") -// << std::endl; - - ActionsDAGPtr step_dag = std::make_shared(); - String result_name; - - std::vector new_condition_nodes; - for (const auto * node : condition_group) - { - const auto & node_in_new_dag = addClonedDAGToDAG(node, step_dag, node_remap); - new_condition_nodes.push_back(&node_in_new_dag); - } - - if (new_condition_nodes.size() > 1) - { - /// Add AND function to combine the conditions - FunctionOverloadResolverPtr func_builder_and = std::make_unique(std::make_shared()); - const auto & and_function_node = step_dag->addFunction(func_builder_and, new_condition_nodes, ""); - step_dag->addOrReplaceInOutputs(and_function_node); - result_name = and_function_node.result_name; - } - else - { - const auto & result_node = *new_condition_nodes.front(); - /// Add cast to UInt8 if needed - if (result_node.result_type->getTypeId() == TypeIndex::UInt8) - { - step_dag->addOrReplaceInOutputs(result_node); - result_name = result_node.result_name; - } - else - { - const auto & cast_node = addCast(step_dag, result_node, "UInt8"); - step_dag->addOrReplaceInOutputs(cast_node); - result_name = cast_node.result_name; - } - } - -// std::cerr << "Step DAG:\n" << step_dag->dumpDAG() << std::endl; - - steps.push_back({step_dag, result_name}); - } - - /// 6. Find all outputs of the original DAG - auto original_outputs = prewhere_info->prewhere_actions->getOutputs(); - /// 7. Find all outputs that were computed in the already built DAGs, mark these nodes as outputs in the steps where they were computed - /// 8. Add computation of the remaining outputs to the last step with the procedure similar to 4 - NameSet all_output_names; - for (const auto * output : original_outputs) - { -// std::cerr << "Original output: " << output->result_name << std::endl; - all_output_names.insert(output->result_name); - if (node_remap.contains(output)) - { - const auto & new_node_info = node_remap[output]; - new_node_info.dag->addOrReplaceInOutputs(*new_node_info.node); - } - else if (output->result_name == prewhere_info->prewhere_column_name) - { - /// Special case for final PREWHERE column: it is an AND combination of all conditions, - /// but we have only the condition for the last step here. - /// However we know that the ultimate result after filtering is constant 1 for the PREWHERE column. - auto const_true = output->result_type->createColumnConst(0, Field{1}); - const auto & prewhere_result_node = - steps.back().actions->addColumn(ColumnWithTypeAndName(const_true, output->result_type, output->result_name)); - steps.back().actions->addOrReplaceInOutputs(prewhere_result_node); - } - else - { - const auto & node_in_new_dag = addClonedDAGToDAG(output, steps.back().actions, node_remap); - steps.back().actions->addOrReplaceInOutputs(node_in_new_dag); - } - } - - /// 9. Build PrewhereExprInfo - { - for (const auto & step : steps) - { -// std::cerr << "Step DAG:\n" << step.actions->dumpDAG() << std::endl; - prewhere.steps.push_back( - { - .actions = std::make_shared(step.actions, actions_settings), - .column_name = step.column_name, - .remove_column = !all_output_names.contains(step.column_name), /// Don't remove if it's in the list of original outputs - .need_filter = false, - }); - } - prewhere.steps.back().need_filter = prewhere_info->need_filter; - } - - return true; -} +bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionActionsSettings & actions_settings, PrewhereExprInfo & prewhere); std::unique_ptr IMergeTreeSelectAlgorithm::getPrewhereActions(PrewhereInfoPtr prewhere_info, const ExpressionActionsSettings & actions_settings, bool enable_multiple_prewhere_read_steps) { @@ -378,8 +104,6 @@ std::unique_ptr IMergeTreeSelectAlgorithm::getPrewhereActions( prewhere_actions->steps.emplace_back(std::move(row_level_filter_step)); } -// std::cerr << "ORIGINAL PREWHERE:\n" << prewhere_info->prewhere_actions->dumpDAG() << std::endl; - if (!enable_multiple_prewhere_read_steps || !tryBuildPrewhereSteps(prewhere_info, actions_settings, *prewhere_actions)) { diff --git a/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp b/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp new file mode 100644 index 00000000000..8284bda34a7 --- /dev/null +++ b/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp @@ -0,0 +1,302 @@ +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +namespace +{ + +/// Stores the ist of columns required to compute a node in the DAG. +struct NodeInfo +{ + NameSet required_columns; +}; + +/// Fills the list of required columns for a node in the DAG. +void fillRequiredColumns(const ActionsDAG::Node * node, std::unordered_map & nodes_info) +{ + if (nodes_info.contains(node)) + return; + + auto & node_info = nodes_info[node]; + + if (node->type == ActionsDAG::ActionType::INPUT) + { + node_info.required_columns.insert(node->result_name); + return; + } + + for (const auto & child : node->children) + { + fillRequiredColumns(child, nodes_info); + const auto & child_info = nodes_info[child]; + node_info.required_columns.insert(child_info.required_columns.begin(), child_info.required_columns.end()); + } +} + +/// Stores information about a node that has already been cloned to one of the new DAGs. +/// This allows to avoid cloning the same sub-DAG into multiple step DAGs but reference previously cloned nodes from earliers steps. +struct DAGNodeRef +{ + ActionsDAGPtr dag; + const ActionsDAG::Node * node; +}; + +using OriginalToNewNodeMap = std::unordered_map; + +/// Clones the part of original DAG responsible for computing the original_dag_node and adds it to the new DAG. +const ActionsDAG::Node & addClonedDAGToDAG(const ActionsDAG::Node * original_dag_node, ActionsDAGPtr new_dag, OriginalToNewNodeMap & node_remap) +{ + /// Look for the node in the map of already known nodes + if (node_remap.contains(original_dag_node)) + { + /// If the node is already in the new DAG, return it + const auto & node_ref = node_remap.at(original_dag_node); + if (node_ref.dag == new_dag) + return *node_ref.node; + + /// If the node is known from the previous steps, add it as an input, except for constants + if (original_dag_node->type != ActionsDAG::ActionType::COLUMN) + { + node_ref.dag->addOrReplaceInOutputs(*node_ref.node); + const auto & new_node = new_dag->addInput(node_ref.node->result_name, node_ref.node->result_type); + node_remap[original_dag_node] = {new_dag, &new_node}; /// TODO: here we update the node reference. Is it always correct? + return new_node; + } + } + + /// If the node is an input, add it as an input + if (original_dag_node->type == ActionsDAG::ActionType::INPUT) + { + const auto & new_node = new_dag->addInput(original_dag_node->result_name, original_dag_node->result_type); + node_remap[original_dag_node] = {new_dag, &new_node}; + return new_node; + } + + /// If the node is a column, add it as an input + if (original_dag_node->type == ActionsDAG::ActionType::COLUMN) + { + const auto & new_node = new_dag->addColumn( + ColumnWithTypeAndName(original_dag_node->column, original_dag_node->result_type, original_dag_node->result_name)); + node_remap[original_dag_node] = {new_dag, &new_node}; + return new_node; + } + + /// TODO: Do we need to handle ALIAS nodes in cloning? + + /// If the node is a function, add it as a function and add its children + if (original_dag_node->type == ActionsDAG::ActionType::FUNCTION) + { + ActionsDAG::NodeRawConstPtrs new_children; + for (const auto & child : original_dag_node->children) + { + const auto & new_child = addClonedDAGToDAG(child, new_dag, node_remap); + new_children.push_back(&new_child); + } + + const auto & new_node = new_dag->addFunction(original_dag_node->function_base, new_children, original_dag_node->result_name); + node_remap[original_dag_node] = {new_dag, &new_node}; + return new_node; + } + + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected node type in PREWHERE actions: {}", original_dag_node->type); +} + +/// Adds a CAST node with the regular name ("CAST(...)") or with the provided name. +/// This is different from ActionsDAG::addCast() because it set the name equal to the original name effectively hiding the value before cast, +/// but it might be required for further steps with its original uncasted type. +const ActionsDAG::Node & addCast(ActionsDAGPtr dag, const ActionsDAG::Node & node_to_cast, const String & type_name, const String & new_name = {}) +{ + Field cast_type_constant_value(type_name); + + ColumnWithTypeAndName column; + column.name = calculateConstantActionNodeName(cast_type_constant_value); + column.column = DataTypeString().createColumnConst(0, cast_type_constant_value); + column.type = std::make_shared(); + + const auto * cast_type_constant_node = &dag->addColumn(std::move(column)); + ActionsDAG::NodeRawConstPtrs children = {&node_to_cast, cast_type_constant_node}; + FunctionOverloadResolverPtr func_builder_cast = CastInternalOverloadResolver::createImpl(); + + return dag->addFunction(func_builder_cast, std::move(children), new_name); +} + +} + +/// We want to build a sequence of steps that will compute parts of the prewhere condition. +/// Each step reads some new columns and computes some new expressions and a filter condition. +/// The last step computes the final filter condition and the remaining expressions that are required for the main query. +/// The goal of this is to, when it is possible, filter out many rows in early steps so that the remaining steps will +/// read less data from the storage. +/// NOTE: The result of executing the steps is exactly the same as if we would execute the original DAG in single step. +/// +/// The steps are built in the following way: +/// 1. List all condition nodes that are combined with AND into PREWHERE condition +/// 2. Collect the set of columns that are used in each condition +/// 3. Sort condition nodes by the number of columns used in them and the overall size of those columns +/// 4. Group conditions with the same set of columns into a single read/compute step +/// 5. Build DAGs for each step: +/// - DFS from the condition root node: +/// - If the node was not computed yet, add it to the DAG and traverse its children +/// - If the node was already computed by one of the previous steps, add it as output for that step and as input for the current step +/// - If the node was already computed by the current step just stop traversing +/// 6. Find all outputs of the original DAG +/// 7. Find all outputs that were computed in the already built DAGs, mark these nodes as outputs in the steps where they were computed +/// 8. Add computation of the remaining outputs to the last step with the procedure similar to 4 +bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionActionsSettings & actions_settings, PrewhereExprInfo & prewhere) +{ + if (!prewhere_info || !prewhere_info->prewhere_actions) + return true; + +// std::cerr << "ORIGINAL PREWHERE:\n" << prewhere_info->prewhere_actions->dumpDAG() << std::endl; + + /// 1. List all condition nodes that are combined with AND into PREWHERE condition + const auto & condition_root = prewhere_info->prewhere_actions->findInOutputs(prewhere_info->prewhere_column_name); + const bool is_conjunction = (condition_root.type == ActionsDAG::ActionType::FUNCTION && condition_root.function_base->getName() == "and"); + if (!is_conjunction) + return false; + auto condition_nodes = condition_root.children; + + /// 2. Collect the set of columns that are used in the condition + std::unordered_map nodes_info; + for (const auto & node : condition_nodes) + { + fillRequiredColumns(node, nodes_info); + } + + /// 3. Sort condition nodes by the number of columns used in them and the overall size of those columns + /// TODO: not sorting for now because the conditions are already sorted by Where Optimizer + + /// 4. Group conditions with the same set of columns into a single read/compute step + std::vector> condition_groups; + for (const auto & node : condition_nodes) + { + const auto & node_info = nodes_info[node]; + if (!condition_groups.empty() && nodes_info[condition_groups.back().back()].required_columns == node_info.required_columns) + condition_groups.back().push_back(node); /// Add to the last group + else + condition_groups.push_back({node}); /// Start new group + } + + /// 5. Build DAGs for each step + struct Step + { + ActionsDAGPtr actions; + String column_name; + }; + std::vector steps; + + OriginalToNewNodeMap node_remap; + + for (const auto & condition_group : condition_groups) + { +// std::cerr +// << "Conditions: ["; +// +// for (const auto & condition : condition_group) +// std::cerr << " \"" << condition->result_name; +// +// std::cerr << "\" ] Columns: " << boost::algorithm::join(nodes_info[condition_group.front()].required_columns, " ") +// << std::endl; + + ActionsDAGPtr step_dag = std::make_shared(); + String result_name; + + std::vector new_condition_nodes; + for (const auto * node : condition_group) + { + const auto & node_in_new_dag = addClonedDAGToDAG(node, step_dag, node_remap); + new_condition_nodes.push_back(&node_in_new_dag); + } + + if (new_condition_nodes.size() > 1) + { + /// Add AND function to combine the conditions + FunctionOverloadResolverPtr func_builder_and = std::make_unique(std::make_shared()); + const auto & and_function_node = step_dag->addFunction(func_builder_and, new_condition_nodes, ""); + step_dag->addOrReplaceInOutputs(and_function_node); + result_name = and_function_node.result_name; + } + else + { + const auto & result_node = *new_condition_nodes.front(); + /// Add cast to UInt8 if needed + if (result_node.result_type->getTypeId() == TypeIndex::UInt8) + { + step_dag->addOrReplaceInOutputs(result_node); + result_name = result_node.result_name; + } + else + { + const auto & cast_node = addCast(step_dag, result_node, "UInt8"); + step_dag->addOrReplaceInOutputs(cast_node); + result_name = cast_node.result_name; + } + } + +// std::cerr << "Step DAG:\n" << step_dag->dumpDAG() << std::endl; + + steps.push_back({step_dag, result_name}); + } + + /// 6. Find all outputs of the original DAG + auto original_outputs = prewhere_info->prewhere_actions->getOutputs(); + /// 7. Find all outputs that were computed in the already built DAGs, mark these nodes as outputs in the steps where they were computed + /// 8. Add computation of the remaining outputs to the last step with the procedure similar to 4 + NameSet all_output_names; + for (const auto * output : original_outputs) + { +// std::cerr << "Original output: " << output->result_name << std::endl; + all_output_names.insert(output->result_name); + if (node_remap.contains(output)) + { + const auto & new_node_info = node_remap[output]; + new_node_info.dag->addOrReplaceInOutputs(*new_node_info.node); + } + else if (output->result_name == prewhere_info->prewhere_column_name) + { + /// Special case for final PREWHERE column: it is an AND combination of all conditions, + /// but we have only the condition for the last step here. + /// However we know that the ultimate result after filtering is constant 1 for the PREWHERE column. + auto const_true = output->result_type->createColumnConst(0, Field{1}); + const auto & prewhere_result_node = + steps.back().actions->addColumn(ColumnWithTypeAndName(const_true, output->result_type, output->result_name)); + steps.back().actions->addOrReplaceInOutputs(prewhere_result_node); + } + else + { + const auto & node_in_new_dag = addClonedDAGToDAG(output, steps.back().actions, node_remap); + steps.back().actions->addOrReplaceInOutputs(node_in_new_dag); + } + } + + /// 9. Build PrewhereExprInfo + { + for (const auto & step : steps) + { +// std::cerr << "Step DAG:\n" << step.actions->dumpDAG() << std::endl; + prewhere.steps.push_back( + { + .actions = std::make_shared(step.actions, actions_settings), + .column_name = step.column_name, + .remove_column = !all_output_names.contains(step.column_name), /// Don't remove if it's in the list of original outputs + .need_filter = false, + }); + } + prewhere.steps.back().need_filter = prewhere_info->need_filter; + } + + return true; +} + +} From 68748d6aef5292606119e63b6e94e3f026aafcea Mon Sep 17 00:00:00 2001 From: flynn Date: Mon, 13 Feb 2023 16:06:00 +0000 Subject: [PATCH 180/717] fix --- src/IO/S3Common.cpp | 41 ----------------- src/IO/S3Common.h | 4 +- src/Storages/S3DataLakeMetaReadHelper.cpp | 54 ++++++++++++++++++++++- src/Storages/S3DataLakeMetaReadHelper.h | 7 +-- 4 files changed, 54 insertions(+), 52 deletions(-) diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp index 54aaddd9056..dc7307ede8c 100644 --- a/src/IO/S3Common.cpp +++ b/src/IO/S3Common.cpp @@ -799,47 +799,6 @@ namespace S3 put_request_throttler); } - std::vector - listFiles(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & prefix, const String & extension) - { - std::vector res; - S3::ListObjectsV2Request request; - Aws::S3::Model::ListObjectsV2Outcome outcome; - - bool is_finished{false}; - - request.SetBucket(bucket); - - request.SetPrefix(prefix); - - while (!is_finished) - { - outcome = client.ListObjectsV2(request); - if (!outcome.IsSuccess()) - throw Exception( - ErrorCodes::S3_ERROR, - "Could not list objects in bucket {} with key {}, S3 exception: {}, message: {}", - quoteString(bucket), - quoteString(key), - backQuote(outcome.GetError().GetExceptionName()), - quoteString(outcome.GetError().GetMessage())); - - const auto & result_batch = outcome.GetResult().GetContents(); - for (const auto & obj : result_batch) - { - const auto & filename = obj.GetKey(); - - if (std::filesystem::path(filename).extension() == extension) - res.push_back(filename); - } - - request.SetContinuationToken(outcome.GetResult().GetNextContinuationToken()); - - is_finished = !outcome.GetResult().GetIsTruncated(); - } - - return res; - } } } diff --git a/src/IO/S3Common.h b/src/IO/S3Common.h index 979544a3959..1f2f7caa6df 100644 --- a/src/IO/S3Common.h +++ b/src/IO/S3Common.h @@ -96,10 +96,8 @@ private: Aws::SDKOptions aws_options; std::atomic s3_requests_logging_enabled; }; - -std::vector -listFiles(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & prefix, const String & extension); } + #endif namespace Poco::Util diff --git a/src/Storages/S3DataLakeMetaReadHelper.cpp b/src/Storages/S3DataLakeMetaReadHelper.cpp index fe5cd704132..c9e1d822abd 100644 --- a/src/Storages/S3DataLakeMetaReadHelper.cpp +++ b/src/Storages/S3DataLakeMetaReadHelper.cpp @@ -2,10 +2,22 @@ #if USE_AWS_S3 +# include +# include # include +# include +# include +# include + namespace DB { + +namespace ErrorCodes +{ + extern const int S3_ERROR; +} + std::shared_ptr S3DataLakeMetaReadHelper::createReadBuffer(const String & key, ContextPtr context, const StorageS3::Configuration & base_configuration) { @@ -23,8 +35,46 @@ std::vector S3DataLakeMetaReadHelper::listFilesMatchSuffix( const StorageS3::Configuration & base_configuration, const String & directory, const String & suffix) { const auto & table_path = base_configuration.url.key; - return S3::listFiles( - *base_configuration.client, base_configuration.url.bucket, table_path, std::filesystem::path(table_path) / directory, suffix); + const auto & bucket = base_configuration.url.bucket; + const auto & client = base_configuration.client; + + std::vector res; + S3::ListObjectsV2Request request; + Aws::S3::Model::ListObjectsV2Outcome outcome; + + bool is_finished{false}; + + request.SetBucket(bucket); + + request.SetPrefix(std::filesystem::path(table_path) / directory); + + while (!is_finished) + { + outcome = client->ListObjectsV2(request); + if (!outcome.IsSuccess()) + throw Exception( + ErrorCodes::S3_ERROR, + "Could not list objects in bucket {} with key {}, S3 exception: {}, message: {}", + quoteString(bucket), + quoteString(base_configuration.url.key), + backQuote(outcome.GetError().GetExceptionName()), + quoteString(outcome.GetError().GetMessage())); + + const auto & result_batch = outcome.GetResult().GetContents(); + for (const auto & obj : result_batch) + { + const auto & filename = obj.GetKey(); + + if (std::filesystem::path(filename).extension() == suffix) + res.push_back(filename); + } + + request.SetContinuationToken(outcome.GetResult().GetNextContinuationToken()); + + is_finished = !outcome.GetResult().GetIsTruncated(); + } + + return res; } std::vector S3DataLakeMetaReadHelper::listFiles(const StorageS3::Configuration & configuration) diff --git a/src/Storages/S3DataLakeMetaReadHelper.h b/src/Storages/S3DataLakeMetaReadHelper.h index 5db2d51a2ec..29a00e47e10 100644 --- a/src/Storages/S3DataLakeMetaReadHelper.h +++ b/src/Storages/S3DataLakeMetaReadHelper.h @@ -6,15 +6,10 @@ # include # include -# include +# include # include -# include -# include -# include -# include - class ReadBuffer; From 18a29e81fc20c791fd77671c8129f27b1202eefd Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Mon, 13 Feb 2023 17:42:27 +0100 Subject: [PATCH 181/717] Better formatting in PrewhereExprInfo::dump() --- src/Storages/MergeTree/MergeTreeRangeReader.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeRangeReader.cpp b/src/Storages/MergeTree/MergeTreeRangeReader.cpp index f5afc0b37d6..4036d352a54 100644 --- a/src/Storages/MergeTree/MergeTreeRangeReader.cpp +++ b/src/Storages/MergeTree/MergeTreeRangeReader.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #ifdef __SSE2__ #include @@ -1370,13 +1371,16 @@ std::string PrewhereExprInfo::dump() const { WriteBufferFromOwnString s; + const char indent[] = "\n "; for (size_t i = 0; i < steps.size(); ++i) { s << "STEP " << i << ":\n" - << " ACTIONS: " << (steps[i].actions ? steps[i].actions->dumpActions() : "nullptr") << "\n" + << " ACTIONS: " << (steps[i].actions ? + (indent + boost::replace_all_copy(steps[i].actions->dumpActions(), "\n", indent)) : + "nullptr") << "\n" << " COLUMN: " << steps[i].column_name << "\n" << " REMOVE_COLUMN: " << steps[i].remove_column << "\n" - << " NEED_FILTER: " << steps[i].need_filter << "\n"; + << " NEED_FILTER: " << steps[i].need_filter << "\n\n"; } return s.str(); From 0fe080cc6f7a3432c19a044bd869c59bf633d393 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Mon, 13 Feb 2023 17:43:41 +0100 Subject: [PATCH 182/717] Logger instead of debug prints --- .../MergeTreeSplitPrewhereIntoReadSteps.cpp | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp b/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp index 8284bda34a7..90ecf3e25f5 100644 --- a/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp +++ b/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp @@ -158,7 +158,9 @@ bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionAction if (!prewhere_info || !prewhere_info->prewhere_actions) return true; -// std::cerr << "ORIGINAL PREWHERE:\n" << prewhere_info->prewhere_actions->dumpDAG() << std::endl; + Poco::Logger * log = &Poco::Logger::get("tryBuildPrewhereSteps"); + + LOG_TRACE(log, "Original PREWHERE DAG:\n{}", prewhere_info->prewhere_actions->dumpDAG()); /// 1. List all condition nodes that are combined with AND into PREWHERE condition const auto & condition_root = prewhere_info->prewhere_actions->findInOutputs(prewhere_info->prewhere_column_name); @@ -200,15 +202,6 @@ bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionAction for (const auto & condition_group : condition_groups) { -// std::cerr -// << "Conditions: ["; -// -// for (const auto & condition : condition_group) -// std::cerr << " \"" << condition->result_name; -// -// std::cerr << "\" ] Columns: " << boost::algorithm::join(nodes_info[condition_group.front()].required_columns, " ") -// << std::endl; - ActionsDAGPtr step_dag = std::make_shared(); String result_name; @@ -244,8 +237,6 @@ bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionAction } } -// std::cerr << "Step DAG:\n" << step_dag->dumpDAG() << std::endl; - steps.push_back({step_dag, result_name}); } @@ -256,7 +247,6 @@ bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionAction NameSet all_output_names; for (const auto * output : original_outputs) { -// std::cerr << "Original output: " << output->result_name << std::endl; all_output_names.insert(output->result_name); if (node_remap.contains(output)) { @@ -284,7 +274,6 @@ bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionAction { for (const auto & step : steps) { -// std::cerr << "Step DAG:\n" << step.actions->dumpDAG() << std::endl; prewhere.steps.push_back( { .actions = std::make_shared(step.actions, actions_settings), @@ -296,6 +285,8 @@ bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionAction prewhere.steps.back().need_filter = prewhere_info->need_filter; } + LOG_TRACE(log, "Resulting PREWHERE:\n{}", prewhere.dump()); + return true; } From e9e6d735ef12a3276b03c1aa4ca2fc9c087c1fea Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Mon, 13 Feb 2023 16:44:54 +0000 Subject: [PATCH 183/717] add query_id header in all queries --- src/Interpreters/executeQuery.cpp | 14 +++++++++++--- src/Interpreters/executeQuery.h | 10 +++++++++- src/Server/HTTPHandler.cpp | 18 +++++++++++++----- src/Server/MySQLHandler.cpp | 4 ++-- .../02564_query_id_header.reference | 5 +++++ .../0_stateless/02564_query_id_header.sh | 13 +++++++++++++ 6 files changed, 53 insertions(+), 11 deletions(-) create mode 100644 tests/queries/0_stateless/02564_query_id_header.reference create mode 100755 tests/queries/0_stateless/02564_query_id_header.sh diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 06d92116adc..6fd3528ec65 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -1249,6 +1249,12 @@ void executeQuery( std::tie(ast, streams) = executeQueryImpl(begin, end, context, false, QueryProcessingStage::Complete, &istr); auto & pipeline = streams.pipeline; + QueryResultDetails result_details + { + .query_id = context->getClientInfo().current_query_id, + .timezone = DateLUT::instance().getTimeZone(), + }; + std::unique_ptr compressed_buffer; try { @@ -1307,9 +1313,8 @@ void executeQuery( out->onProgress(progress); }); - if (set_result_details) - set_result_details( - context->getClientInfo().current_query_id, out->getContentType(), format_name, DateLUT::instance().getTimeZone()); + result_details.content_type = out->getContentType(); + result_details.format = format_name; pipeline.complete(std::move(out)); } @@ -1318,6 +1323,9 @@ void executeQuery( pipeline.setProgressCallback(context->getProgressCallback()); } + if (set_result_details) + set_result_details(result_details); + if (pipeline.initialized()) { CompletedPipelineExecutor executor(pipeline); diff --git a/src/Interpreters/executeQuery.h b/src/Interpreters/executeQuery.h index 9c561d8b88c..93152cc1de6 100644 --- a/src/Interpreters/executeQuery.h +++ b/src/Interpreters/executeQuery.h @@ -11,7 +11,15 @@ namespace DB class ReadBuffer; class WriteBuffer; -using SetResultDetailsFunc = std::function; +struct QueryResultDetails +{ + String query_id; + std::optional content_type; + std::optional format; + std::optional timezone; +}; + +using SetResultDetailsFunc = std::function; /// Parse and execute a query. void executeQuery( diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp index 29bfa8065ba..1e249b77079 100644 --- a/src/Server/HTTPHandler.cpp +++ b/src/Server/HTTPHandler.cpp @@ -819,12 +819,20 @@ void HTTPHandler::processQuery( customizeContext(request, context); executeQuery(*in, *used_output.out_maybe_delayed_and_compressed, /* allow_into_outfile = */ false, context, - [&response, this] (const String & current_query_id, const String & content_type, const String & format, const String & timezone) + [&response, this] (const QueryResultDetails & details) { - response.setContentType(content_type_override.value_or(content_type)); - response.add("X-ClickHouse-Query-Id", current_query_id); - response.add("X-ClickHouse-Format", format); - response.add("X-ClickHouse-Timezone", timezone); + response.add("X-ClickHouse-Query-Id", details.query_id); + + if (content_type_override) + response.setContentType(*content_type_override); + else if (details.content_type) + response.setContentType(*details.content_type); + + if (details.format) + response.add("X-ClickHouse-Format", *details.format); + + if (details.timezone) + response.add("X-ClickHouse-Timezone", *details.timezone); } ); diff --git a/src/Server/MySQLHandler.cpp b/src/Server/MySQLHandler.cpp index 3715d658730..e3467d664e7 100644 --- a/src/Server/MySQLHandler.cpp +++ b/src/Server/MySQLHandler.cpp @@ -352,9 +352,9 @@ void MySQLHandler::comQuery(ReadBuffer & payload) format_settings.mysql_wire.max_packet_size = max_packet_size; format_settings.mysql_wire.sequence_id = &sequence_id; - auto set_result_details = [&with_output](const String &, const String &, const String &format, const String &) + auto set_result_details = [&with_output](const QueryResultDetails & details) { - if (format != "MySQLWire") + if (details.format && *details.format != "MySQLWire") throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "MySQL protocol does not support custom output formats"); with_output = true; }; diff --git a/tests/queries/0_stateless/02564_query_id_header.reference b/tests/queries/0_stateless/02564_query_id_header.reference new file mode 100644 index 00000000000..655f9fa9eaa --- /dev/null +++ b/tests/queries/0_stateless/02564_query_id_header.reference @@ -0,0 +1,5 @@ +X-ClickHouse-Query-Id +X-ClickHouse-Query-Id +X-ClickHouse-Query-Id +X-ClickHouse-Query-Id +X-ClickHouse-Query-Id diff --git a/tests/queries/0_stateless/02564_query_id_header.sh b/tests/queries/0_stateless/02564_query_id_header.sh new file mode 100755 index 00000000000..440182cd243 --- /dev/null +++ b/tests/queries/0_stateless/02564_query_id_header.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS t_query_id_header" + +${CLICKHOUSE_CURL} -sS -v "${CLICKHOUSE_URL}" -d "CREATE TABLE t_query_id_header (a UInt64) ENGINE = Memory" 2>&1 | grep -o "X-ClickHouse-Query-Id" +${CLICKHOUSE_CURL} -sS -v "${CLICKHOUSE_URL}" -d "INSERT INTO t_query_id_header VALUES (1)" 2>&1 | grep -o "X-ClickHouse-Query-Id" +${CLICKHOUSE_CURL} -sS -v "${CLICKHOUSE_URL}" -d "EXISTS TABLE t_query_id_header" 2>&1 | grep -o "X-ClickHouse-Query-Id" +${CLICKHOUSE_CURL} -sS -v "${CLICKHOUSE_URL}" -d "SELECT * FROM t_query_id_header" 2>&1 | grep -o "X-ClickHouse-Query-Id" +${CLICKHOUSE_CURL} -sS -v "${CLICKHOUSE_URL}" -d "DROP TABLE t_query_id_header" 2>&1 | grep -o "X-ClickHouse-Query-Id" From a4d9688775d8d39b8b462b4711d649776d4dcfaa Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 13 Feb 2023 19:35:07 +0100 Subject: [PATCH 184/717] fix 'Directory not empty after drop' with zero copy replication --- src/Storages/MergeTree/MergeTreeData.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index c3c4cd3082d..3c260484d95 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -1852,11 +1852,11 @@ void MergeTreeData::stopOutdatedDataPartsLoadingTask() /// (Only files on the first level of nesting are considered). static bool isOldPartDirectory(const DiskPtr & disk, const String & directory_path, time_t threshold) { - if (!disk->isDirectory(directory_path) || disk->getLastModified(directory_path).epochTime() >= threshold) + if (!disk->isDirectory(directory_path) || disk->getLastModified(directory_path).epochTime() > threshold) return false; for (auto it = disk->iterateDirectory(directory_path); it->isValid(); it->next()) - if (disk->getLastModified(it->path()).epochTime() >= threshold) + if (disk->getLastModified(it->path()).epochTime() > threshold) return false; return true; From 67565aaa792ed38f2d5662b466ce4de6c99bc30e Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Mon, 13 Feb 2023 22:50:22 +0100 Subject: [PATCH 185/717] A test for combined condition node --- .../02559_multiple_read_steps_in_prewhere.reference | 6 ++++++ .../02559_multiple_read_steps_in_prewhere.sql | 12 ++++++++++++ 2 files changed, 18 insertions(+) diff --git a/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.reference b/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.reference index 0c46a90257d..f2da126fc92 100644 --- a/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.reference +++ b/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.reference @@ -38,3 +38,9 @@ SELECT cast(id1 as UInt16) AS cond1, (id2 % 40000) AS cond2, (cond1 AND cond2) A SELECT cast(id1 as UInt16) AS cond1 FROM test_02559 PREWHERE cond1 LIMIT 10; -- { serverError ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER } SELECT count() FROM test_02559 PREWHERE id2>=0 AND (1 OR ignore(id1)) WHERE ignore(id1)=0; 10 +CREATE ROW POLICY 02559_filter_1 ON test_02559 USING id2=2 AS permissive TO ALL; +SELECT * FROM test_02559; +2 2 +CREATE ROW POLICY 02559_filter_2 ON test_02559 USING id2<=2 AS restrictive TO ALL; +SELECT * FROM test_02559; +2 2 diff --git a/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.sql b/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.sql index e39f8ac846a..0b434eae7df 100644 --- a/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.sql +++ b/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.sql @@ -4,6 +4,9 @@ CREATE TABLE test_02559 (id1 UInt64, id2 UInt64) ENGINE=MergeTree ORDER BY id1; INSERT INTO test_02559 SELECT number, number FROM numbers(10); +DROP ROW POLICY IF EXISTS 02559_filter_1 ON test_02559; +DROP ROW POLICY IF EXISTS 02559_filter_2 ON test_02559; + SET enable_multiple_prewhere_read_steps=true, move_all_conditions_to_prewhere=true; -- { echoOn } @@ -22,6 +25,15 @@ SELECT cast(id1 as UInt16) AS cond1 FROM test_02559 PREWHERE cond1 LIMIT 10; -- SELECT count() FROM test_02559 PREWHERE id2>=0 AND (1 OR ignore(id1)) WHERE ignore(id1)=0; +CREATE ROW POLICY 02559_filter_1 ON test_02559 USING id2=2 AS permissive TO ALL; +SELECT * FROM test_02559; + +CREATE ROW POLICY 02559_filter_2 ON test_02559 USING id2<=2 AS restrictive TO ALL; +SELECT * FROM test_02559; + -- { echoOff } +DROP ROW POLICY IF EXISTS 02559_filter_1 ON test_02559; +DROP ROW POLICY IF EXISTS 02559_filter_2 ON test_02559; + DROP TABLE test_02559; From d76e4f60a83e2ac7c9032d59605ff98aba80abb2 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Mon, 13 Feb 2023 23:14:18 +0100 Subject: [PATCH 186/717] Fix for combined node for condition group not found --- .../MergeTreeSplitPrewhereIntoReadSteps.cpp | 47 ++++++++++++------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp b/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp index 90ecf3e25f5..d41572c9db7 100644 --- a/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp +++ b/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp @@ -1,6 +1,5 @@ #include #include -#include #include #include #include @@ -44,7 +43,7 @@ void fillRequiredColumns(const ActionsDAG::Node * node, std::unordered_map; +/// Result name -> DAGNodeRef +using OriginalToNewNodeMap = std::unordered_map; /// Clones the part of original DAG responsible for computing the original_dag_node and adds it to the new DAG. const ActionsDAG::Node & addClonedDAGToDAG(const ActionsDAG::Node * original_dag_node, ActionsDAGPtr new_dag, OriginalToNewNodeMap & node_remap) { + const String & node_name = original_dag_node->result_name; /// Look for the node in the map of already known nodes - if (node_remap.contains(original_dag_node)) + if (node_remap.contains(node_name)) { /// If the node is already in the new DAG, return it - const auto & node_ref = node_remap.at(original_dag_node); + const auto & node_ref = node_remap.at(node_name); if (node_ref.dag == new_dag) return *node_ref.node; @@ -70,7 +71,7 @@ const ActionsDAG::Node & addClonedDAGToDAG(const ActionsDAG::Node * original_dag { node_ref.dag->addOrReplaceInOutputs(*node_ref.node); const auto & new_node = new_dag->addInput(node_ref.node->result_name, node_ref.node->result_type); - node_remap[original_dag_node] = {new_dag, &new_node}; /// TODO: here we update the node reference. Is it always correct? + node_remap[node_name] = {new_dag, &new_node}; /// TODO: here we update the node reference. Is it always correct? return new_node; } } @@ -79,7 +80,7 @@ const ActionsDAG::Node & addClonedDAGToDAG(const ActionsDAG::Node * original_dag if (original_dag_node->type == ActionsDAG::ActionType::INPUT) { const auto & new_node = new_dag->addInput(original_dag_node->result_name, original_dag_node->result_type); - node_remap[original_dag_node] = {new_dag, &new_node}; + node_remap[node_name] = {new_dag, &new_node}; return new_node; } @@ -88,7 +89,7 @@ const ActionsDAG::Node & addClonedDAGToDAG(const ActionsDAG::Node * original_dag { const auto & new_node = new_dag->addColumn( ColumnWithTypeAndName(original_dag_node->column, original_dag_node->result_type, original_dag_node->result_name)); - node_remap[original_dag_node] = {new_dag, &new_node}; + node_remap[node_name] = {new_dag, &new_node}; return new_node; } @@ -105,22 +106,36 @@ const ActionsDAG::Node & addClonedDAGToDAG(const ActionsDAG::Node * original_dag } const auto & new_node = new_dag->addFunction(original_dag_node->function_base, new_children, original_dag_node->result_name); - node_remap[original_dag_node] = {new_dag, &new_node}; + node_remap[node_name] = {new_dag, &new_node}; return new_node; } throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected node type in PREWHERE actions: {}", original_dag_node->type); } +const ActionsDAG::Node & addFunction( + ActionsDAGPtr new_dag, + const FunctionOverloadResolverPtr & function, + ActionsDAG::NodeRawConstPtrs children, + OriginalToNewNodeMap & node_remap) +{ + const auto & new_node = new_dag->addFunction(function, children, ""); + node_remap[new_node.result_name] = {new_dag, &new_node}; + return new_node; +} + /// Adds a CAST node with the regular name ("CAST(...)") or with the provided name. /// This is different from ActionsDAG::addCast() because it set the name equal to the original name effectively hiding the value before cast, /// but it might be required for further steps with its original uncasted type. -const ActionsDAG::Node & addCast(ActionsDAGPtr dag, const ActionsDAG::Node & node_to_cast, const String & type_name, const String & new_name = {}) +const ActionsDAG::Node & addCast( + ActionsDAGPtr dag, + const ActionsDAG::Node & node_to_cast, + const String & type_name, + OriginalToNewNodeMap & node_remap) { Field cast_type_constant_value(type_name); ColumnWithTypeAndName column; - column.name = calculateConstantActionNodeName(cast_type_constant_value); column.column = DataTypeString().createColumnConst(0, cast_type_constant_value); column.type = std::make_shared(); @@ -128,7 +143,7 @@ const ActionsDAG::Node & addCast(ActionsDAGPtr dag, const ActionsDAG::Node & nod ActionsDAG::NodeRawConstPtrs children = {&node_to_cast, cast_type_constant_node}; FunctionOverloadResolverPtr func_builder_cast = CastInternalOverloadResolver::createImpl(); - return dag->addFunction(func_builder_cast, std::move(children), new_name); + return addFunction(dag, func_builder_cast, std::move(children), node_remap); } } @@ -216,7 +231,7 @@ bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionAction { /// Add AND function to combine the conditions FunctionOverloadResolverPtr func_builder_and = std::make_unique(std::make_shared()); - const auto & and_function_node = step_dag->addFunction(func_builder_and, new_condition_nodes, ""); + const auto & and_function_node = addFunction(step_dag, func_builder_and, new_condition_nodes, node_remap); step_dag->addOrReplaceInOutputs(and_function_node); result_name = and_function_node.result_name; } @@ -231,7 +246,7 @@ bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionAction } else { - const auto & cast_node = addCast(step_dag, result_node, "UInt8"); + const auto & cast_node = addCast(step_dag, result_node, "UInt8", node_remap); step_dag->addOrReplaceInOutputs(cast_node); result_name = cast_node.result_name; } @@ -248,9 +263,9 @@ bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionAction for (const auto * output : original_outputs) { all_output_names.insert(output->result_name); - if (node_remap.contains(output)) + if (node_remap.contains(output->result_name)) { - const auto & new_node_info = node_remap[output]; + const auto & new_node_info = node_remap[output->result_name]; new_node_info.dag->addOrReplaceInOutputs(*new_node_info.node); } else if (output->result_name == prewhere_info->prewhere_column_name) From e712fbecb2ee7aeb2dd1bd1913218d3ba21d6252 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 14 Feb 2023 00:45:28 +0100 Subject: [PATCH 187/717] fix race between drop and create --- src/Interpreters/DatabaseCatalog.cpp | 1 + src/Interpreters/DatabaseCatalog.h | 4 +- src/Storages/StorageReplicatedMergeTree.cpp | 41 ++++++++++---- src/Storages/StorageReplicatedMergeTree.h | 2 +- tests/integration/helpers/cluster.py | 4 +- .../configs/config.d/storage_conf.xml | 2 + .../test.py | 56 +++++++++++++++++++ 7 files changed, 96 insertions(+), 14 deletions(-) diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index ad5d9d4d325..c3ac7122362 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -147,6 +147,7 @@ void DatabaseCatalog::initializeAndLoadTemporaryDatabase() unused_dir_hide_timeout_sec = getContext()->getConfigRef().getInt64("database_catalog_unused_dir_hide_timeout_sec", unused_dir_hide_timeout_sec); unused_dir_rm_timeout_sec = getContext()->getConfigRef().getInt64("database_catalog_unused_dir_rm_timeout_sec", unused_dir_rm_timeout_sec); unused_dir_cleanup_period_sec = getContext()->getConfigRef().getInt64("database_catalog_unused_dir_cleanup_period_sec", unused_dir_cleanup_period_sec); + drop_error_cooldown_sec = getContext()->getConfigRef().getInt64("database_catalog_drop_error_cooldown_sec", drop_error_cooldown_sec); auto db_for_temporary_and_external_tables = std::make_shared(TEMPORARY_DATABASE, getContext()); attachDatabase(TEMPORARY_DATABASE, db_for_temporary_and_external_tables); diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h index ba3625626da..c37029d8b6b 100644 --- a/src/Interpreters/DatabaseCatalog.h +++ b/src/Interpreters/DatabaseCatalog.h @@ -278,7 +278,6 @@ private: bool maybeRemoveDirectory(const String & disk_name, const DiskPtr & disk, const String & unused_dir); static constexpr size_t reschedule_time_ms = 100; - static constexpr time_t drop_error_cooldown_sec = 5; mutable std::mutex databases_mutex; @@ -325,6 +324,9 @@ private: time_t unused_dir_rm_timeout_sec = default_unused_dir_rm_timeout_sec; static constexpr time_t default_unused_dir_cleanup_period_sec = 24 * 60 * 60; /// 1 day time_t unused_dir_cleanup_period_sec = default_unused_dir_cleanup_period_sec; + + static constexpr time_t default_drop_error_cooldown_sec = 5; + time_t drop_error_cooldown_sec = default_drop_error_cooldown_sec; }; /// This class is useful when creating a table or database. diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index d3590657a5c..c92d4efab0d 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -903,17 +903,16 @@ void StorageReplicatedMergeTree::drop() /// in this case, has_metadata_in_zookeeper = false, and we also permit to drop the table. bool maybe_has_metadata_in_zookeeper = !has_metadata_in_zookeeper.has_value() || *has_metadata_in_zookeeper; + zkutil::ZooKeeperPtr zookeeper; if (maybe_has_metadata_in_zookeeper) { /// Table can be shut down, restarting thread is not active /// and calling StorageReplicatedMergeTree::getZooKeeper()/getAuxiliaryZooKeeper() won't suffice. - zkutil::ZooKeeperPtr zookeeper = getZooKeeperIfTableShutDown(); + zookeeper = getZooKeeperIfTableShutDown(); /// If probably there is metadata in ZooKeeper, we don't allow to drop the table. if (!zookeeper) throw Exception(ErrorCodes::TABLE_IS_READ_ONLY, "Can't drop readonly replicated table (need to drop data in ZooKeeper as well)"); - - dropReplica(zookeeper, zookeeper_path, replica_name, log, getSettings()); } /// Wait for loading of all outdated parts because @@ -927,10 +926,13 @@ void StorageReplicatedMergeTree::drop() } dropAllData(); + + if (maybe_has_metadata_in_zookeeper) + dropReplica(zookeeper, zookeeper_path, replica_name, log, getSettings(), &has_metadata_in_zookeeper); } void StorageReplicatedMergeTree::dropReplica(zkutil::ZooKeeperPtr zookeeper, const String & zookeeper_path, const String & replica, - Poco::Logger * logger, MergeTreeSettingsPtr table_settings) + Poco::Logger * logger, MergeTreeSettingsPtr table_settings, std::optional * has_metadata_out) { if (zookeeper->expired()) throw Exception(ErrorCodes::TABLE_WAS_NOT_DROPPED, "Table was not dropped because ZooKeeper session has expired."); @@ -988,12 +990,16 @@ void StorageReplicatedMergeTree::dropReplica(zkutil::ZooKeeperPtr zookeeper, con Coordination::errorMessage(code), remote_replica_path); /// And finally remove everything else recursively - zookeeper->tryRemoveRecursive(remote_replica_path); - } + /// It may left some garbage if replica_path subtree is concurrently modified + zookeeper->tryRemoveChildrenRecursive(remote_replica_path); - /// It may left some garbage if replica_path subtree are concurrently modified - if (zookeeper->exists(remote_replica_path)) - LOG_ERROR(logger, "Replica was not completely removed from ZooKeeper, {} still exists and may contain some garbage.", remote_replica_path); + /// Update has_metadata_in_zookeeper to avoid retries. Otherwise we can accidentally remove metadata of a new table on retries + if (has_metadata_out) + *has_metadata_out = false; + + if (zookeeper->tryRemove(remote_replica_path) != Coordination::Error::ZOK) + LOG_ERROR(logger, "Replica was not completely removed from ZooKeeper, {} still exists and may contain some garbage.", remote_replica_path); + } /// Check that `zookeeper_path` exists: it could have been deleted by another replica after execution of previous line. Strings replicas; @@ -8152,6 +8158,12 @@ StorageReplicatedMergeTree::unlockSharedData(const IMergeTreeDataPart & part, co auto shared_id = getTableSharedID(); if (shared_id == toString(UUIDHelpers::Nil)) { + if (zookeeper->exists(zookeeper_path)) + { + LOG_WARNING(log, "Not removing shared data for part {} because replica does not have metadata in ZooKeeper, " + "but table path exist and other replicas may exist. It may leave some garbage on S3", part.name); + return std::make_pair(false, NameSet{}); + } LOG_TRACE(log, "Part {} blobs can be removed, because table {} completely dropped", part.name, getStorageID().getNameForLogs()); return std::make_pair(true, NameSet{}); } @@ -8177,9 +8189,18 @@ StorageReplicatedMergeTree::unlockSharedData(const IMergeTreeDataPart & part, co return std::make_pair(true, NameSet{}); } - /// If table was completely dropped (no meta in zookeeper) we can safely remove parts if (has_metadata_in_zookeeper.has_value() && !has_metadata_in_zookeeper) + { + if (zookeeper->exists(zookeeper_path)) + { + LOG_WARNING(log, "Not removing shared data for part {} because replica does not have metadata in ZooKeeper, " + "but table path exist and other replicas may exist. It may leave some garbage on S3", part.name); + return std::make_pair(false, NameSet{}); + } + + /// If table was completely dropped (no meta in zookeeper) we can safely remove parts return std::make_pair(true, NameSet{}); + } /// We remove parts during table shutdown. If exception happen, restarting thread will be already turned /// off and nobody will reconnect our zookeeper connection. In this case we use zookeeper connection from diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 0d3856ce672..75e5629b627 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -228,7 +228,7 @@ public: /** Remove a specific replica from zookeeper. */ static void dropReplica(zkutil::ZooKeeperPtr zookeeper, const String & zookeeper_path, const String & replica, - Poco::Logger * logger, MergeTreeSettingsPtr table_settings = nullptr); + Poco::Logger * logger, MergeTreeSettingsPtr table_settings = nullptr, std::optional * has_metadata_out = nullptr); /// Removes table from ZooKeeper after the last replica was dropped static bool removeTableNodesFromZooKeeper(zkutil::ZooKeeperPtr zookeeper, const String & zookeeper_path, diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 0073c25e5d5..0c6b60ee166 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -3263,7 +3263,7 @@ class ClickHouseInstance: sleep_time=0.5, check_callback=lambda x: True, ): - logging.debug(f"Executing query {sql} on {self.name}") + #logging.debug(f"Executing query {sql} on {self.name}") result = None for i in range(retry_count): try: @@ -3282,7 +3282,7 @@ class ClickHouseInstance: return result time.sleep(sleep_time) except Exception as ex: - logging.debug("Retry {} got exception {}".format(i + 1, ex)) + #logging.debug("Retry {} got exception {}".format(i + 1, ex)) time.sleep(sleep_time) if result is not None: diff --git a/tests/integration/test_replicated_merge_tree_s3_zero_copy/configs/config.d/storage_conf.xml b/tests/integration/test_replicated_merge_tree_s3_zero_copy/configs/config.d/storage_conf.xml index bd59694f65a..15239041478 100644 --- a/tests/integration/test_replicated_merge_tree_s3_zero_copy/configs/config.d/storage_conf.xml +++ b/tests/integration/test_replicated_merge_tree_s3_zero_copy/configs/config.d/storage_conf.xml @@ -53,4 +53,6 @@ 0 + 3 + 0 diff --git a/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py b/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py index 60a1b9b9746..434957de53f 100644 --- a/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py +++ b/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py @@ -1,9 +1,11 @@ import logging import random import string +import time import pytest from helpers.cluster import ClickHouseCluster +from helpers.network import PartitionManager logging.getLogger().setLevel(logging.INFO) logging.getLogger().addHandler(logging.StreamHandler()) @@ -127,3 +129,57 @@ def test_insert_select_replicated(cluster, min_rows_for_wide_part, files_per_par assert len( list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True)) ) == (3 * FILES_OVERHEAD) + (files_per_part * 3) + + +def test_drop_table(cluster): + node = list(cluster.instances.values())[0] + node2 = list(cluster.instances.values())[1] + node.query( + "create table test_drop_table (n int) engine=ReplicatedMergeTree('/test/drop_table', '1') order by n partition by n % 99 settings storage_policy='s3'" + ) + node2.query( + "create table test_drop_table (n int) engine=ReplicatedMergeTree('/test/drop_table', '2') order by n partition by n % 99 settings storage_policy='s3'" + ) + node.query("insert into test_drop_table select * from numbers(1000)") + node2.query("system sync replica test_drop_table") + + with PartitionManager() as pm: + pm._add_rule( + { + "probability": 0.01, + "destination": node.ip_address, + "source_port": 2181, + "action": "REJECT --reject-with tcp-reset", + } + ) + pm._add_rule( + { + "probability": 0.01, + "source": node.ip_address, + "destination_port": 2181, + "action": "REJECT --reject-with tcp-reset", + } + ) + node.query("drop table test_drop_table") + for i in range(0, 100): + node.query_and_get_answer_with_error( + "create table if not exists test_drop_table (n int) " + "engine=ReplicatedMergeTree('/test/drop_table', '1') " + "order by n partition by n % 99 settings storage_policy='s3'" + ) + time.sleep(0.2) + + replicas = node.query_with_retry( + "select name from system.zookeeper where path='/test/drop_table/replicas'" + ) + if "1" in replicas and "test_drop_table" not in node.query("show tables"): + node2.query("system drop replica '1' from table test_drop_table") + node.query( + "create table test_drop_table (n int) engine=ReplicatedMergeTree('/test/drop_table', '1') " + "order by n partition by n % 99 settings storage_policy='s3'" + ) + node.query("system sync replica test_drop_table", settings={"receive_timeout": 60}) + node2.query("drop table test_drop_table") + assert "1000\t499500\n" == node.query( + "select count(n), sum(n) from test_drop_table" + ) From dce8dca4e5f6d0391fd52e287aea42a09645bfa7 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 13 Feb 2023 23:53:15 +0000 Subject: [PATCH 188/717] Automatic style fix --- tests/integration/helpers/cluster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 0c6b60ee166..f6445f07482 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -3263,7 +3263,7 @@ class ClickHouseInstance: sleep_time=0.5, check_callback=lambda x: True, ): - #logging.debug(f"Executing query {sql} on {self.name}") + # logging.debug(f"Executing query {sql} on {self.name}") result = None for i in range(retry_count): try: @@ -3282,7 +3282,7 @@ class ClickHouseInstance: return result time.sleep(sleep_time) except Exception as ex: - #logging.debug("Retry {} got exception {}".format(i + 1, ex)) + # logging.debug("Retry {} got exception {}".format(i + 1, ex)) time.sleep(sleep_time) if result is not None: From bbf94a2664be9b5ba3bdc5f10ccde150cae08f34 Mon Sep 17 00:00:00 2001 From: lzydmxy <13126752315@163.com> Date: Tue, 14 Feb 2023 17:29:44 +0800 Subject: [PATCH 189/717] Apply `ALTER TABLE table_name ON CLUSTER cluster MOVE PARTITION|PART partition_expr TO DISK|VOLUME 'disk_name'` to all replicas. Because `ALTER TABLE t MOVE` is not replicated. --- src/Interpreters/DDLWorker.cpp | 7 +++---- src/Parsers/ASTAlterQuery.cpp | 18 ++++++++++++++++++ src/Parsers/ASTAlterQuery.h | 2 ++ 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index 0f91212e6a9..54d1067de31 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -691,10 +691,9 @@ bool DDLWorker::taskShouldBeExecutedOnLeader(const ASTPtr & ast_ddl, const Stora if (auto * alter = ast_ddl->as()) { // Setting alters should be executed on all replicas - if (alter->isSettingsAlter()) - return false; - - if (alter->isFreezeAlter()) + if (alter->isSettingsAlter() || + alter->isFreezeAlter() || + alter->isMovePartitionToDiskOrVolumeAlter()) return false; } diff --git a/src/Parsers/ASTAlterQuery.cpp b/src/Parsers/ASTAlterQuery.cpp index 5d347446d37..426b63a9d28 100644 --- a/src/Parsers/ASTAlterQuery.cpp +++ b/src/Parsers/ASTAlterQuery.cpp @@ -533,6 +533,24 @@ bool ASTAlterQuery::isDropPartitionAlter() const return isOneCommandTypeOnly(ASTAlterCommand::DROP_PARTITION) || isOneCommandTypeOnly(ASTAlterCommand::DROP_DETACHED_PARTITION); } +bool ASTAlterQuery::isMovePartitionToDiskOrVolumeAlter() const +{ + if (command_list) + { + if (command_list->children.empty()) + return false; + for (const auto & child : command_list->children) + { + const auto & command = child->as(); + if (command.type != ASTAlterCommand::MOVE_PARTITION || + (command.move_destination_type != DataDestinationType::DISK && command.move_destination_type != DataDestinationType::VOLUME)) + return false; + } + return true; + } + return false; +} + /** Get the text that identifies this element. */ String ASTAlterQuery::getID(char delim) const diff --git a/src/Parsers/ASTAlterQuery.h b/src/Parsers/ASTAlterQuery.h index 4a8c9c14ea9..2a48f5bbd9e 100644 --- a/src/Parsers/ASTAlterQuery.h +++ b/src/Parsers/ASTAlterQuery.h @@ -239,6 +239,8 @@ public: bool isDropPartitionAlter() const; + bool isMovePartitionToDiskOrVolumeAlter() const; + String getID(char) const override; ASTPtr clone() const override; From 7e24b5f059b5ebca569327f856bfc07e1a54b70f Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 14 Feb 2023 12:10:02 +0000 Subject: [PATCH 190/717] Fix tests & review comment --- .../Optimizations/removeRedundantDistinct.cpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp index e37e08cd7b8..25afd1cf2d4 100644 --- a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp +++ b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp @@ -29,7 +29,7 @@ namespace void logActionsDAG(const String & prefix, const ActionsDAGPtr & actions) { if constexpr (debug_logging_enabled) - LOG_DEBUG(&Poco::Logger::get("redundantDistinct"), "{}: {}", prefix, actions->dumpDAG()); + LOG_DEBUG(&Poco::Logger::get("redundantDistinct"), "{} :\n{}", prefix, actions->dumpDAG()); } void logDebug(String key, String value) @@ -44,10 +44,10 @@ namespace /// find non-const columns in DISTINCT const ColumnsWithTypeAndName & distinct_columns = distinct->getOutputStream().header.getColumnsWithTypeAndName(); std::set non_const_columns; - const Names & column_names = distinct->getColumnNames(); + std::unordered_set column_names(cbegin(distinct->getColumnNames()), cend(distinct->getColumnNames())); for (const auto & column : distinct_columns) { - if (!isColumnConst(*column.column) && find(cbegin(column_names), cend(column_names), column.name) != column_names.cend()) + if (!isColumnConst(*column.column) && column_names.contains(column.name)) non_const_columns.emplace(column.name); } return non_const_columns; @@ -124,11 +124,12 @@ namespace if (dag_stack.empty()) return nullptr; - ActionsDAGPtr path_actions = dag_stack.back(); + ActionsDAGPtr path_actions = dag_stack.back()->clone(); dag_stack.pop_back(); while (!dag_stack.empty()) { ActionsDAGPtr clone = dag_stack.back()->clone(); + logActionsDAG("DAG to merge", clone); dag_stack.pop_back(); path_actions->mergeInplace(std::move(*clone)); } @@ -174,7 +175,7 @@ namespace if (aggregation_before_distinct) { ActionsDAGPtr actions = buildActionsForPlanPath(dag_stack); - logActionsDAG("aggregation pass: merged DAG:\n{}", actions); + logActionsDAG("aggregation pass: merged DAG", actions); const auto distinct_columns = getDistinctColumns(distinct_step); @@ -239,7 +240,7 @@ namespace { /// build actions DAG to find original column names path_actions = buildActionsForPlanPath(dag_stack); - logActionsDAG("distinct pass: merged DAG:\n{}", path_actions); + logActionsDAG("distinct pass: merged DAG", path_actions); /// compare columns of two DISTINCTs for (const auto & column : distinct_columns) From b9d31601078a35965586ae262bb1e8179c288e2f Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Tue, 14 Feb 2023 13:01:06 +0000 Subject: [PATCH 191/717] Add parameterized RENAME queries --- src/Interpreters/AddDefaultDatabaseVisitor.h | 8 +-- src/Interpreters/InterpreterCreateQuery.cpp | 12 +++- src/Interpreters/InterpreterRenameQuery.cpp | 20 +++--- src/Interpreters/InterpreterRenameQuery.h | 8 +-- .../MySQL/InterpretersMySQLDDLQuery.cpp | 16 ++--- src/Interpreters/SystemLog.cpp | 31 ++++----- src/Parsers/ASTRenameQuery.h | 36 ++++++++--- src/Parsers/ParserRenameQuery.cpp | 64 +++++++------------ .../PostgreSQLReplicationHandler.cpp | 12 +++- src/Storages/StorageMaterializedView.cpp | 25 ++++---- ...661_parameterized_rename_queries.reference | 3 + .../02661_parameterized_rename_queries.sql | 52 +++++++++++++++ 12 files changed, 180 insertions(+), 107 deletions(-) create mode 100644 tests/queries/0_stateless/02661_parameterized_rename_queries.reference create mode 100644 tests/queries/0_stateless/02661_parameterized_rename_queries.sql diff --git a/src/Interpreters/AddDefaultDatabaseVisitor.h b/src/Interpreters/AddDefaultDatabaseVisitor.h index 2e63e6e1b43..08d159b42ca 100644 --- a/src/Interpreters/AddDefaultDatabaseVisitor.h +++ b/src/Interpreters/AddDefaultDatabaseVisitor.h @@ -270,10 +270,10 @@ private: for (ASTRenameQuery::Element & elem : node.elements) { - if (elem.from.database.empty()) - elem.from.database = database_name; - if (elem.to.database.empty()) - elem.to.database = database_name; + if (!elem.from.database) + elem.from.database = std::make_shared(database_name); + if (!elem.to.database) + elem.to.database = std::make_shared(database_name); } } diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 7e5a4688034..cdd4d836c14 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1494,8 +1494,16 @@ BlockIO InterpreterCreateQuery::doCreateOrReplaceTable(ASTCreateQuery & create, auto ast_rename = std::make_shared(); ASTRenameQuery::Element elem { - ASTRenameQuery::Table{create.getDatabase(), create.getTable()}, - ASTRenameQuery::Table{create.getDatabase(), table_to_replace_name} + ASTRenameQuery::Table + { + create.getDatabase().empty() ? nullptr : std::make_shared(create.getDatabase()), + std::make_shared(create.getTable()) + }, + ASTRenameQuery::Table + { + create.getDatabase().empty() ? nullptr : std::make_shared(create.getDatabase()), + std::make_shared(table_to_replace_name) + } }; ast_rename->elements.push_back(std::move(elem)); diff --git a/src/Interpreters/InterpreterRenameQuery.cpp b/src/Interpreters/InterpreterRenameQuery.cpp index 3dfdd867ae7..01c04999287 100644 --- a/src/Interpreters/InterpreterRenameQuery.cpp +++ b/src/Interpreters/InterpreterRenameQuery.cpp @@ -180,18 +180,18 @@ AccessRightsElements InterpreterRenameQuery::getRequiredAccess(InterpreterRename { if (type == RenameType::RenameTable) { - required_access.emplace_back(AccessType::SELECT | AccessType::DROP_TABLE, elem.from.database, elem.from.table); - required_access.emplace_back(AccessType::CREATE_TABLE | AccessType::INSERT, elem.to.database, elem.to.table); + required_access.emplace_back(AccessType::SELECT | AccessType::DROP_TABLE, elem.from.getDatabase(), elem.from.getTable()); + required_access.emplace_back(AccessType::CREATE_TABLE | AccessType::INSERT, elem.to.getDatabase(), elem.to.getTable()); if (rename.exchange) { - required_access.emplace_back(AccessType::CREATE_TABLE | AccessType::INSERT , elem.from.database, elem.from.table); - required_access.emplace_back(AccessType::SELECT | AccessType::DROP_TABLE, elem.to.database, elem.to.table); + required_access.emplace_back(AccessType::CREATE_TABLE | AccessType::INSERT , elem.from.getDatabase(), elem.from.getTable()); + required_access.emplace_back(AccessType::SELECT | AccessType::DROP_TABLE, elem.to.getDatabase(), elem.to.getTable()); } } else if (type == RenameType::RenameDatabase) { - required_access.emplace_back(AccessType::SELECT | AccessType::DROP_DATABASE, elem.from.database); - required_access.emplace_back(AccessType::CREATE_DATABASE | AccessType::INSERT, elem.to.database); + required_access.emplace_back(AccessType::SELECT | AccessType::DROP_DATABASE, elem.from.getDatabase()); + required_access.emplace_back(AccessType::CREATE_DATABASE | AccessType::INSERT, elem.to.getDatabase()); } else { @@ -207,14 +207,14 @@ void InterpreterRenameQuery::extendQueryLogElemImpl(QueryLogElement & elem, cons for (const auto & element : rename.elements) { { - String database = backQuoteIfNeed(element.from.database.empty() ? getContext()->getCurrentDatabase() : element.from.database); + String database = backQuoteIfNeed(!element.from.database ? getContext()->getCurrentDatabase() : element.from.getDatabase()); elem.query_databases.insert(database); - elem.query_tables.insert(database + "." + backQuoteIfNeed(element.from.table)); + elem.query_tables.insert(database + "." + backQuoteIfNeed(element.from.getTable())); } { - String database = backQuoteIfNeed(element.to.database.empty() ? getContext()->getCurrentDatabase() : element.to.database); + String database = backQuoteIfNeed(!element.to.database ? getContext()->getCurrentDatabase() : element.to.getDatabase()); elem.query_databases.insert(database); - elem.query_tables.insert(database + "." + backQuoteIfNeed(element.to.table)); + elem.query_tables.insert(database + "." + backQuoteIfNeed(element.to.getTable())); } } } diff --git a/src/Interpreters/InterpreterRenameQuery.h b/src/Interpreters/InterpreterRenameQuery.h index 31d3d3d6ad9..25c707d9962 100644 --- a/src/Interpreters/InterpreterRenameQuery.h +++ b/src/Interpreters/InterpreterRenameQuery.h @@ -28,10 +28,10 @@ struct UniqueTableName struct RenameDescription { RenameDescription(const ASTRenameQuery::Element & elem, const String & current_database) : - from_database_name(elem.from.database.empty() ? current_database : elem.from.database), - from_table_name(elem.from.table), - to_database_name(elem.to.database.empty() ? current_database : elem.to.database), - to_table_name(elem.to.table), + from_database_name(!elem.from.database ? current_database : elem.from.getDatabase()), + from_table_name(elem.from.getTable()), + to_database_name(!elem.to.database ? current_database : elem.to.getDatabase()), + to_table_name(elem.to.getTable()), if_exists(elem.if_exists) {} diff --git a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp index 7ba7749e89b..4e54b806bd1 100644 --- a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp +++ b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp @@ -581,8 +581,8 @@ ASTs InterpreterRenameImpl::getRewrittenQueries( ASTRenameQuery::Elements elements; for (const auto & rename_element : rename_query.elements) { - const auto & to_database = resolveDatabase(rename_element.to.database, mysql_database, mapped_to_database, context); - const auto & from_database = resolveDatabase(rename_element.from.database, mysql_database, mapped_to_database, context); + const auto & to_database = resolveDatabase(rename_element.to.getDatabase(), mysql_database, mapped_to_database, context); + const auto & from_database = resolveDatabase(rename_element.from.getDatabase(), mysql_database, mapped_to_database, context); if ((from_database == mapped_to_database || to_database == mapped_to_database) && to_database != from_database) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot rename with other database for external ddl query."); @@ -590,9 +590,9 @@ ASTs InterpreterRenameImpl::getRewrittenQueries( if (from_database == mapped_to_database) { elements.push_back(ASTRenameQuery::Element()); - elements.back().from.database = mapped_to_database; + elements.back().from.database = std::make_shared(mapped_to_database); elements.back().from.table = rename_element.from.table; - elements.back().to.database = mapped_to_database; + elements.back().to.database = std::make_shared(mapped_to_database); elements.back().to.table = rename_element.to.table; } } @@ -758,10 +758,10 @@ ASTs InterpreterAlterImpl::getRewrittenQueries( if (rewritten_rename_query->elements.empty()) rewritten_rename_query->elements.push_back(ASTRenameQuery::Element()); - rewritten_rename_query->elements.back().from.database = mapped_to_database; - rewritten_rename_query->elements.back().from.table = alter_query.table; - rewritten_rename_query->elements.back().to.database = mapped_to_database; - rewritten_rename_query->elements.back().to.table = alter_command->new_table_name; + rewritten_rename_query->elements.back().from.database = std::make_shared(mapped_to_database); + rewritten_rename_query->elements.back().from.table = std::make_shared(alter_query.table); + rewritten_rename_query->elements.back().to.database = std::make_shared(mapped_to_database); + rewritten_rename_query->elements.back().to.table = std::make_shared(alter_command->new_table_name); } } diff --git a/src/Interpreters/SystemLog.cpp b/src/Interpreters/SystemLog.cpp index c280177ae63..9d8547abcf2 100644 --- a/src/Interpreters/SystemLog.cpp +++ b/src/Interpreters/SystemLog.cpp @@ -476,29 +476,30 @@ void SystemLog::prepareTable() ++suffix; auto rename = std::make_shared(); - - ASTRenameQuery::Table from; - from.database = table_id.database_name; - from.table = table_id.table_name; - - ASTRenameQuery::Table to; - to.database = table_id.database_name; - to.table = table_id.table_name + "_" + toString(suffix); - - ASTRenameQuery::Element elem; - elem.from = from; - elem.to = to; - - rename->elements.emplace_back(elem); + ASTRenameQuery::Element elem + { + ASTRenameQuery::Table + { + table_id.database_name.empty() ? nullptr : std::make_shared(table_id.database_name), + std::make_shared(table_id.table_name) + }, + ASTRenameQuery::Table + { + table_id.database_name.empty() ? nullptr : std::make_shared(table_id.database_name), + std::make_shared(table_id.table_name + "_" + toString(suffix)) + } + }; LOG_DEBUG( log, "Existing table {} for system log has obsolete or different structure. Renaming it to {}.\nOld: {}\nNew: {}\n.", description, - backQuoteIfNeed(to.table), + backQuoteIfNeed(elem.to.getTable()), old_create_query, create_query); + rename->elements.emplace_back(std::move(elem)); + auto query_context = Context::createCopy(context); query_context->makeQueryContext(); InterpreterRenameQuery(rename, query_context).execute(); diff --git a/src/Parsers/ASTRenameQuery.h b/src/Parsers/ASTRenameQuery.h index 723f680b492..5d07cb976af 100644 --- a/src/Parsers/ASTRenameQuery.h +++ b/src/Parsers/ASTRenameQuery.h @@ -3,6 +3,8 @@ #include #include #include +#include +#include #include #include @@ -17,8 +19,22 @@ class ASTRenameQuery : public ASTQueryWithOutput, public ASTQueryWithOnCluster public: struct Table { - String database; - String table; + ASTPtr database; + ASTPtr table; + + String getDatabase() const + { + String name; + tryGetIdentifierNameInto(database, name); + return name; + } + + String getTable() const + { + String name; + tryGetIdentifierNameInto(table, name); + return name; + } }; struct Element @@ -56,10 +72,10 @@ public: query.cluster.clear(); for (Element & elem : query.elements) { - if (elem.from.database.empty()) - elem.from.database = params.default_database; - if (elem.to.database.empty()) - elem.to.database = params.default_database; + if (!elem.from.database) + elem.from.database = std::make_shared(params.default_database); + if (!elem.to.database) + elem.to.database = std::make_shared(params.default_database); } return query_ptr; @@ -77,9 +93,9 @@ protected: if (elements.at(0).if_exists) settings.ostr << (settings.hilite ? hilite_keyword : "") << "IF EXISTS " << (settings.hilite ? hilite_none : ""); - settings.ostr << backQuoteIfNeed(elements.at(0).from.database); + settings.ostr << backQuoteIfNeed(elements.at(0).from.getDatabase()); settings.ostr << (settings.hilite ? hilite_keyword : "") << " TO " << (settings.hilite ? hilite_none : ""); - settings.ostr << backQuoteIfNeed(elements.at(0).to.database); + settings.ostr << backQuoteIfNeed(elements.at(0).to.getDatabase()); formatOnCluster(settings); return; } @@ -103,9 +119,9 @@ protected: if (it->if_exists) settings.ostr << (settings.hilite ? hilite_keyword : "") << "IF EXISTS " << (settings.hilite ? hilite_none : ""); - settings.ostr << (!it->from.database.empty() ? backQuoteIfNeed(it->from.database) + "." : "") << backQuoteIfNeed(it->from.table) + settings.ostr << (it->from.database ? backQuoteIfNeed(it->from.getDatabase()) + "." : "") << backQuoteIfNeed(it->from.getTable()) << (settings.hilite ? hilite_keyword : "") << (exchange ? " AND " : " TO ") << (settings.hilite ? hilite_none : "") - << (!it->to.database.empty() ? backQuoteIfNeed(it->to.database) + "." : "") << backQuoteIfNeed(it->to.table); + << (it->to.database ? backQuoteIfNeed(it->to.getDatabase()) + "." : "") << backQuoteIfNeed(it->to.getTable()); } formatOnCluster(settings); diff --git a/src/Parsers/ParserRenameQuery.cpp b/src/Parsers/ParserRenameQuery.cpp index e180d998df4..cb98554afba 100644 --- a/src/Parsers/ParserRenameQuery.cpp +++ b/src/Parsers/ParserRenameQuery.cpp @@ -3,40 +3,12 @@ #include #include +#include namespace DB { - -/// Parse database.table or table. -static bool parseDatabaseAndTable( - ASTRenameQuery::Table & db_and_table, IParser::Pos & pos, Expected & expected) -{ - ParserIdentifier name_p; - ParserToken s_dot(TokenType::Dot); - - ASTPtr database; - ASTPtr table; - - if (!name_p.parse(pos, table, expected)) - return false; - - if (s_dot.ignore(pos, expected)) - { - database = table; - if (!name_p.parse(pos, table, expected)) - return false; - } - - db_and_table.database.clear(); - tryGetIdentifierNameInto(database, db_and_table.database); - tryGetIdentifierNameInto(table, db_and_table.table); - - return true; -} - - bool ParserRenameQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { ParserKeyword s_rename_table("RENAME TABLE"); @@ -67,7 +39,7 @@ bool ParserRenameQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { ASTPtr from_db; ASTPtr to_db; - ParserIdentifier db_name_p; + ParserIdentifier db_name_p(true); bool if_exists = s_if_exists.ignore(pos, expected); if (!db_name_p.parse(pos, from_db, expected)) return false; @@ -87,8 +59,10 @@ bool ParserRenameQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) query->database = true; query->elements.emplace({}); query->elements.front().if_exists = if_exists; - tryGetIdentifierNameInto(from_db, query->elements.front().from.database); - tryGetIdentifierNameInto(to_db, query->elements.front().to.database); + query->elements.front().from.database = from_db; + query->elements.front().to.database = to_db; + query->children.push_back(from_db); + query->children.push_back(to_db); query->cluster = cluster_str; node = query; return true; @@ -96,24 +70,35 @@ bool ParserRenameQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) else return false; - ASTRenameQuery::Elements elements; - const auto ignore_delim = [&] { return exchange ? s_and.ignore(pos) : s_to.ignore(pos); }; + auto query = std::make_shared(); + + ASTRenameQuery::Elements & elements = query->elements; + while (true) { if (!elements.empty() && !s_comma.ignore(pos)) break; - ASTRenameQuery::Element& ref = elements.emplace_back(); + ASTRenameQuery::Element & ref = elements.emplace_back(); if (!exchange) ref.if_exists = s_if_exists.ignore(pos, expected); - if (!parseDatabaseAndTable(ref.from, pos, expected) + if (!parseDatabaseAndTableAsAST(pos, expected, ref.from.database, ref.from.table) || !ignore_delim() - || !parseDatabaseAndTable(ref.to, pos, expected)) + || !parseDatabaseAndTableAsAST(pos, expected, ref.to.database, ref.to.table)) return false; + + if (ref.from.database) + query->children.push_back(ref.from.database); + if (ref.from.table) + query->children.push_back(ref.from.table); + if (ref.to.database) + query->children.push_back(ref.to.database); + if (ref.to.table) + query->children.push_back(ref.to.table); } String cluster_str; @@ -123,13 +108,10 @@ bool ParserRenameQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) return false; } - auto query = std::make_shared(); query->cluster = cluster_str; - node = query; - - query->elements = elements; query->exchange = exchange; query->dictionary = dictionary; + node = query; return true; } diff --git a/src/Storages/PostgreSQL/PostgreSQLReplicationHandler.cpp b/src/Storages/PostgreSQL/PostgreSQLReplicationHandler.cpp index f450604fded..d9f63bda6d0 100644 --- a/src/Storages/PostgreSQL/PostgreSQLReplicationHandler.cpp +++ b/src/Storages/PostgreSQL/PostgreSQLReplicationHandler.cpp @@ -982,8 +982,16 @@ void PostgreSQLReplicationHandler::reloadFromSnapshot(const std::vector(); ASTRenameQuery::Element elem { - ASTRenameQuery::Table{table_id.database_name, table_id.table_name}, - ASTRenameQuery::Table{temp_table_id.database_name, temp_table_id.table_name} + ASTRenameQuery::Table + { + table_id.database_name.empty() ? nullptr : std::make_shared(table_id.database_name), + std::make_shared(table_id.table_name) + }, + ASTRenameQuery::Table + { + temp_table_id.database_name.empty() ? nullptr : std::make_shared(temp_table_id.database_name), + std::make_shared(temp_table_id.table_name) + } }; ast_rename->elements.push_back(std::move(elem)); ast_rename->exchange = true; diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp index 971ecf8dbf2..be090bd9a62 100644 --- a/src/Storages/StorageMaterializedView.cpp +++ b/src/Storages/StorageMaterializedView.cpp @@ -336,19 +336,22 @@ void StorageMaterializedView::renameInMemory(const StorageID & new_table_id) auto new_target_table_name = generateInnerTableName(new_table_id); auto rename = std::make_shared(); - ASTRenameQuery::Table from; assert(target_table_id.database_name == old_table_id.database_name); - from.database = target_table_id.database_name; - from.table = target_table_id.table_name; - ASTRenameQuery::Table to; - to.database = new_table_id.database_name; - to.table = new_target_table_name; - - ASTRenameQuery::Element elem; - elem.from = from; - elem.to = to; - rename->elements.emplace_back(elem); + ASTRenameQuery::Element elem + { + ASTRenameQuery::Table + { + target_table_id.database_name.empty() ? nullptr : std::make_shared(target_table_id.database_name), + std::make_shared(target_table_id.table_name) + }, + ASTRenameQuery::Table + { + target_table_id.database_name.empty() ? nullptr : std::make_shared(target_table_id.database_name), + std::make_shared(new_target_table_name) + } + }; + rename->elements.emplace_back(std::move(elem)); InterpreterRenameQuery(rename, getContext()).execute(); target_table_id.database_name = new_table_id.database_name; diff --git a/tests/queries/0_stateless/02661_parameterized_rename_queries.reference b/tests/queries/0_stateless/02661_parameterized_rename_queries.reference new file mode 100644 index 00000000000..e21faf9023e --- /dev/null +++ b/tests/queries/0_stateless/02661_parameterized_rename_queries.reference @@ -0,0 +1,3 @@ +02661_db1 +02661_t1 +02661_d1 diff --git a/tests/queries/0_stateless/02661_parameterized_rename_queries.sql b/tests/queries/0_stateless/02661_parameterized_rename_queries.sql new file mode 100644 index 00000000000..eecb282083f --- /dev/null +++ b/tests/queries/0_stateless/02661_parameterized_rename_queries.sql @@ -0,0 +1,52 @@ +-- Tags: no-parallel + +-- Case 1: RENAME DATABASE + +DROP DATABASE IF EXISTS 02661_db; +DROP DATABASE IF EXISTS 02661_db1; + +SET param_old_db_name = 02661_db; +SET param_new_db_name = 02661_db1; + +CREATE DATABASE {old_db_name:Identifier}; +RENAME DATABASE {old_db_name:Identifier} TO {new_db_name:Identifier}; + +SELECT name FROM system.databases WHERE name = {new_db_name:String}; + +-- Case 2: RENAME TABLE + +DROP TABLE IF EXISTS 02661_t; +DROP TABLE IF EXISTS 02661_t1; + +SET param_old_tbl_name = 02661_t; +SET param_new_tbl_name = 02661_t1; + +CREATE TABLE {new_db_name:Identifier}.{old_tbl_name:Identifier} (a UInt64) ENGINE = MergeTree ORDER BY tuple(); +RENAME TABLE {new_db_name:Identifier}.{old_tbl_name:Identifier} TO {new_db_name:Identifier}.{new_tbl_name:Identifier}; + +SELECT name FROM system.tables WHERE name = {new_tbl_name:String}; + +-- Case 3: RENAME DICTIONARY + +DROP DICTIONARY IF EXISTS 02661_d; +DROP DICTIONARY IF EXISTS 02661_d1; + +SET param_old_dict_name = 02661_d; +SET param_new_dict_name = 02661_d1; + +CREATE DICTIONARY {new_db_name:Identifier}.{old_dict_name:Identifier} (id UInt64, val UInt8) PRIMARY KEY id SOURCE(NULL()) LAYOUT(FLAT()) LIFETIME(0); +RENAME DICTIONARY {new_db_name:Identifier}.{old_dict_name:Identifier} TO {new_db_name:Identifier}.{new_dict_name:Identifier}; + +SELECT name FROM system.dictionaries WHERE name = {new_dict_name:String}; + +-- Case 4: EXCHANGE TABLES + +CREATE TABLE {new_db_name:Identifier}.{old_tbl_name:Identifier} (a UInt64) ENGINE = MergeTree ORDER BY tuple(); +EXCHANGE TABLES {new_db_name:Identifier}.{old_tbl_name:Identifier} AND {new_db_name:Identifier}.{new_tbl_name:Identifier}; + +-- Case 5: EXCHANGE DICTIONARIES + +CREATE DICTIONARY {new_db_name:Identifier}.{old_dict_name:Identifier} (id UInt64, val UInt8) PRIMARY KEY id SOURCE(NULL()) LAYOUT(FLAT()) LIFETIME(0); +EXCHANGE DICTIONARIES {new_db_name:Identifier}.{old_dict_name:Identifier} AND {new_db_name:Identifier}.{new_dict_name:Identifier}; + +DROP DATABASE {new_db_name:Identifier}; From f7604cc686dd10eaf11b686385f959d14d449af4 Mon Sep 17 00:00:00 2001 From: John Skopis Date: Fri, 19 Nov 2021 15:19:20 +0000 Subject: [PATCH 192/717] [feat] Add ProfileEvents map to PartLog closes #10316 --- src/Common/CurrentMetrics.cpp | 1 + src/Interpreters/PartLog.cpp | 32 ++++++++++++++-- src/Interpreters/PartLog.h | 14 +++++-- src/Storages/MergeTree/IExecutableTask.h | 12 ++++++ .../MergeTree/MergeFromLogEntryTask.cpp | 5 ++- .../MergeTree/MergePlainMergeTreeTask.cpp | 6 ++- .../MergeTree/MergePlainMergeTreeTask.h | 15 ++++++++ .../MergeTree/MergeTreeBackgroundExecutor.cpp | 18 +++++++++ src/Storages/MergeTree/MergeTreeData.cpp | 8 +++- src/Storages/MergeTree/MergeTreeData.h | 3 +- .../MergeTree/MutateFromLogEntryTask.cpp | 5 ++- .../MergeTree/MutatePlainMergeTreeTask.cpp | 6 ++- .../MergeTree/MutatePlainMergeTreeTask.h | 14 +++++++ src/Storages/MergeTree/MutateTask.cpp | 38 +++++++++++++++++++ .../MergeTree/ReplicatedMergeMutateTaskBase.h | 10 +++++ src/Storages/MergeTree/TaskObserverMetrics.h | 37 ++++++++++++++++++ .../MergeTree/tests/gtest_executor.cpp | 15 ++++++++ src/Storages/StorageReplicatedMergeTree.cpp | 15 ++++++-- 18 files changed, 239 insertions(+), 15 deletions(-) create mode 100644 src/Storages/MergeTree/TaskObserverMetrics.h diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index 0395f2470af..62d56dead62 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -13,6 +13,7 @@ M(BackgroundMergesAndMutationsPoolTask, "Number of active merges and mutations in an associated background pool") \ M(BackgroundFetchesPoolTask, "Number of active fetches in an associated background pool") \ M(BackgroundCommonPoolTask, "Number of active tasks in an associated background pool") \ + M(BackgroundMaintPoolTask, "Number of active tasks in BackgroundProcessingPool (maint)") \ M(BackgroundMovePoolTask, "Number of active tasks in BackgroundProcessingPool for moves") \ M(BackgroundSchedulePoolTask, "Number of active tasks in BackgroundSchedulePool. This pool is used for periodic ReplicatedMergeTree tasks, like cleaning old data parts, altering data parts, replica re-initialization, etc.") \ M(BackgroundBufferFlushSchedulePoolTask, "Number of active tasks in BackgroundBufferFlushSchedulePool. This pool is used for periodic Buffer flushes") \ diff --git a/src/Interpreters/PartLog.cpp b/src/Interpreters/PartLog.cpp index 4a1349680fd..b39f7423bbe 100644 --- a/src/Interpreters/PartLog.cpp +++ b/src/Interpreters/PartLog.cpp @@ -11,6 +11,9 @@ #include #include #include +#include +#include +#include #include @@ -121,6 +124,17 @@ NamesAndTypesList PartLogElement::getNamesAndTypes() /// Is there an error during the execution or commit {"error", std::make_shared()}, {"exception", std::make_shared()}, + + {"ProfileEvents", std::make_shared(std::make_shared(), std::make_shared())}, + }; +} + +NamesAndAliases PartLogElement::getNamesAndAliases() +{ + return + { + {"ProfileEvents.Names", {std::make_shared(std::make_shared())}, "mapKeys(ProfileEvents)"}, + {"ProfileEvents.Values", {std::make_shared(std::make_shared())}, "mapValues(ProfileEvents)"}, }; } @@ -163,18 +177,28 @@ void PartLogElement::appendToBlock(MutableColumns & columns) const columns[i++]->insert(error); columns[i++]->insert(exception); + + if (profile_counters) + { + auto * column = columns[i++].get(); + ProfileEvents::dumpToMapColumn(*profile_counters, column, true); + } + else + { + columns[i++]->insertDefault(); + } } bool PartLog::addNewPart( - ContextPtr current_context, const MutableDataPartPtr & part, UInt64 elapsed_ns, const ExecutionStatus & execution_status) + ContextPtr current_context, const MutableDataPartPtr & part, UInt64 elapsed_ns, const ExecutionStatus & execution_status, std::shared_ptr profile_counters_) { - return addNewParts(current_context, {part}, elapsed_ns, execution_status); + return addNewParts(current_context, {part}, elapsed_ns, execution_status, profile_counters_); } bool PartLog::addNewParts( - ContextPtr current_context, const PartLog::MutableDataPartsVector & parts, UInt64 elapsed_ns, const ExecutionStatus & execution_status) + ContextPtr current_context, const PartLog::MutableDataPartsVector & parts, UInt64 elapsed_ns, const ExecutionStatus & execution_status, std::shared_ptr profile_counters_) { if (parts.empty()) return true; @@ -221,6 +245,8 @@ bool PartLog::addNewParts( elem.error = static_cast(execution_status.code); elem.exception = execution_status.message; + elem.profile_counters = profile_counters_; + part_log->add(elem); } } diff --git a/src/Interpreters/PartLog.h b/src/Interpreters/PartLog.h index 392e76d85d1..75b2539bda9 100644 --- a/src/Interpreters/PartLog.h +++ b/src/Interpreters/PartLog.h @@ -8,6 +8,10 @@ #include #include +namespace ProfileEvents +{ + class Counters; +} namespace DB { @@ -81,13 +85,15 @@ struct PartLogElement UInt16 error = 0; String exception; + std::shared_ptr profile_counters; + static std::string name() { return "PartLog"; } static MergeReasonType getMergeReasonType(MergeType merge_type); static PartMergeAlgorithm getMergeAlgorithm(MergeAlgorithm merge_algorithm_); static NamesAndTypesList getNamesAndTypes(); - static NamesAndAliases getNamesAndAliases() { return {}; } + static NamesAndAliases getNamesAndAliases(); void appendToBlock(MutableColumns & columns) const; static const char * getCustomColumnList() { return nullptr; } }; @@ -106,9 +112,11 @@ class PartLog : public SystemLog public: /// Add a record about creation of new part. static bool addNewPart(ContextPtr context, const MutableDataPartPtr & part, UInt64 elapsed_ns, - const ExecutionStatus & execution_status = {}); + const ExecutionStatus & execution_status = {}, + std::shared_ptr profile_counters_ = {}); static bool addNewParts(ContextPtr context, const MutableDataPartsVector & parts, UInt64 elapsed_ns, - const ExecutionStatus & execution_status = {}); + const ExecutionStatus & execution_status = {}, + std::shared_ptr profile_counters_ = {}); }; } diff --git a/src/Storages/MergeTree/IExecutableTask.h b/src/Storages/MergeTree/IExecutableTask.h index 9617960c182..d878d57dec8 100644 --- a/src/Storages/MergeTree/IExecutableTask.h +++ b/src/Storages/MergeTree/IExecutableTask.h @@ -29,7 +29,9 @@ class IExecutableTask { public: using TaskResultCallback = std::function; + virtual bool onResume() = 0; virtual bool executeStep() = 0; + virtual bool onSuspend() = 0; virtual void onCompleted() = 0; virtual StorageID getStorageID() = 0; virtual UInt64 getPriority() = 0; @@ -54,6 +56,11 @@ public: , job_result_callback(std::forward(job_result_callback_)) , id(id_) {} + bool onResume() override + { + return true; + } + bool executeStep() override { res = job_to_execute(); @@ -61,6 +68,11 @@ public: return false; } + bool onSuspend() override + { + return true; + } + void onCompleted() override { job_result_callback(!res); } StorageID getStorageID() override { return id; } UInt64 getPriority() override diff --git a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp index 2d2013bd648..fab9b7c0f7e 100644 --- a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp @@ -289,9 +289,12 @@ ReplicatedMergeMutateTaskBase::PrepareResult MergeFromLogEntryTask::prepare() return {true, true, [this, stopwatch = *stopwatch_ptr] (const ExecutionStatus & execution_status) { + auto & thread_status = CurrentThread::get(); + thread_status.finalizePerformanceCounters(); + auto profile_counters = std::make_shared(thread_status.performance_counters.getPartiallyAtomicSnapshot()); storage.writePartLog( PartLogElement::MERGE_PARTS, execution_status, stopwatch.elapsed(), - entry.new_part_name, part, parts, merge_mutate_entry.get()); + entry.new_part_name, part, parts, merge_mutate_entry.get(), profile_counters); }}; } diff --git a/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp b/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp index 9f24839f1e1..ccd132beb4e 100644 --- a/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp +++ b/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp @@ -85,6 +85,9 @@ void MergePlainMergeTreeTask::prepare() write_part_log = [this] (const ExecutionStatus & execution_status) { + auto & thread_status = CurrentThread::get(); + thread_status.finalizePerformanceCounters(); + auto profile_counters = std::make_shared(thread_status.performance_counters.getPartiallyAtomicSnapshot()); merge_task.reset(); storage.writePartLog( PartLogElement::MERGE_PARTS, @@ -93,7 +96,8 @@ void MergePlainMergeTreeTask::prepare() future_part->name, new_part, future_part->parts, - merge_list_entry.get()); + merge_list_entry.get(), + profile_counters); }; merge_task = storage.merger_mutator.mergePartsToTemporaryPart( diff --git a/src/Storages/MergeTree/MergePlainMergeTreeTask.h b/src/Storages/MergeTree/MergePlainMergeTreeTask.h index d84db36bac2..472518c2723 100644 --- a/src/Storages/MergeTree/MergePlainMergeTreeTask.h +++ b/src/Storages/MergeTree/MergePlainMergeTreeTask.h @@ -5,6 +5,8 @@ #include #include #include +#include + namespace DB { @@ -34,7 +36,18 @@ public: priority += item->getBytesOnDisk(); } + bool onResume() override + { + return observer.doResume(); + } + bool executeStep() override; + + bool onSuspend() override + { + return observer.doSuspend(); + } + void onCompleted() override; StorageID getStorageID() override; UInt64 getPriority() override { return priority; } @@ -82,6 +95,8 @@ private: MergeTreeTransactionHolder txn_holder; MergeTreeTransactionPtr txn; + + TaskObserverMetrics observer; }; diff --git a/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp b/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp index 5bc3fda88bb..028da3a3b5f 100644 --- a/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp @@ -130,6 +130,15 @@ void MergeTreeBackgroundExecutor::routine(TaskRuntimeDataPtr item) bool need_execute_again = false; + try + { + item->task->onResume(); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } + try { ALLOW_ALLOCATIONS_IN_SCOPE; @@ -153,6 +162,15 @@ void MergeTreeBackgroundExecutor::routine(TaskRuntimeDataPtr item) }); } + try + { + item->task->onSuspend(); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } + if (need_execute_again) { std::lock_guard guard(mutex); diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index c3c4cd3082d..31c797556f6 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -7280,7 +7280,8 @@ void MergeTreeData::writePartLog( const String & new_part_name, const DataPartPtr & result_part, const DataPartsVector & source_parts, - const MergeListEntry * merge_entry) + const MergeListEntry * merge_entry, + std::shared_ptr profile_counters) try { auto table_id = getStorageID(); @@ -7342,6 +7343,11 @@ try part_log_elem.peak_memory_usage = (*merge_entry)->memory_tracker.getPeak(); } + if (profile_counters) + { + part_log_elem.profile_counters = profile_counters; + } + part_log->add(part_log_elem); } catch (...) diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 34bc3d24d66..ffedb0ebfdc 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -1298,7 +1298,8 @@ protected: const String & new_part_name, const DataPartPtr & result_part, const DataPartsVector & source_parts, - const MergeListEntry * merge_entry); + const MergeListEntry * merge_entry, + std::shared_ptr profile_counters = nullptr); /// If part is assigned to merge or mutation (possibly replicated) /// Should be overridden by children, because they can have different diff --git a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp index b83c058f7fd..322d8e78585 100644 --- a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp @@ -184,9 +184,12 @@ ReplicatedMergeMutateTaskBase::PrepareResult MutateFromLogEntryTask::prepare() return {true, true, [this] (const ExecutionStatus & execution_status) { + auto & thread_status = CurrentThread::get(); + thread_status.finalizePerformanceCounters(); + auto profile_counters = std::make_shared(thread_status.performance_counters.getPartiallyAtomicSnapshot()); storage.writePartLog( PartLogElement::MUTATE_PART, execution_status, stopwatch_ptr->elapsed(), - entry.new_part_name, new_part, future_mutated_part->parts, merge_mutate_entry.get()); + entry.new_part_name, new_part, future_mutated_part->parts, merge_mutate_entry.get(), profile_counters); }}; } diff --git a/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp b/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp index 76ba921b705..d611773a5d5 100644 --- a/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp +++ b/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp @@ -38,6 +38,9 @@ void MutatePlainMergeTreeTask::prepare() write_part_log = [this] (const ExecutionStatus & execution_status) { + auto & thread_status = CurrentThread::get(); + thread_status.finalizePerformanceCounters(); + auto profile_counters = std::make_shared(thread_status.performance_counters.getPartiallyAtomicSnapshot()); mutate_task.reset(); storage.writePartLog( PartLogElement::MUTATE_PART, @@ -46,7 +49,8 @@ void MutatePlainMergeTreeTask::prepare() future_part->name, new_part, future_part->parts, - merge_list_entry.get()); + merge_list_entry.get(), + profile_counters); }; fake_query_context = Context::createCopy(storage.getContext()); diff --git a/src/Storages/MergeTree/MutatePlainMergeTreeTask.h b/src/Storages/MergeTree/MutatePlainMergeTreeTask.h index e2b019c08ce..577529422de 100644 --- a/src/Storages/MergeTree/MutatePlainMergeTreeTask.h +++ b/src/Storages/MergeTree/MutatePlainMergeTreeTask.h @@ -8,6 +8,7 @@ #include #include #include +#include namespace DB { @@ -38,7 +39,18 @@ public: priority += part->getBytesOnDisk(); } + bool onSuspend() override + { + return observer.doSuspend(); + } + bool executeStep() override; + + bool onResume() override + { + return observer.doResume(); + } + void onCompleted() override; StorageID getStorageID() override; UInt64 getPriority() override { return priority; } @@ -76,6 +88,8 @@ private: ContextMutablePtr fake_query_context; MutateTaskPtr mutate_task; + + TaskObserverMetrics observer; }; diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index 4a7224b0722..5416e731d84 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -812,6 +813,11 @@ public: StorageID getStorageID() override { throw Exception(ErrorCodes::LOGICAL_ERROR, "Not implemented"); } UInt64 getPriority() override { throw Exception(ErrorCodes::LOGICAL_ERROR, "Not implemented"); } + bool onSuspend() override + { + return observer.doSuspend(); + } + bool executeStep() override { auto & current_level_parts = level_parts[current_level]; @@ -898,6 +904,12 @@ public: /// Need execute again return true; } + + bool onResume() override + { + return observer.doResume(); + } + private: String name; MergeTreeData::MutableDataPartsVector parts; @@ -913,6 +925,8 @@ private: /// TODO(nikitamikhaylov): make this constant a setting static constexpr size_t max_parts_to_merge_in_one_level = 10; + + TaskObserverMetrics observer; }; @@ -1134,6 +1148,12 @@ public: StorageID getStorageID() override { throw Exception(ErrorCodes::LOGICAL_ERROR, "Not implemented"); } UInt64 getPriority() override { throw Exception(ErrorCodes::LOGICAL_ERROR, "Not implemented"); } + + bool onSuspend() override + { + return observer.doSuspend(); + } + bool executeStep() override { switch (state) @@ -1168,6 +1188,11 @@ public: return false; } + bool onResume() override + { + return observer.doResume(); + } + private: void prepare() @@ -1251,6 +1276,8 @@ private: MutationContextPtr ctx; std::unique_ptr part_merger_writer_task; + + TaskObserverMetrics observer; }; class MutateSomePartColumnsTask : public IExecutableTask @@ -1262,6 +1289,11 @@ public: StorageID getStorageID() override { throw Exception(ErrorCodes::LOGICAL_ERROR, "Not implemented"); } UInt64 getPriority() override { throw Exception(ErrorCodes::LOGICAL_ERROR, "Not implemented"); } + bool onSuspend() override + { + return observer.doSuspend(); + } + bool executeStep() override { switch (state) @@ -1295,6 +1327,11 @@ public: return false; } + bool onResume() override + { + return observer.doResume(); + } + private: void prepare() @@ -1455,6 +1492,7 @@ private: MergedColumnOnlyOutputStreamPtr out; std::unique_ptr part_merger_writer_task{nullptr}; + TaskObserverMetrics observer; }; diff --git a/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.h b/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.h index d8495d35d90..6080d488ca4 100644 --- a/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.h +++ b/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.h @@ -4,6 +4,7 @@ #include #include +#include namespace DB { @@ -34,7 +35,15 @@ public: ~ReplicatedMergeMutateTaskBase() override = default; void onCompleted() override; StorageID getStorageID() override; + bool onSuspend() override + { + return observer.doSuspend(); + } bool executeStep() override; + bool onResume() override + { + return observer.doResume(); + } protected: using PartLogWriter = std::function; @@ -83,6 +92,7 @@ private: PartLogWriter part_log_writer{}; State state{State::NEED_PREPARE}; IExecutableTask::TaskResultCallback task_result_callback; + TaskObserverMetrics observer; }; } diff --git a/src/Storages/MergeTree/TaskObserverMetrics.h b/src/Storages/MergeTree/TaskObserverMetrics.h new file mode 100644 index 00000000000..c6e49435c7d --- /dev/null +++ b/src/Storages/MergeTree/TaskObserverMetrics.h @@ -0,0 +1,37 @@ +#pragma once + +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +class TaskObserverMetrics : public boost::noncopyable +{ +public: + TaskObserverMetrics() : thread_group(std::make_shared()) {} + ~TaskObserverMetrics() {} + + bool doResume() + { + CurrentThread::attachTo(thread_group); + return true; + } + + bool doSuspend() + { + CurrentThread::detachQueryIfNotDetached(); + return true; + } + + +private: + ThreadGroupStatusPtr thread_group; +}; + + +} diff --git a/src/Storages/MergeTree/tests/gtest_executor.cpp b/src/Storages/MergeTree/tests/gtest_executor.cpp index b89692869fd..1ae2123ce5a 100644 --- a/src/Storages/MergeTree/tests/gtest_executor.cpp +++ b/src/Storages/MergeTree/tests/gtest_executor.cpp @@ -24,6 +24,11 @@ public: { } + bool onSuspend() override + { + suspend_calls++ + } + bool executeStep() override { auto sleep_time = distribution(generator); @@ -36,6 +41,11 @@ public: return false; } + bool onSuspend() override + { + resume_calls++ + } + StorageID getStorageID() override { return {"test", name}; @@ -55,7 +65,9 @@ private: std::uniform_int_distribution<> distribution; String name; + size_t suspend_calls; std::function on_completed; + size_t resume_calls; }; @@ -93,6 +105,9 @@ TEST(Executor, RemoveTasks) thread.join(); ASSERT_EQ(CurrentMetrics::values[CurrentMetrics::BackgroundMergesAndMutationsPoolTask], 0); + /// TODO: move to a test by itself + ASSERT_EQ(batch*tasks_kinds, suspend_calls); + ASSERT_EQ(batch*tasks_kinds, resume_calls); executor->wait(); } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 0fb3e25ee85..3672bd44bb2 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -1602,8 +1602,11 @@ bool StorageReplicatedMergeTree::executeLogEntry(LogEntry & entry) renameTempPartAndReplace(part, transaction); checkPartChecksumsAndCommit(transaction, part); + auto & thread_status = CurrentThread::get(); + thread_status.finalizePerformanceCounters(); + auto profile_counters = std::make_shared(thread_status.performance_counters.getPartiallyAtomicSnapshot()); writePartLog(PartLogElement::Type::NEW_PART, {}, 0 /** log entry is fake so we don't measure the time */, - part->name, part, {} /** log entry is fake so there are no initial parts */, nullptr); + part->name, part, {} /** log entry is fake so there are no initial parts */, nullptr, profile_counters); return true; } @@ -4007,9 +4010,12 @@ bool StorageReplicatedMergeTree::fetchPart( auto write_part_log = [&] (const ExecutionStatus & execution_status) { + auto & thread_status = CurrentThread::get(); + thread_status.finalizePerformanceCounters(); + auto profile_counters = std::make_shared(thread_status.performance_counters.getPartiallyAtomicSnapshot()); writePartLog( PartLogElement::DOWNLOAD_PART, execution_status, stopwatch.elapsed(), - part_name, part, replaced_parts, nullptr); + part_name, part, replaced_parts, nullptr, profile_counters); }; DataPartPtr part_to_clone; @@ -4243,9 +4249,12 @@ MutableDataPartStoragePtr StorageReplicatedMergeTree::fetchExistsPart( auto write_part_log = [&] (const ExecutionStatus & execution_status) { + auto & thread_status = CurrentThread::get(); + thread_status.finalizePerformanceCounters(); + auto profile_counters = std::make_shared(thread_status.performance_counters.getPartiallyAtomicSnapshot()); writePartLog( PartLogElement::DOWNLOAD_PART, execution_status, stopwatch.elapsed(), - part_name, part, replaced_parts, nullptr); + part_name, part, replaced_parts, nullptr, profile_counters); }; std::function get_part; From 96e4411694fb1a693414018ba76dab667e35508d Mon Sep 17 00:00:00 2001 From: Nicolae Vartolomei Date: Wed, 6 Apr 2022 09:08:10 +0000 Subject: [PATCH 193/717] Fix style --- src/Storages/MergeTree/TaskObserverMetrics.h | 32 ++++++++------------ 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/src/Storages/MergeTree/TaskObserverMetrics.h b/src/Storages/MergeTree/TaskObserverMetrics.h index c6e49435c7d..b518682f0c9 100644 --- a/src/Storages/MergeTree/TaskObserverMetrics.h +++ b/src/Storages/MergeTree/TaskObserverMetrics.h @@ -5,33 +5,27 @@ namespace DB { -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; -} - class TaskObserverMetrics : public boost::noncopyable { public: - TaskObserverMetrics() : thread_group(std::make_shared()) {} - ~TaskObserverMetrics() {} + TaskObserverMetrics() : thread_group(std::make_shared()) { } + ~TaskObserverMetrics() { } - bool doResume() - { - CurrentThread::attachTo(thread_group); - return true; - } + bool doResume() + { + CurrentThread::attachTo(thread_group); + return true; + } - bool doSuspend() - { - CurrentThread::detachQueryIfNotDetached(); - return true; - } + bool doSuspend() + { + CurrentThread::detachQueryIfNotDetached(); + return true; + } private: - ThreadGroupStatusPtr thread_group; + ThreadGroupStatusPtr thread_group; }; - } From f554ff9d99893af48a693b08ce4b56eefeae5e7c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 4 Aug 2022 06:07:32 +0200 Subject: [PATCH 194/717] Remove trash --- src/Common/CurrentMetrics.cpp | 1 - src/Common/CurrentThread.h | 16 +++++- src/Interpreters/PartLog.cpp | 2 +- src/Storages/MergeTree/IExecutableTask.h | 12 ----- .../MergeTree/MergeFromLogEntryTask.cpp | 2 +- .../MergeTree/MergePlainMergeTreeTask.cpp | 3 ++ .../MergeTree/MergePlainMergeTreeTask.h | 17 +----- .../MergeTree/MergeTreeBackgroundExecutor.cpp | 18 ------- .../MergeTree/MergeTreeBackgroundExecutor.h | 1 + src/Storages/MergeTree/MergeTreeData.cpp | 3 +- src/Storages/MergeTree/MergeTreeData.h | 2 +- .../MergeTree/MutateFromLogEntryTask.cpp | 2 +- .../MergeTree/MutatePlainMergeTreeTask.cpp | 5 +- .../MergeTree/MutatePlainMergeTreeTask.h | 15 +----- src/Storages/MergeTree/MutateTask.cpp | 54 +++++-------------- .../ReplicatedMergeMutateTaskBase.cpp | 4 +- .../MergeTree/ReplicatedMergeMutateTaskBase.h | 24 ++++----- src/Storages/MergeTree/TaskObserverMetrics.h | 31 ----------- .../MergeTree/tests/gtest_executor.cpp | 18 +------ src/Storages/StorageReplicatedMergeTree.cpp | 6 +-- 20 files changed, 63 insertions(+), 173 deletions(-) delete mode 100644 src/Storages/MergeTree/TaskObserverMetrics.h diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index 62d56dead62..0395f2470af 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -13,7 +13,6 @@ M(BackgroundMergesAndMutationsPoolTask, "Number of active merges and mutations in an associated background pool") \ M(BackgroundFetchesPoolTask, "Number of active fetches in an associated background pool") \ M(BackgroundCommonPoolTask, "Number of active tasks in an associated background pool") \ - M(BackgroundMaintPoolTask, "Number of active tasks in BackgroundProcessingPool (maint)") \ M(BackgroundMovePoolTask, "Number of active tasks in BackgroundProcessingPool for moves") \ M(BackgroundSchedulePoolTask, "Number of active tasks in BackgroundSchedulePool. This pool is used for periodic ReplicatedMergeTree tasks, like cleaning old data parts, altering data parts, replica re-initialization, etc.") \ M(BackgroundBufferFlushSchedulePoolTask, "Number of active tasks in BackgroundBufferFlushSchedulePool. This pool is used for periodic Buffer flushes") \ diff --git a/src/Common/CurrentThread.h b/src/Common/CurrentThread.h index cbe60365798..9548a927cfd 100644 --- a/src/Common/CurrentThread.h +++ b/src/Common/CurrentThread.h @@ -92,7 +92,7 @@ public: static void detachQueryIfNotDetached(); /// Initializes query with current thread as master thread in constructor, and detaches it in destructor - struct QueryScope + struct QueryScope : private boost::noncopyable { explicit QueryScope(ContextMutablePtr query_context); explicit QueryScope(ContextPtr query_context); @@ -102,6 +102,20 @@ public: bool log_peak_memory_usage_in_destructor = true; }; + class ScopedAttach : private boost::noncopyable + { + public: + explicit ScopedAttach(const ThreadGroupStatusPtr & thread_group) + { + CurrentThread::attachTo(thread_group); + } + + ~ScopedAttach() + { + CurrentThread::detachQuery(); + } + }; + private: static void defaultThreadDeleter(); diff --git a/src/Interpreters/PartLog.cpp b/src/Interpreters/PartLog.cpp index b39f7423bbe..b422f4f7f03 100644 --- a/src/Interpreters/PartLog.cpp +++ b/src/Interpreters/PartLog.cpp @@ -193,7 +193,7 @@ void PartLogElement::appendToBlock(MutableColumns & columns) const bool PartLog::addNewPart( ContextPtr current_context, const MutableDataPartPtr & part, UInt64 elapsed_ns, const ExecutionStatus & execution_status, std::shared_ptr profile_counters_) { - return addNewParts(current_context, {part}, elapsed_ns, execution_status, profile_counters_); + return addNewParts(current_context, {part}, elapsed_ns, execution_status, std::move(profile_counters_)); } diff --git a/src/Storages/MergeTree/IExecutableTask.h b/src/Storages/MergeTree/IExecutableTask.h index d878d57dec8..9617960c182 100644 --- a/src/Storages/MergeTree/IExecutableTask.h +++ b/src/Storages/MergeTree/IExecutableTask.h @@ -29,9 +29,7 @@ class IExecutableTask { public: using TaskResultCallback = std::function; - virtual bool onResume() = 0; virtual bool executeStep() = 0; - virtual bool onSuspend() = 0; virtual void onCompleted() = 0; virtual StorageID getStorageID() = 0; virtual UInt64 getPriority() = 0; @@ -56,11 +54,6 @@ public: , job_result_callback(std::forward(job_result_callback_)) , id(id_) {} - bool onResume() override - { - return true; - } - bool executeStep() override { res = job_to_execute(); @@ -68,11 +61,6 @@ public: return false; } - bool onSuspend() override - { - return true; - } - void onCompleted() override { job_result_callback(!res); } StorageID getStorageID() override { return id; } UInt64 getPriority() override diff --git a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp index fab9b7c0f7e..abb871be372 100644 --- a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp @@ -294,7 +294,7 @@ ReplicatedMergeMutateTaskBase::PrepareResult MergeFromLogEntryTask::prepare() auto profile_counters = std::make_shared(thread_status.performance_counters.getPartiallyAtomicSnapshot()); storage.writePartLog( PartLogElement::MERGE_PARTS, execution_status, stopwatch.elapsed(), - entry.new_part_name, part, parts, merge_mutate_entry.get(), profile_counters); + entry.new_part_name, part, parts, merge_mutate_entry.get(), std::move(profile_counters)); }}; } diff --git a/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp b/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp index ccd132beb4e..95bcfba937a 100644 --- a/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp +++ b/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp @@ -27,6 +27,9 @@ void MergePlainMergeTreeTask::onCompleted() bool MergePlainMergeTreeTask::executeStep() { + /// Metrics will be saved in the thread_group. + CurrentThread::ScopedAttach scoped_attach(thread_group); + /// Make out memory tracker a parent of current thread memory tracker MemoryTrackerThreadSwitcherPtr switcher; if (merge_list_entry) diff --git a/src/Storages/MergeTree/MergePlainMergeTreeTask.h b/src/Storages/MergeTree/MergePlainMergeTreeTask.h index 472518c2723..4d0cecf1e55 100644 --- a/src/Storages/MergeTree/MergePlainMergeTreeTask.h +++ b/src/Storages/MergeTree/MergePlainMergeTreeTask.h @@ -5,7 +5,6 @@ #include #include #include -#include namespace DB @@ -36,18 +35,7 @@ public: priority += item->getBytesOnDisk(); } - bool onResume() override - { - return observer.doResume(); - } - bool executeStep() override; - - bool onSuspend() override - { - return observer.doSuspend(); - } - void onCompleted() override; StorageID getStorageID() override; UInt64 getPriority() override { return priority; } @@ -59,7 +47,6 @@ public: } private: - void prepare(); void finish(); @@ -95,8 +82,8 @@ private: MergeTreeTransactionHolder txn_holder; MergeTreeTransactionPtr txn; - - TaskObserverMetrics observer; + + ThreadGroupStatusPtr thread_group = std::make_shared(); }; diff --git a/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp b/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp index 028da3a3b5f..5bc3fda88bb 100644 --- a/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp @@ -130,15 +130,6 @@ void MergeTreeBackgroundExecutor::routine(TaskRuntimeDataPtr item) bool need_execute_again = false; - try - { - item->task->onResume(); - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__); - } - try { ALLOW_ALLOCATIONS_IN_SCOPE; @@ -162,15 +153,6 @@ void MergeTreeBackgroundExecutor::routine(TaskRuntimeDataPtr item) }); } - try - { - item->task->onSuspend(); - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__); - } - if (need_execute_again) { std::lock_guard guard(mutex); diff --git a/src/Storages/MergeTree/MergeTreeBackgroundExecutor.h b/src/Storages/MergeTree/MergeTreeBackgroundExecutor.h index 5c1178a1bc1..cf4d4b08c4d 100644 --- a/src/Storages/MergeTree/MergeTreeBackgroundExecutor.h +++ b/src/Storages/MergeTree/MergeTreeBackgroundExecutor.h @@ -17,6 +17,7 @@ #include #include + namespace DB { namespace ErrorCodes diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 31c797556f6..07385067ede 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -7345,7 +7345,7 @@ try if (profile_counters) { - part_log_elem.profile_counters = profile_counters; + part_log_elem.profile_counters = profile_counters; } part_log->add(part_log_elem); @@ -7493,6 +7493,7 @@ bool MergeTreeData::moveParts(const CurrentlyMovingPartsTaggerPtr & moving_tagge moving_part.part->name, cloned_part, {moving_part.part}, + nullptr, nullptr); }; diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index ffedb0ebfdc..10ba0045826 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -1299,7 +1299,7 @@ protected: const DataPartPtr & result_part, const DataPartsVector & source_parts, const MergeListEntry * merge_entry, - std::shared_ptr profile_counters = nullptr); + std::shared_ptr profile_counters); /// If part is assigned to merge or mutation (possibly replicated) /// Should be overridden by children, because they can have different diff --git a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp index 322d8e78585..2e546574e42 100644 --- a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp @@ -189,7 +189,7 @@ ReplicatedMergeMutateTaskBase::PrepareResult MutateFromLogEntryTask::prepare() auto profile_counters = std::make_shared(thread_status.performance_counters.getPartiallyAtomicSnapshot()); storage.writePartLog( PartLogElement::MUTATE_PART, execution_status, stopwatch_ptr->elapsed(), - entry.new_part_name, new_part, future_mutated_part->parts, merge_mutate_entry.get(), profile_counters); + entry.new_part_name, new_part, future_mutated_part->parts, merge_mutate_entry.get(), std::move(profile_counters)); }}; } diff --git a/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp b/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp index d611773a5d5..4a29291b20d 100644 --- a/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp +++ b/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp @@ -62,8 +62,12 @@ void MutatePlainMergeTreeTask::prepare() time(nullptr), fake_query_context, merge_mutate_entry->txn, merge_mutate_entry->tagger->reserved_space, table_lock_holder); } + bool MutatePlainMergeTreeTask::executeStep() { + /// Metrics will be saved in the thread_group. + CurrentThread::ScopedAttach scoped_attach(thread_group); + /// Make out memory tracker a parent of current thread memory tracker MemoryTrackerThreadSwitcherPtr switcher; if (merge_list_entry) @@ -127,5 +131,4 @@ bool MutatePlainMergeTreeTask::executeStep() return false; } - } diff --git a/src/Storages/MergeTree/MutatePlainMergeTreeTask.h b/src/Storages/MergeTree/MutatePlainMergeTreeTask.h index 577529422de..6356595a16b 100644 --- a/src/Storages/MergeTree/MutatePlainMergeTreeTask.h +++ b/src/Storages/MergeTree/MutatePlainMergeTreeTask.h @@ -8,7 +8,7 @@ #include #include #include -#include + namespace DB { @@ -39,18 +39,7 @@ public: priority += part->getBytesOnDisk(); } - bool onSuspend() override - { - return observer.doSuspend(); - } - bool executeStep() override; - - bool onResume() override - { - return observer.doResume(); - } - void onCompleted() override; StorageID getStorageID() override; UInt64 getPriority() override { return priority; } @@ -89,7 +78,7 @@ private: ContextMutablePtr fake_query_context; MutateTaskPtr mutate_task; - TaskObserverMetrics observer; + ThreadGroupStatusPtr thread_group = std::make_shared(); }; diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index 5416e731d84..295a88d5f76 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -17,7 +17,6 @@ #include #include #include -#include #include #include #include @@ -813,13 +812,11 @@ public: StorageID getStorageID() override { throw Exception(ErrorCodes::LOGICAL_ERROR, "Not implemented"); } UInt64 getPriority() override { throw Exception(ErrorCodes::LOGICAL_ERROR, "Not implemented"); } - bool onSuspend() override - { - return observer.doSuspend(); - } - bool executeStep() override { + /// Metrics will be saved in the thread_group. + CurrentThread::ScopedAttach scoped_attach(thread_group); + auto & current_level_parts = level_parts[current_level]; auto & next_level_parts = level_parts[next_level]; @@ -905,11 +902,6 @@ public: return true; } - bool onResume() override - { - return observer.doResume(); - } - private: String name; MergeTreeData::MutableDataPartsVector parts; @@ -926,7 +918,7 @@ private: /// TODO(nikitamikhaylov): make this constant a setting static constexpr size_t max_parts_to_merge_in_one_level = 10; - TaskObserverMetrics observer; + ThreadGroupStatusPtr thread_group = std::make_shared(); }; @@ -937,9 +929,7 @@ private: // In short it executed a mutation for the part an original part and for its every projection -/** - * - * An overview of how the process of mutation works for projections: +/** An overview of how the process of mutation works for projections: * * The mutation for original parts is executed block by block, * but additionally we execute a SELECT query for each projection over a current block. @@ -1148,14 +1138,11 @@ public: StorageID getStorageID() override { throw Exception(ErrorCodes::LOGICAL_ERROR, "Not implemented"); } UInt64 getPriority() override { throw Exception(ErrorCodes::LOGICAL_ERROR, "Not implemented"); } - - bool onSuspend() override - { - return observer.doSuspend(); - } - bool executeStep() override { + /// Metrics will be saved in the thread_group. + CurrentThread::ScopedAttach scoped_attach(thread_group); + switch (state) { case State::NEED_PREPARE: @@ -1188,13 +1175,7 @@ public: return false; } - bool onResume() override - { - return observer.doResume(); - } - private: - void prepare() { if (ctx->new_data_part->isStoredOnDisk()) @@ -1277,9 +1258,10 @@ private: std::unique_ptr part_merger_writer_task; - TaskObserverMetrics observer; + ThreadGroupStatusPtr thread_group = std::make_shared(); }; + class MutateSomePartColumnsTask : public IExecutableTask { public: @@ -1289,13 +1271,11 @@ public: StorageID getStorageID() override { throw Exception(ErrorCodes::LOGICAL_ERROR, "Not implemented"); } UInt64 getPriority() override { throw Exception(ErrorCodes::LOGICAL_ERROR, "Not implemented"); } - bool onSuspend() override - { - return observer.doSuspend(); - } - bool executeStep() override { + /// Metrics will be saved in the thread_group. + CurrentThread::ScopedAttach scoped_attach(thread_group); + switch (state) { case State::NEED_PREPARE: @@ -1327,13 +1307,7 @@ public: return false; } - bool onResume() override - { - return observer.doResume(); - } - private: - void prepare() { if (ctx->execute_ttl_type != ExecuteTTLType::NONE) @@ -1492,7 +1466,7 @@ private: MergedColumnOnlyOutputStreamPtr out; std::unique_ptr part_merger_writer_task{nullptr}; - TaskObserverMetrics observer; + ThreadGroupStatusPtr thread_group = std::make_shared(); }; diff --git a/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp b/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp index a22aab8d6ce..2ef36654d95 100644 --- a/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp @@ -29,6 +29,9 @@ void ReplicatedMergeMutateTaskBase::onCompleted() bool ReplicatedMergeMutateTaskBase::executeStep() { + /// Metrics will be saved in the thread_group. + CurrentThread::ScopedAttach scoped_attach(thread_group); + std::exception_ptr saved_exception; bool retryable_error = false; @@ -83,7 +86,6 @@ bool ReplicatedMergeMutateTaskBase::executeStep() saved_exception = std::current_exception(); } - if (!retryable_error && saved_exception) { std::lock_guard lock(storage.queue.state_mutex); diff --git a/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.h b/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.h index 6080d488ca4..e08074cd4ff 100644 --- a/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.h +++ b/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.h @@ -4,16 +4,16 @@ #include #include -#include + namespace DB { class StorageReplicatedMergeTree; -/** - * This is used as a base of MergeFromLogEntryTask and MutateFromLogEntryTaskBase - */ + +/** This is used as a base of MergeFromLogEntryTask and MutateFromLogEntryTaskBase + */ class ReplicatedMergeMutateTaskBase : public IExecutableTask { public: @@ -33,17 +33,12 @@ public: } ~ReplicatedMergeMutateTaskBase() override = default; + void onCompleted() override; + StorageID getStorageID() override; - bool onSuspend() override - { - return observer.doSuspend(); - } + bool executeStep() override; - bool onResume() override - { - return observer.doResume(); - } protected: using PartLogWriter = std::function; @@ -70,7 +65,6 @@ protected: StorageReplicatedMergeTree & storage; private: - enum class CheckExistingPartResult { PART_EXISTS, @@ -78,7 +72,7 @@ private: }; CheckExistingPartResult checkExistingPart(); - bool executeImpl() ; + bool executeImpl(); enum class State { @@ -92,7 +86,7 @@ private: PartLogWriter part_log_writer{}; State state{State::NEED_PREPARE}; IExecutableTask::TaskResultCallback task_result_callback; - TaskObserverMetrics observer; + ThreadGroupStatusPtr thread_group = std::make_shared(); }; } diff --git a/src/Storages/MergeTree/TaskObserverMetrics.h b/src/Storages/MergeTree/TaskObserverMetrics.h deleted file mode 100644 index b518682f0c9..00000000000 --- a/src/Storages/MergeTree/TaskObserverMetrics.h +++ /dev/null @@ -1,31 +0,0 @@ -#pragma once - -#include - -namespace DB -{ - -class TaskObserverMetrics : public boost::noncopyable -{ -public: - TaskObserverMetrics() : thread_group(std::make_shared()) { } - ~TaskObserverMetrics() { } - - bool doResume() - { - CurrentThread::attachTo(thread_group); - return true; - } - - bool doSuspend() - { - CurrentThread::detachQueryIfNotDetached(); - return true; - } - - -private: - ThreadGroupStatusPtr thread_group; -}; - -} diff --git a/src/Storages/MergeTree/tests/gtest_executor.cpp b/src/Storages/MergeTree/tests/gtest_executor.cpp index 1ae2123ce5a..60149220643 100644 --- a/src/Storages/MergeTree/tests/gtest_executor.cpp +++ b/src/Storages/MergeTree/tests/gtest_executor.cpp @@ -24,11 +24,6 @@ public: { } - bool onSuspend() override - { - suspend_calls++ - } - bool executeStep() override { auto sleep_time = distribution(generator); @@ -41,11 +36,6 @@ public: return false; } - bool onSuspend() override - { - resume_calls++ - } - StorageID getStorageID() override { return {"test", name}; @@ -65,9 +55,7 @@ private: std::uniform_int_distribution<> distribution; String name; - size_t suspend_calls; std::function on_completed; - size_t resume_calls; }; @@ -87,8 +75,7 @@ TEST(Executor, RemoveTasks) for (size_t i = 0; i < batch; ++i) for (size_t j = 0; j < tasks_kinds; ++j) ASSERT_TRUE( - executor->trySchedule(std::make_shared(std::to_string(j))) - ); + executor->trySchedule(std::make_shared(std::to_string(j)))); std::vector threads(batch); @@ -105,9 +92,6 @@ TEST(Executor, RemoveTasks) thread.join(); ASSERT_EQ(CurrentMetrics::values[CurrentMetrics::BackgroundMergesAndMutationsPoolTask], 0); - /// TODO: move to a test by itself - ASSERT_EQ(batch*tasks_kinds, suspend_calls); - ASSERT_EQ(batch*tasks_kinds, resume_calls); executor->wait(); } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 3672bd44bb2..e9958a68406 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -1606,7 +1606,7 @@ bool StorageReplicatedMergeTree::executeLogEntry(LogEntry & entry) thread_status.finalizePerformanceCounters(); auto profile_counters = std::make_shared(thread_status.performance_counters.getPartiallyAtomicSnapshot()); writePartLog(PartLogElement::Type::NEW_PART, {}, 0 /** log entry is fake so we don't measure the time */, - part->name, part, {} /** log entry is fake so there are no initial parts */, nullptr, profile_counters); + part->name, part, {} /** log entry is fake so there are no initial parts */, nullptr, std::move(profile_counters)); return true; } @@ -4015,7 +4015,7 @@ bool StorageReplicatedMergeTree::fetchPart( auto profile_counters = std::make_shared(thread_status.performance_counters.getPartiallyAtomicSnapshot()); writePartLog( PartLogElement::DOWNLOAD_PART, execution_status, stopwatch.elapsed(), - part_name, part, replaced_parts, nullptr, profile_counters); + part_name, part, replaced_parts, nullptr, std::move(profile_counters)); }; DataPartPtr part_to_clone; @@ -4254,7 +4254,7 @@ MutableDataPartStoragePtr StorageReplicatedMergeTree::fetchExistsPart( auto profile_counters = std::make_shared(thread_status.performance_counters.getPartiallyAtomicSnapshot()); writePartLog( PartLogElement::DOWNLOAD_PART, execution_status, stopwatch.elapsed(), - part_name, part, replaced_parts, nullptr, profile_counters); + part_name, part, replaced_parts, nullptr, std::move(profile_counters)); }; std::function get_part; From f15623141ede304c5de0111c1f7075caef1c74d9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 6 Aug 2022 20:46:42 +0200 Subject: [PATCH 195/717] Fix error --- src/Common/CurrentThread.h | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/Common/CurrentThread.h b/src/Common/CurrentThread.h index 9548a927cfd..24c9dac844a 100644 --- a/src/Common/CurrentThread.h +++ b/src/Common/CurrentThread.h @@ -104,15 +104,24 @@ public: class ScopedAttach : private boost::noncopyable { + private: + bool attached = false; public: explicit ScopedAttach(const ThreadGroupStatusPtr & thread_group) { - CurrentThread::attachTo(thread_group); + if (!CurrentThread::getGroup()) + { + CurrentThread::attachToIfDetached(thread_group); + attached = true; + } } ~ScopedAttach() { - CurrentThread::detachQuery(); + if (attached) + { + CurrentThread::detachQuery(); + } } }; From 5f9f72f9376a6c691fc8de75cfdae1e594f8ad76 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 6 Aug 2022 20:53:17 +0200 Subject: [PATCH 196/717] Add a smoke test --- .../02378_part_log_profile_events.reference | 1 + .../0_stateless/02378_part_log_profile_events.sql | 13 +++++++++++++ 2 files changed, 14 insertions(+) create mode 100644 tests/queries/0_stateless/02378_part_log_profile_events.reference create mode 100644 tests/queries/0_stateless/02378_part_log_profile_events.sql diff --git a/tests/queries/0_stateless/02378_part_log_profile_events.reference b/tests/queries/0_stateless/02378_part_log_profile_events.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/02378_part_log_profile_events.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/02378_part_log_profile_events.sql b/tests/queries/0_stateless/02378_part_log_profile_events.sql new file mode 100644 index 00000000000..9ffd6aa4135 --- /dev/null +++ b/tests/queries/0_stateless/02378_part_log_profile_events.sql @@ -0,0 +1,13 @@ +DROP TABLE IF EXISTS test; +CREATE TABLE test (x UInt64) ENGINE = MergeTree ORDER BY x; + +SET max_block_size = 1; +INSERT INTO test SELECT * FROM system.numbers LIMIT 1000; +OPTIMIZE TABLE test FINAL; + +SYSTEM FLUSH LOGS; + +-- ProfileEvents field exist and contains something plausible: +SELECT count() > 0 FROM system.part_log WHERE ProfileEvents['MergedRows'] = 1000 AND table = 'test' AND database = currentDatabase() AND event_time >= now() - 600; + +DROP TABLE test; From a228f7f419827915b99aca81789aa3f2d0d4c560 Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 23 Jan 2023 12:45:28 +0000 Subject: [PATCH 197/717] [wip] ProfileCounters for each part --- src/Common/CurrentThread.cpp | 2 + src/Common/CurrentThread.h | 23 ---------- src/Common/ProfileEventsScope.cpp | 32 ++++++++++++++ src/Common/ProfileEventsScope.h | 35 +++++++++++++++ src/Common/ThreadStatus.cpp | 1 + src/Common/ThreadStatus.h | 4 ++ src/Interpreters/PartLog.cpp | 44 +++++++++++++------ src/Interpreters/PartLog.h | 36 +++++++++++++-- src/Interpreters/ProfileEventsExt.cpp | 16 +++++++ src/Interpreters/ProfileEventsExt.h | 2 + src/Interpreters/ThreadStatusExt.cpp | 6 +++ .../MergeTree/MergePlainMergeTreeTask.cpp | 7 +-- .../MergeTree/MergePlainMergeTreeTask.h | 2 +- src/Storages/MergeTree/MergeTreeData.cpp | 8 +++- src/Storages/MergeTree/MergeTreeSink.cpp | 25 ++++++++--- .../MergeTree/MutateFromLogEntryTask.cpp | 6 +-- .../MergeTree/MutatePlainMergeTreeTask.cpp | 11 +++-- .../MergeTree/MutatePlainMergeTreeTask.h | 2 +- src/Storages/MergeTree/MutateTask.cpp | 25 ++++++----- .../ReplicatedMergeMutateTaskBase.cpp | 5 ++- .../MergeTree/ReplicatedMergeMutateTaskBase.h | 12 +++-- src/Storages/StorageReplicatedMergeTree.cpp | 26 +++++------ .../02378_part_log_profile_events.reference | 2 + .../02378_part_log_profile_events.sql | 44 ++++++++++++++++--- 24 files changed, 277 insertions(+), 99 deletions(-) create mode 100644 src/Common/ProfileEventsScope.cpp create mode 100644 src/Common/ProfileEventsScope.h diff --git a/src/Common/CurrentThread.cpp b/src/Common/CurrentThread.cpp index e54b2c8abe4..14c9f4418e0 100644 --- a/src/Common/CurrentThread.cpp +++ b/src/Common/CurrentThread.cpp @@ -40,6 +40,8 @@ ThreadStatus & CurrentThread::get() ProfileEvents::Counters & CurrentThread::getProfileEvents() { + if (unlikely(subthread_profile_events)) + return *subthread_profile_events; return current_thread ? current_thread->performance_counters : ProfileEvents::global_counters; } diff --git a/src/Common/CurrentThread.h b/src/Common/CurrentThread.h index 24c9dac844a..c07b34acae3 100644 --- a/src/Common/CurrentThread.h +++ b/src/Common/CurrentThread.h @@ -102,29 +102,6 @@ public: bool log_peak_memory_usage_in_destructor = true; }; - class ScopedAttach : private boost::noncopyable - { - private: - bool attached = false; - public: - explicit ScopedAttach(const ThreadGroupStatusPtr & thread_group) - { - if (!CurrentThread::getGroup()) - { - CurrentThread::attachToIfDetached(thread_group); - attached = true; - } - } - - ~ScopedAttach() - { - if (attached) - { - CurrentThread::detachQuery(); - } - } - }; - private: static void defaultThreadDeleter(); diff --git a/src/Common/ProfileEventsScope.cpp b/src/Common/ProfileEventsScope.cpp new file mode 100644 index 00000000000..6dcb5f8fcf3 --- /dev/null +++ b/src/Common/ProfileEventsScope.cpp @@ -0,0 +1,32 @@ +#include + +namespace DB +{ +extern thread_local constinit ProfileEvents::Counters * subthread_profile_events; + + +ScopedProfileEvents::ScopedProfileEvents() + : performance_counters_holder(std::make_unique()) + , performance_counters_scope(performance_counters_holder.get()) +{ + CurrentThread::get().attachProfileCountersScope(performance_counters_scope); +} + +ScopedProfileEvents::ScopedProfileEvents(ProfileEvents::Counters * performance_counters_scope_) + : performance_counters_scope(performance_counters_scope_) +{ + CurrentThread::get().attachProfileCountersScope(performance_counters_scope); +} + +std::shared_ptr ScopedProfileEvents::getSnapshot() +{ + return std::make_shared(performance_counters_scope->getPartiallyAtomicSnapshot()); +} + +ScopedProfileEvents::~ScopedProfileEvents() +{ + subthread_profile_events = nullptr; +} + + +} diff --git a/src/Common/ProfileEventsScope.h b/src/Common/ProfileEventsScope.h new file mode 100644 index 00000000000..59ba4f616d9 --- /dev/null +++ b/src/Common/ProfileEventsScope.h @@ -0,0 +1,35 @@ +#pragma once + +#include +#include + +namespace DB +{ + + +/// Use specific performance counters for current thread in the current scope. +class ScopedProfileEvents : private boost::noncopyable +{ +public: + /// Counters are owned by this object. + ScopedProfileEvents(); + + /// Shared counters are stored outisde. + /// Useful when we calculate metrics entering into some scope several times. + explicit ScopedProfileEvents(ProfileEvents::Counters * performance_counters_scope_); + + std::shared_ptr getSnapshot(); + + ~ScopedProfileEvents(); + +private: + /// If set, then performance_counters_scope is owned by this object. + /// Otherwise, counters are passed to the constructor from outside. + std::unique_ptr performance_counters_holder; + + ProfileEvents::Counters * performance_counters_scope; +}; + + +} + diff --git a/src/Common/ThreadStatus.cpp b/src/Common/ThreadStatus.cpp index 9f9a78c4036..6949942f745 100644 --- a/src/Common/ThreadStatus.cpp +++ b/src/Common/ThreadStatus.cpp @@ -25,6 +25,7 @@ namespace ErrorCodes } thread_local ThreadStatus constinit * current_thread = nullptr; +thread_local ProfileEvents::Counters constinit * subthread_profile_events = nullptr; #if !defined(SANITIZER) namespace diff --git a/src/Common/ThreadStatus.h b/src/Common/ThreadStatus.h index 69c5732ddb6..d44cd57b812 100644 --- a/src/Common/ThreadStatus.h +++ b/src/Common/ThreadStatus.h @@ -107,6 +107,7 @@ using ThreadGroupStatusPtr = std::shared_ptr; * - https://github.com/ClickHouse/ClickHouse/pull/40078 */ extern thread_local constinit ThreadStatus * current_thread; +extern thread_local constinit ProfileEvents::Counters * subthread_profile_events; /** Encapsulates all per-thread info (ProfileEvents, MemoryTracker, query_id, query context, etc.). * The object must be created in thread function and destroyed in the same thread before the exit. @@ -139,6 +140,7 @@ public: Deleter deleter; protected: + /// Group of threads, to which this thread attached ThreadGroupStatusPtr thread_group; std::atomic thread_state{ThreadState::DetachedFromQuery}; @@ -244,6 +246,8 @@ public: /// Attaches slave thread to existing thread group void attachQuery(const ThreadGroupStatusPtr & thread_group_, bool check_detached = true); + void attachProfileCountersScope(ProfileEvents::Counters * performance_counters_scope); + InternalTextLogsQueuePtr getInternalTextLogsQueue() const { return thread_state == Died ? nullptr : logs_queue_ptr.lock(); diff --git a/src/Interpreters/PartLog.cpp b/src/Interpreters/PartLog.cpp index b422f4f7f03..b736fa43e0c 100644 --- a/src/Interpreters/PartLog.cpp +++ b/src/Interpreters/PartLog.cpp @@ -189,16 +189,8 @@ void PartLogElement::appendToBlock(MutableColumns & columns) const } } - -bool PartLog::addNewPart( - ContextPtr current_context, const MutableDataPartPtr & part, UInt64 elapsed_ns, const ExecutionStatus & execution_status, std::shared_ptr profile_counters_) -{ - return addNewParts(current_context, {part}, elapsed_ns, execution_status, std::move(profile_counters_)); -} - - bool PartLog::addNewParts( - ContextPtr current_context, const PartLog::MutableDataPartsVector & parts, UInt64 elapsed_ns, const ExecutionStatus & execution_status, std::shared_ptr profile_counters_) + ContextPtr current_context, const PartLog::PartLogEntries & parts, const ExecutionStatus & execution_status) { if (parts.empty()) return true; @@ -207,15 +199,17 @@ bool PartLog::addNewParts( try { - auto table_id = parts.front()->storage.getStorageID(); + auto table_id = parts.front().part->storage.getStorageID(); part_log = current_context->getPartLog(table_id.database_name); // assume parts belong to the same table if (!part_log) return false; auto query_id = CurrentThread::getQueryId(); - for (const auto & part : parts) + for (const auto & part_log_entry : parts) { + const auto & part = part_log_entry.part; + PartLogElement elem; if (!query_id.empty()) @@ -228,7 +222,7 @@ bool PartLog::addNewParts( const auto time_now = std::chrono::system_clock::now(); elem.event_time = timeInSeconds(time_now); elem.event_time_microseconds = timeInMicroseconds(time_now); - elem.duration_ms = elapsed_ns / 1000000; + elem.duration_ms = part_log_entry.elapsed_ns / 1000000; elem.database_name = table_id.database_name; elem.table_name = table_id.table_name; @@ -245,7 +239,7 @@ bool PartLog::addNewParts( elem.error = static_cast(execution_status.code); elem.exception = execution_status.message; - elem.profile_counters = profile_counters_; + elem.profile_counters = part_log_entry.profile_counters; part_log->add(elem); } @@ -259,4 +253,28 @@ bool PartLog::addNewParts( return true; } +bool PartLog::addNewPart(ContextPtr context, const PartLog::PartLogEntry & part, const ExecutionStatus & execution_status) +{ + return addNewParts(context, {part}, execution_status); +} + +bool PartLog::addNewParts(ContextPtr context, const PartLog::MutableDataPartsVector & parts, UInt64 elapsed_ns, + const ExecutionStatus & execution_status) +{ + PartLog::PartLogEntries part_log_entries; + part_log_entries.reserve(parts.size()); + + for (const auto & part : parts) + part_log_entries.emplace_back(part, elapsed_ns); + + return addNewParts(context, part_log_entries, execution_status); +} + +bool PartLog::addNewPart(ContextPtr context, const PartLog::MutableDataPartPtr & part, UInt64 elapsed_ns, const ExecutionStatus & execution_status) +{ + PartLog::PartLogEntries part_log_entries; + part_log_entries.emplace_back(part, elapsed_ns); + return addNewParts(context, part_log_entries, execution_status); +} + } diff --git a/src/Interpreters/PartLog.h b/src/Interpreters/PartLog.h index 75b2539bda9..57044cba844 100644 --- a/src/Interpreters/PartLog.h +++ b/src/Interpreters/PartLog.h @@ -109,14 +109,42 @@ class PartLog : public SystemLog using MutableDataPartPtr = std::shared_ptr; using MutableDataPartsVector = std::vector; + using ProfileCountersSnapshotPtr = std::shared_ptr; + public: + struct PartLogEntry + { + std::shared_ptr part; + ProfileCountersSnapshotPtr profile_counters; + UInt64 elapsed_ns; + + PartLogEntry(std::shared_ptr part_, UInt64 elapsed_ns_) + : part(std::move(part_)), elapsed_ns(elapsed_ns_) + { + } + + PartLogEntry(std::shared_ptr part_, UInt64 elapsed_ns_, ProfileCountersSnapshotPtr profile_counters_) + : part(std::move(part_)) + , profile_counters(std::move(profile_counters_)) + , elapsed_ns(elapsed_ns_) + { + } + }; + + using PartLogEntries = std::vector; + + /// Add a record about creation of new part. + static bool addNewPart(ContextPtr context, const PartLogEntry & part, + const ExecutionStatus & execution_status = {}); + + static bool addNewParts(ContextPtr context, const PartLogEntries & parts, + const ExecutionStatus & execution_status = {}); + /// Add a record about creation of new part. static bool addNewPart(ContextPtr context, const MutableDataPartPtr & part, UInt64 elapsed_ns, - const ExecutionStatus & execution_status = {}, - std::shared_ptr profile_counters_ = {}); + const ExecutionStatus & execution_status = {}); static bool addNewParts(ContextPtr context, const MutableDataPartsVector & parts, UInt64 elapsed_ns, - const ExecutionStatus & execution_status = {}, - std::shared_ptr profile_counters_ = {}); + const ExecutionStatus & execution_status = {}); }; } diff --git a/src/Interpreters/ProfileEventsExt.cpp b/src/Interpreters/ProfileEventsExt.cpp index 0f6b52b2611..b5a49c8f381 100644 --- a/src/Interpreters/ProfileEventsExt.cpp +++ b/src/Interpreters/ProfileEventsExt.cpp @@ -19,6 +19,22 @@ std::shared_ptr TypeEnum = std::make_shared(GAUGE)}, }); +String dumpToString(const Counters::Snapshot & counters, bool nonzero_only) +{ + std::vector ss; + for (Event event = 0; event < Counters::num_counters; ++event) + { + UInt64 value = counters[event]; + + if (nonzero_only && 0 == value) + continue; + + const char * desc = getName(event); + ss.push_back(fmt::format("{}: {}", desc, value)); + } + return fmt::format("[{}]", fmt::join(ss, ", ")); +} + /// Put implementation here to avoid extra linking dependencies for clickhouse_common_io void dumpToMapColumn(const Counters::Snapshot & counters, DB::IColumn * column, bool nonzero_only) { diff --git a/src/Interpreters/ProfileEventsExt.h b/src/Interpreters/ProfileEventsExt.h index 7d9fc512d15..be88ca02593 100644 --- a/src/Interpreters/ProfileEventsExt.h +++ b/src/Interpreters/ProfileEventsExt.h @@ -21,6 +21,8 @@ struct ProfileEventsSnapshot using ThreadIdToCountersSnapshot = std::unordered_map; +String dumpToString(const Counters::Snapshot & counters, bool nonzero_only = true); + /// Dumps profile events to columns Map(String, UInt64) void dumpToMapColumn(const Counters::Snapshot & counters, DB::IColumn * column, bool nonzero_only = true); diff --git a/src/Interpreters/ThreadStatusExt.cpp b/src/Interpreters/ThreadStatusExt.cpp index 816b03f3a0e..b4f82b2a215 100644 --- a/src/Interpreters/ThreadStatusExt.cpp +++ b/src/Interpreters/ThreadStatusExt.cpp @@ -161,6 +161,12 @@ void ThreadStatus::attachQuery(const ThreadGroupStatusPtr & thread_group_, bool setupState(thread_group_); } +void ThreadStatus::attachProfileCountersScope(ProfileEvents::Counters * performance_counters_scope) +{ + subthread_profile_events = performance_counters_scope; + performance_counters_scope->setParent(&performance_counters); +} + void ThreadStatus::initPerformanceCounters() { performance_counters_finalized = false; diff --git a/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp b/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp index 95bcfba937a..5346a932fba 100644 --- a/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp +++ b/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB { @@ -28,7 +29,7 @@ void MergePlainMergeTreeTask::onCompleted() bool MergePlainMergeTreeTask::executeStep() { /// Metrics will be saved in the thread_group. - CurrentThread::ScopedAttach scoped_attach(thread_group); + ScopedProfileEvents profile_events_scope(&profile_counters); /// Make out memory tracker a parent of current thread memory tracker MemoryTrackerThreadSwitcherPtr switcher; @@ -90,7 +91,7 @@ void MergePlainMergeTreeTask::prepare() { auto & thread_status = CurrentThread::get(); thread_status.finalizePerformanceCounters(); - auto profile_counters = std::make_shared(thread_status.performance_counters.getPartiallyAtomicSnapshot()); + auto profile_counters_snapshot = std::make_shared(profile_counters.getPartiallyAtomicSnapshot()); merge_task.reset(); storage.writePartLog( PartLogElement::MERGE_PARTS, @@ -100,7 +101,7 @@ void MergePlainMergeTreeTask::prepare() new_part, future_part->parts, merge_list_entry.get(), - profile_counters); + std::move(profile_counters_snapshot)); }; merge_task = storage.merger_mutator.mergePartsToTemporaryPart( diff --git a/src/Storages/MergeTree/MergePlainMergeTreeTask.h b/src/Storages/MergeTree/MergePlainMergeTreeTask.h index 4d0cecf1e55..eb659bf38ec 100644 --- a/src/Storages/MergeTree/MergePlainMergeTreeTask.h +++ b/src/Storages/MergeTree/MergePlainMergeTreeTask.h @@ -83,7 +83,7 @@ private: MergeTreeTransactionHolder txn_holder; MergeTreeTransactionPtr txn; - ThreadGroupStatusPtr thread_group = std::make_shared(); + ProfileEvents::Counters profile_counters; }; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 07385067ede..c5160a27d48 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -7347,6 +7348,10 @@ try { part_log_elem.profile_counters = profile_counters; } + else + { + LOG_WARNING(log, "Profile counters are not set"); + } part_log->add(part_log_elem); } @@ -7483,6 +7488,7 @@ bool MergeTreeData::moveParts(const CurrentlyMovingPartsTaggerPtr & moving_tagge { Stopwatch stopwatch; MutableDataPartPtr cloned_part; + ScopedProfileEvents profile_events_scope; auto write_part_log = [&](const ExecutionStatus & execution_status) { @@ -7494,7 +7500,7 @@ bool MergeTreeData::moveParts(const CurrentlyMovingPartsTaggerPtr & moving_tagge cloned_part, {moving_part.part}, nullptr, - nullptr); + profile_events_scope.getSnapshot()); }; // Register in global moves list (StorageSystemMoves) diff --git a/src/Storages/MergeTree/MergeTreeSink.cpp b/src/Storages/MergeTree/MergeTreeSink.cpp index 99f6b1855e4..be645cf27c0 100644 --- a/src/Storages/MergeTree/MergeTreeSink.cpp +++ b/src/Storages/MergeTree/MergeTreeSink.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace ProfileEvents { @@ -47,6 +48,7 @@ struct MergeTreeSink::DelayedChunk MergeTreeDataWriter::TemporaryPart temp_part; UInt64 elapsed_ns; String block_dedup_token; + ProfileEvents::Counters part_counters; }; std::vector partitions; @@ -70,12 +72,18 @@ void MergeTreeSink::consume(Chunk chunk) for (auto & current_block : part_blocks) { - Stopwatch watch; - String block_dedup_token; + ProfileEvents::Counters part_counters; - auto temp_part = storage.writer.writeTempPart(current_block, metadata_snapshot, context); + UInt64 elapsed_ns = 0; + MergeTreeDataWriter::TemporaryPart temp_part; - UInt64 elapsed_ns = watch.elapsed(); + { + ScopedProfileEvents scoped_attach(&part_counters); + + Stopwatch watch; + temp_part = storage.writer.writeTempPart(current_block, metadata_snapshot, context); + elapsed_ns = watch.elapsed(); + } /// If optimize_on_insert setting is true, current_block could become empty after merge /// and we didn't create part. @@ -85,6 +93,7 @@ void MergeTreeSink::consume(Chunk chunk) if (!support_parallel_write && temp_part.part->getDataPartStorage().supportParallelWrite()) support_parallel_write = true; + String block_dedup_token; if (storage.getDeduplicationLog()) { const String & dedup_token = settings.insert_deduplication_token; @@ -119,7 +128,8 @@ void MergeTreeSink::consume(Chunk chunk) { .temp_part = std::move(temp_part), .elapsed_ns = elapsed_ns, - .block_dedup_token = std::move(block_dedup_token) + .block_dedup_token = std::move(block_dedup_token), + .part_counters = std::move(part_counters), }); } @@ -135,6 +145,8 @@ void MergeTreeSink::finishDelayedChunk() for (auto & partition : delayed_chunk->partitions) { + ScopedProfileEvents scoped_attach(&partition.part_counters); + partition.temp_part.finalize(); auto & part = partition.temp_part.part; @@ -168,7 +180,8 @@ void MergeTreeSink::finishDelayedChunk() /// Part can be deduplicated, so increment counters and add to part log only if it's really added if (added) { - PartLog::addNewPart(storage.getContext(), part, partition.elapsed_ns); + auto counters_snapshot = std::make_shared(partition.part_counters.getPartiallyAtomicSnapshot()); + PartLog::addNewPart(storage.getContext(), PartLog::PartLogEntry(part, partition.elapsed_ns, counters_snapshot)); storage.incrementInsertedPartsProfileEvent(part->getType()); /// Initiate async merge - it will be done if it's good time for merge and if there are space in 'background_pool'. diff --git a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp index 2e546574e42..4428f6c2bce 100644 --- a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp @@ -184,12 +184,10 @@ ReplicatedMergeMutateTaskBase::PrepareResult MutateFromLogEntryTask::prepare() return {true, true, [this] (const ExecutionStatus & execution_status) { - auto & thread_status = CurrentThread::get(); - thread_status.finalizePerformanceCounters(); - auto profile_counters = std::make_shared(thread_status.performance_counters.getPartiallyAtomicSnapshot()); + auto profile_counters_snapshot = std::make_shared(profile_counters.getPartiallyAtomicSnapshot()); storage.writePartLog( PartLogElement::MUTATE_PART, execution_status, stopwatch_ptr->elapsed(), - entry.new_part_name, new_part, future_mutated_part->parts, merge_mutate_entry.get(), std::move(profile_counters)); + entry.new_part_name, new_part, future_mutated_part->parts, merge_mutate_entry.get(), std::move(profile_counters_snapshot)); }}; } diff --git a/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp b/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp index 4a29291b20d..944a2d1070b 100644 --- a/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp +++ b/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp @@ -2,6 +2,7 @@ #include #include +#include namespace DB { @@ -38,9 +39,7 @@ void MutatePlainMergeTreeTask::prepare() write_part_log = [this] (const ExecutionStatus & execution_status) { - auto & thread_status = CurrentThread::get(); - thread_status.finalizePerformanceCounters(); - auto profile_counters = std::make_shared(thread_status.performance_counters.getPartiallyAtomicSnapshot()); + auto profile_counters_snapshot = std::make_shared(profile_counters.getPartiallyAtomicSnapshot()); mutate_task.reset(); storage.writePartLog( PartLogElement::MUTATE_PART, @@ -50,7 +49,7 @@ void MutatePlainMergeTreeTask::prepare() new_part, future_part->parts, merge_list_entry.get(), - profile_counters); + std::move(profile_counters_snapshot)); }; fake_query_context = Context::createCopy(storage.getContext()); @@ -65,8 +64,8 @@ void MutatePlainMergeTreeTask::prepare() bool MutatePlainMergeTreeTask::executeStep() { - /// Metrics will be saved in the thread_group. - CurrentThread::ScopedAttach scoped_attach(thread_group); + /// Metrics will be saved in the local profile_counters. + ScopedProfileEvents profile_events_scope(&profile_counters); /// Make out memory tracker a parent of current thread memory tracker MemoryTrackerThreadSwitcherPtr switcher; diff --git a/src/Storages/MergeTree/MutatePlainMergeTreeTask.h b/src/Storages/MergeTree/MutatePlainMergeTreeTask.h index 6356595a16b..ae2ac039543 100644 --- a/src/Storages/MergeTree/MutatePlainMergeTreeTask.h +++ b/src/Storages/MergeTree/MutatePlainMergeTreeTask.h @@ -78,7 +78,7 @@ private: ContextMutablePtr fake_query_context; MutateTaskPtr mutate_task; - ThreadGroupStatusPtr thread_group = std::make_shared(); + ProfileEvents::Counters profile_counters; }; diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index 295a88d5f76..a5ea5c90b95 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -20,6 +20,7 @@ #include #include #include +#include namespace CurrentMetrics @@ -814,8 +815,8 @@ public: bool executeStep() override { - /// Metrics will be saved in the thread_group. - CurrentThread::ScopedAttach scoped_attach(thread_group); + /// Metrics will be saved in the local profile_counters. + ScopedProfileEvents profile_events_scope(&profile_counters); auto & current_level_parts = level_parts[current_level]; auto & next_level_parts = level_parts[next_level]; @@ -918,7 +919,7 @@ private: /// TODO(nikitamikhaylov): make this constant a setting static constexpr size_t max_parts_to_merge_in_one_level = 10; - ThreadGroupStatusPtr thread_group = std::make_shared(); + ProfileEvents::Counters profile_counters; }; @@ -929,7 +930,9 @@ private: // In short it executed a mutation for the part an original part and for its every projection -/** An overview of how the process of mutation works for projections: +/** + * + * An overview of how the process of mutation works for projections: * * The mutation for original parts is executed block by block, * but additionally we execute a SELECT query for each projection over a current block. @@ -1140,8 +1143,8 @@ public: bool executeStep() override { - /// Metrics will be saved in the thread_group. - CurrentThread::ScopedAttach scoped_attach(thread_group); + /// Metrics will be saved in the local profile_counters. + ScopedProfileEvents profile_events_scope(&profile_counters); switch (state) { @@ -1176,6 +1179,7 @@ public: } private: + void prepare() { if (ctx->new_data_part->isStoredOnDisk()) @@ -1258,7 +1262,7 @@ private: std::unique_ptr part_merger_writer_task; - ThreadGroupStatusPtr thread_group = std::make_shared(); + ProfileEvents::Counters profile_counters; }; @@ -1273,8 +1277,8 @@ public: bool executeStep() override { - /// Metrics will be saved in the thread_group. - CurrentThread::ScopedAttach scoped_attach(thread_group); + /// Metrics will be saved in the local profile_counters. + ScopedProfileEvents profile_events_scope(&profile_counters); switch (state) { @@ -1308,6 +1312,7 @@ public: } private: + void prepare() { if (ctx->execute_ttl_type != ExecuteTTLType::NONE) @@ -1466,7 +1471,7 @@ private: MergedColumnOnlyOutputStreamPtr out; std::unique_ptr part_merger_writer_task{nullptr}; - ThreadGroupStatusPtr thread_group = std::make_shared(); + ProfileEvents::Counters profile_counters; }; diff --git a/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp b/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp index 2ef36654d95..d81bbd67441 100644 --- a/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp @@ -2,6 +2,7 @@ #include #include +#include namespace DB @@ -29,8 +30,8 @@ void ReplicatedMergeMutateTaskBase::onCompleted() bool ReplicatedMergeMutateTaskBase::executeStep() { - /// Metrics will be saved in the thread_group. - CurrentThread::ScopedAttach scoped_attach(thread_group); + /// Metrics will be saved in the local profile_counters. + ScopedProfileEvents profile_events_scope(&profile_counters); std::exception_ptr saved_exception; diff --git a/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.h b/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.h index e08074cd4ff..d9a1cbff166 100644 --- a/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.h +++ b/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.h @@ -11,9 +11,9 @@ namespace DB class StorageReplicatedMergeTree; - -/** This is used as a base of MergeFromLogEntryTask and MutateFromLogEntryTaskBase - */ +/** + * This is used as a base of MergeFromLogEntryTask and MutateFromLogEntryTaskBase + */ class ReplicatedMergeMutateTaskBase : public IExecutableTask { public: @@ -33,11 +33,8 @@ public: } ~ReplicatedMergeMutateTaskBase() override = default; - void onCompleted() override; - StorageID getStorageID() override; - bool executeStep() override; protected: @@ -63,6 +60,8 @@ protected: MergeList::EntryPtr merge_mutate_entry{nullptr}; Poco::Logger * log; StorageReplicatedMergeTree & storage; + /// ProfileEvents for current part will be stored here + ProfileEvents::Counters profile_counters; private: enum class CheckExistingPartResult @@ -86,7 +85,6 @@ private: PartLogWriter part_log_writer{}; State state{State::NEED_PREPARE}; IExecutableTask::TaskResultCallback task_result_callback; - ThreadGroupStatusPtr thread_group = std::make_shared(); }; } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index e9958a68406..b9c178cab5c 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -2,8 +2,10 @@ #include #include -#include "Common/hex.h" + +#include #include +#include #include #include #include @@ -1592,6 +1594,8 @@ bool StorageReplicatedMergeTree::executeLogEntry(LogEntry & entry) if (entry.type == LogEntry::ATTACH_PART) { + ScopedProfileEvents profile_events_scope; + if (MutableDataPartPtr part = attachPartHelperFoundValidPart(entry)) { LOG_TRACE(log, "Found valid local part for {}, preparing the transaction", part->name); @@ -1602,11 +1606,9 @@ bool StorageReplicatedMergeTree::executeLogEntry(LogEntry & entry) renameTempPartAndReplace(part, transaction); checkPartChecksumsAndCommit(transaction, part); - auto & thread_status = CurrentThread::get(); - thread_status.finalizePerformanceCounters(); - auto profile_counters = std::make_shared(thread_status.performance_counters.getPartiallyAtomicSnapshot()); writePartLog(PartLogElement::Type::NEW_PART, {}, 0 /** log entry is fake so we don't measure the time */, - part->name, part, {} /** log entry is fake so there are no initial parts */, nullptr, std::move(profile_counters)); + part->name, part, {} /** log entry is fake so there are no initial parts */, nullptr, + profile_events_scope.getSnapshot()); return true; } @@ -4007,15 +4009,14 @@ bool StorageReplicatedMergeTree::fetchPart( Stopwatch stopwatch; MutableDataPartPtr part; DataPartsVector replaced_parts; + ScopedProfileEvents profile_events_scope; auto write_part_log = [&] (const ExecutionStatus & execution_status) { - auto & thread_status = CurrentThread::get(); - thread_status.finalizePerformanceCounters(); - auto profile_counters = std::make_shared(thread_status.performance_counters.getPartiallyAtomicSnapshot()); writePartLog( PartLogElement::DOWNLOAD_PART, execution_status, stopwatch.elapsed(), - part_name, part, replaced_parts, nullptr, std::move(profile_counters)); + part_name, part, replaced_parts, nullptr, + profile_events_scope.getSnapshot()); }; DataPartPtr part_to_clone; @@ -4246,15 +4247,14 @@ MutableDataPartStoragePtr StorageReplicatedMergeTree::fetchExistsPart( Stopwatch stopwatch; MutableDataPartPtr part; DataPartsVector replaced_parts; + ScopedProfileEvents profile_events_scope; auto write_part_log = [&] (const ExecutionStatus & execution_status) { - auto & thread_status = CurrentThread::get(); - thread_status.finalizePerformanceCounters(); - auto profile_counters = std::make_shared(thread_status.performance_counters.getPartiallyAtomicSnapshot()); writePartLog( PartLogElement::DOWNLOAD_PART, execution_status, stopwatch.elapsed(), - part_name, part, replaced_parts, nullptr, std::move(profile_counters)); + part_name, part, replaced_parts, nullptr, + profile_events_scope.getSnapshot()); }; std::function get_part; diff --git a/tests/queries/0_stateless/02378_part_log_profile_events.reference b/tests/queries/0_stateless/02378_part_log_profile_events.reference index d00491fd7e5..e8183f05f5d 100644 --- a/tests/queries/0_stateless/02378_part_log_profile_events.reference +++ b/tests/queries/0_stateless/02378_part_log_profile_events.reference @@ -1 +1,3 @@ 1 +1 +1 diff --git a/tests/queries/0_stateless/02378_part_log_profile_events.sql b/tests/queries/0_stateless/02378_part_log_profile_events.sql index 9ffd6aa4135..af8fe8a2669 100644 --- a/tests/queries/0_stateless/02378_part_log_profile_events.sql +++ b/tests/queries/0_stateless/02378_part_log_profile_events.sql @@ -1,13 +1,47 @@ DROP TABLE IF EXISTS test; -CREATE TABLE test (x UInt64) ENGINE = MergeTree ORDER BY x; -SET max_block_size = 1; -INSERT INTO test SELECT * FROM system.numbers LIMIT 1000; +CREATE TABLE test (key UInt64, val UInt64) engine = MergeTree Order by key PARTITION BY key >= 128; + +SET max_block_size = 64, max_insert_block_size = 64, min_insert_block_size_rows = 64; + +INSERT INTO test SELECT number AS key, sipHash64(number) AS val FROM numbers(512); + +SYSTEM FLUSH LOGS; +SELECT + count(DISTINCT query_id) == 1 + AND count() >= 512 / 64 -- 512 rows inserted, 64 rows per block + AND SUM(ProfileEvents['MergeTreeDataWriterRows']) == 512 + AND SUM(ProfileEvents['MergeTreeDataWriterUncompressedBytes']) > 1024 + AND SUM(ProfileEvents['MergeTreeDataWriterCompressedBytes']) > 1024 + AND SUM(ProfileEvents['MergeTreeDataWriterBlocks']) >= 8 +FROM system.part_log +WHERE event_time > now() - INTERVAL 10 MINUTE + AND database == currentDatabase() AND table == 'test' + AND event_type == 'NewPart' +; + OPTIMIZE TABLE test FINAL; SYSTEM FLUSH LOGS; +SELECT + count() >= 2 AND SUM(ProfileEvents['MergedRows']) >= 512 +FROM system.part_log +WHERE event_time > now() - INTERVAL 10 MINUTE + AND database == currentDatabase() AND table == 'test' + AND event_type == 'MergeParts' +; --- ProfileEvents field exist and contains something plausible: -SELECT count() > 0 FROM system.part_log WHERE ProfileEvents['MergedRows'] = 1000 AND table = 'test' AND database = currentDatabase() AND event_time >= now() - 600; +ALTER TABLE test UPDATE val = 0 WHERE key % 2 == 0 SETTINGS mutations_sync = 2; + +SYSTEM FLUSH LOGS; +SELECT + count() == 2 + AND SUM(ProfileEvents['SelectedRows']) == 512 + AND SUM(ProfileEvents['FileOpen']) > 2 +FROM system.part_log +WHERE event_time > now() - INTERVAL 10 MINUTE + AND database == currentDatabase() AND table == 'test' + AND event_type == 'MutatePart' +; DROP TABLE test; From 3f77bc0ecf219e60798f551bc92778ce38c4ff74 Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 23 Jan 2023 18:35:18 +0000 Subject: [PATCH 198/717] Add 02378_part_log_profile_events_replicated --- ...rt_log_profile_events_replicated.reference | 1 + ...378_part_log_profile_events_replicated.sql | 40 +++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 tests/queries/0_stateless/02378_part_log_profile_events_replicated.reference create mode 100644 tests/queries/0_stateless/02378_part_log_profile_events_replicated.sql diff --git a/tests/queries/0_stateless/02378_part_log_profile_events_replicated.reference b/tests/queries/0_stateless/02378_part_log_profile_events_replicated.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/02378_part_log_profile_events_replicated.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/02378_part_log_profile_events_replicated.sql b/tests/queries/0_stateless/02378_part_log_profile_events_replicated.sql new file mode 100644 index 00000000000..d61b680bb87 --- /dev/null +++ b/tests/queries/0_stateless/02378_part_log_profile_events_replicated.sql @@ -0,0 +1,40 @@ + +-- Tags: long, replica, no-replicated-database, no-parallel + +DROP TABLE IF EXISTS part_log_profile_events_r1 NO DELAY; +DROP TABLE IF EXISTS part_log_profile_events_r2 NO DELAY; + +CREATE TABLE part_log_profile_events_r1 (x UInt64) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/test_02378/part_log_profile_events', 'r1') +ORDER BY x +PARTITION BY x >= 128 +; + +CREATE TABLE part_log_profile_events_r2 (x UInt64) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/test_02378/part_log_profile_events', 'r2') +ORDER BY x +PARTITION BY x >= 128 +; + +-- SYSTEM STOP MERGES part_log_profile_events_r1; +-- SYSTEM STOP MERGES part_log_profile_events_r2; + +SET max_block_size = 64, max_insert_block_size = 64, min_insert_block_size_rows = 64; + +INSERT INTO part_log_profile_events_r1 SELECT number FROM numbers(1000); + +SYSTEM SYNC REPLICA part_log_profile_events_r2; + +SYSTEM FLUSH LOGS; + +SELECT + count() > 1 + AND SUM(ProfileEvents['ZooKeeperTransactions']) >= 4 +FROM system.part_log +WHERE event_time > now() - INTERVAL 10 MINUTE + AND database == currentDatabase() AND table == 'part_log_profile_events_r2' + AND event_type == 'DownloadPart' +; + +DROP TABLE part_log_profile_events_r1 NO DELAY; +DROP TABLE part_log_profile_events_r2 NO DELAY; From 994dcb93f95cf269cf4a1db3aa793cfffe8114d3 Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 24 Jan 2023 08:24:36 +0000 Subject: [PATCH 199/717] Rename ScopedProfileEvents -> ProfileEventsScope --- src/Common/ProfileEventsScope.cpp | 9 +++++---- src/Common/ProfileEventsScope.h | 11 +++++------ src/Storages/MergeTree/MergePlainMergeTreeTask.cpp | 2 +- src/Storages/MergeTree/MergeTreeData.cpp | 3 ++- src/Storages/MergeTree/MergeTreeSink.cpp | 4 ++-- src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp | 2 +- src/Storages/MergeTree/MutateTask.cpp | 6 +++--- .../MergeTree/ReplicatedMergeMutateTaskBase.cpp | 2 +- src/Storages/StorageReplicatedMergeTree.cpp | 6 +++--- 9 files changed, 23 insertions(+), 22 deletions(-) diff --git a/src/Common/ProfileEventsScope.cpp b/src/Common/ProfileEventsScope.cpp index 6dcb5f8fcf3..d5181692cae 100644 --- a/src/Common/ProfileEventsScope.cpp +++ b/src/Common/ProfileEventsScope.cpp @@ -2,28 +2,29 @@ namespace DB { + extern thread_local constinit ProfileEvents::Counters * subthread_profile_events; -ScopedProfileEvents::ScopedProfileEvents() +ProfileEventsScope::ProfileEventsScope() : performance_counters_holder(std::make_unique()) , performance_counters_scope(performance_counters_holder.get()) { CurrentThread::get().attachProfileCountersScope(performance_counters_scope); } -ScopedProfileEvents::ScopedProfileEvents(ProfileEvents::Counters * performance_counters_scope_) +ProfileEventsScope::ProfileEventsScope(ProfileEvents::Counters * performance_counters_scope_) : performance_counters_scope(performance_counters_scope_) { CurrentThread::get().attachProfileCountersScope(performance_counters_scope); } -std::shared_ptr ScopedProfileEvents::getSnapshot() +std::shared_ptr ProfileEventsScope::getSnapshot() { return std::make_shared(performance_counters_scope->getPartiallyAtomicSnapshot()); } -ScopedProfileEvents::~ScopedProfileEvents() +ProfileEventsScope::~ProfileEventsScope() { subthread_profile_events = nullptr; } diff --git a/src/Common/ProfileEventsScope.h b/src/Common/ProfileEventsScope.h index 59ba4f616d9..653a1ed55e7 100644 --- a/src/Common/ProfileEventsScope.h +++ b/src/Common/ProfileEventsScope.h @@ -6,21 +6,20 @@ namespace DB { - /// Use specific performance counters for current thread in the current scope. -class ScopedProfileEvents : private boost::noncopyable +class ProfileEventsScope : private boost::noncopyable { public: /// Counters are owned by this object. - ScopedProfileEvents(); + ProfileEventsScope(); - /// Shared counters are stored outisde. + /// Shared counters are stored outside. /// Useful when we calculate metrics entering into some scope several times. - explicit ScopedProfileEvents(ProfileEvents::Counters * performance_counters_scope_); + explicit ProfileEventsScope(ProfileEvents::Counters * performance_counters_scope_); std::shared_ptr getSnapshot(); - ~ScopedProfileEvents(); + ~ProfileEventsScope(); private: /// If set, then performance_counters_scope is owned by this object. diff --git a/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp b/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp index 5346a932fba..de88a1984e7 100644 --- a/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp +++ b/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp @@ -29,7 +29,7 @@ void MergePlainMergeTreeTask::onCompleted() bool MergePlainMergeTreeTask::executeStep() { /// Metrics will be saved in the thread_group. - ScopedProfileEvents profile_events_scope(&profile_counters); + ProfileEventsScope profile_events_scope(&profile_counters); /// Make out memory tracker a parent of current thread memory tracker MemoryTrackerThreadSwitcherPtr switcher; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index c5160a27d48..57924331cce 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -7488,7 +7488,8 @@ bool MergeTreeData::moveParts(const CurrentlyMovingPartsTaggerPtr & moving_tagge { Stopwatch stopwatch; MutableDataPartPtr cloned_part; - ScopedProfileEvents profile_events_scope; + ProfileEventsScope profile_events_scope; + auto write_part_log = [&](const ExecutionStatus & execution_status) { diff --git a/src/Storages/MergeTree/MergeTreeSink.cpp b/src/Storages/MergeTree/MergeTreeSink.cpp index be645cf27c0..1e607767f86 100644 --- a/src/Storages/MergeTree/MergeTreeSink.cpp +++ b/src/Storages/MergeTree/MergeTreeSink.cpp @@ -78,7 +78,7 @@ void MergeTreeSink::consume(Chunk chunk) MergeTreeDataWriter::TemporaryPart temp_part; { - ScopedProfileEvents scoped_attach(&part_counters); + ProfileEventsScope scoped_attach(&part_counters); Stopwatch watch; temp_part = storage.writer.writeTempPart(current_block, metadata_snapshot, context); @@ -145,7 +145,7 @@ void MergeTreeSink::finishDelayedChunk() for (auto & partition : delayed_chunk->partitions) { - ScopedProfileEvents scoped_attach(&partition.part_counters); + ProfileEventsScope scoped_attach(&partition.part_counters); partition.temp_part.finalize(); diff --git a/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp b/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp index 944a2d1070b..9bd0f148d6c 100644 --- a/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp +++ b/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp @@ -65,7 +65,7 @@ void MutatePlainMergeTreeTask::prepare() bool MutatePlainMergeTreeTask::executeStep() { /// Metrics will be saved in the local profile_counters. - ScopedProfileEvents profile_events_scope(&profile_counters); + ProfileEventsScope profile_events_scope(&profile_counters); /// Make out memory tracker a parent of current thread memory tracker MemoryTrackerThreadSwitcherPtr switcher; diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index a5ea5c90b95..563137e6a8c 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -816,7 +816,7 @@ public: bool executeStep() override { /// Metrics will be saved in the local profile_counters. - ScopedProfileEvents profile_events_scope(&profile_counters); + ProfileEventsScope profile_events_scope(&profile_counters); auto & current_level_parts = level_parts[current_level]; auto & next_level_parts = level_parts[next_level]; @@ -1144,7 +1144,7 @@ public: bool executeStep() override { /// Metrics will be saved in the local profile_counters. - ScopedProfileEvents profile_events_scope(&profile_counters); + ProfileEventsScope profile_events_scope(&profile_counters); switch (state) { @@ -1278,7 +1278,7 @@ public: bool executeStep() override { /// Metrics will be saved in the local profile_counters. - ScopedProfileEvents profile_events_scope(&profile_counters); + ProfileEventsScope profile_events_scope(&profile_counters); switch (state) { diff --git a/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp b/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp index d81bbd67441..9ce7eb42666 100644 --- a/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp @@ -31,7 +31,7 @@ void ReplicatedMergeMutateTaskBase::onCompleted() bool ReplicatedMergeMutateTaskBase::executeStep() { /// Metrics will be saved in the local profile_counters. - ScopedProfileEvents profile_events_scope(&profile_counters); + ProfileEventsScope profile_events_scope(&profile_counters); std::exception_ptr saved_exception; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index b9c178cab5c..8fc2adcad07 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -1594,7 +1594,7 @@ bool StorageReplicatedMergeTree::executeLogEntry(LogEntry & entry) if (entry.type == LogEntry::ATTACH_PART) { - ScopedProfileEvents profile_events_scope; + ProfileEventsScope profile_events_scope; if (MutableDataPartPtr part = attachPartHelperFoundValidPart(entry)) { @@ -4009,7 +4009,7 @@ bool StorageReplicatedMergeTree::fetchPart( Stopwatch stopwatch; MutableDataPartPtr part; DataPartsVector replaced_parts; - ScopedProfileEvents profile_events_scope; + ProfileEventsScope profile_events_scope; auto write_part_log = [&] (const ExecutionStatus & execution_status) { @@ -4247,7 +4247,7 @@ MutableDataPartStoragePtr StorageReplicatedMergeTree::fetchExistsPart( Stopwatch stopwatch; MutableDataPartPtr part; DataPartsVector replaced_parts; - ScopedProfileEvents profile_events_scope; + ProfileEventsScope profile_events_scope; auto write_part_log = [&] (const ExecutionStatus & execution_status) { From 07d7478bc71e963bca3b9e80ca0a8fe1fe4da7ca Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 24 Jan 2023 11:23:17 +0000 Subject: [PATCH 200/717] wip --- .../MergeTree/MergeFromLogEntryTask.cpp | 10 ++++---- .../MergeTree/MergePlainMergeTreeTask.cpp | 2 -- .../02378_part_log_profile_events.reference | 6 ++--- .../02378_part_log_profile_events.sql | 23 +++++++++++-------- 4 files changed, 22 insertions(+), 19 deletions(-) diff --git a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp index abb871be372..0d8dc7f68a6 100644 --- a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp @@ -2,6 +2,7 @@ #include #include +#include #include namespace ProfileEvents @@ -267,6 +268,7 @@ ReplicatedMergeMutateTaskBase::PrepareResult MergeFromLogEntryTask::prepare() transaction_ptr = std::make_unique(storage, NO_TRANSACTION_RAW); stopwatch_ptr = std::make_unique(); + ProfileEventsScope profile_events_scope(&profile_counters); merge_task = storage.merger_mutator.mergePartsToTemporaryPart( future_merged_part, @@ -289,18 +291,18 @@ ReplicatedMergeMutateTaskBase::PrepareResult MergeFromLogEntryTask::prepare() return {true, true, [this, stopwatch = *stopwatch_ptr] (const ExecutionStatus & execution_status) { - auto & thread_status = CurrentThread::get(); - thread_status.finalizePerformanceCounters(); - auto profile_counters = std::make_shared(thread_status.performance_counters.getPartiallyAtomicSnapshot()); + auto profile_counters_snapshot = std::make_shared(profile_counters.getPartiallyAtomicSnapshot()); storage.writePartLog( PartLogElement::MERGE_PARTS, execution_status, stopwatch.elapsed(), - entry.new_part_name, part, parts, merge_mutate_entry.get(), std::move(profile_counters)); + entry.new_part_name, part, parts, merge_mutate_entry.get(), std::move(profile_counters_snapshot)); }}; } bool MergeFromLogEntryTask::finalize(ReplicatedMergeMutateTaskBase::PartLogWriter write_part_log) { + ProfileEventsScope profile_events_scope(&profile_counters); + part = merge_task->getFuture().get(); storage.merger_mutator.renameMergedTemporaryPart(part, parts, NO_TRANSACTION_PTR, *transaction_ptr); diff --git a/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp b/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp index de88a1984e7..855b93dc90e 100644 --- a/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp +++ b/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp @@ -89,8 +89,6 @@ void MergePlainMergeTreeTask::prepare() write_part_log = [this] (const ExecutionStatus & execution_status) { - auto & thread_status = CurrentThread::get(); - thread_status.finalizePerformanceCounters(); auto profile_counters_snapshot = std::make_shared(profile_counters.getPartiallyAtomicSnapshot()); merge_task.reset(); storage.writePartLog( diff --git a/tests/queries/0_stateless/02378_part_log_profile_events.reference b/tests/queries/0_stateless/02378_part_log_profile_events.reference index e8183f05f5d..c09e6c997c5 100644 --- a/tests/queries/0_stateless/02378_part_log_profile_events.reference +++ b/tests/queries/0_stateless/02378_part_log_profile_events.reference @@ -1,3 +1,3 @@ -1 -1 -1 +Ok Ok Ok Ok Ok Ok +Ok Ok +Ok Ok Ok diff --git a/tests/queries/0_stateless/02378_part_log_profile_events.sql b/tests/queries/0_stateless/02378_part_log_profile_events.sql index af8fe8a2669..af68360dbbb 100644 --- a/tests/queries/0_stateless/02378_part_log_profile_events.sql +++ b/tests/queries/0_stateless/02378_part_log_profile_events.sql @@ -7,13 +7,14 @@ SET max_block_size = 64, max_insert_block_size = 64, min_insert_block_size_rows INSERT INTO test SELECT number AS key, sipHash64(number) AS val FROM numbers(512); SYSTEM FLUSH LOGS; + SELECT - count(DISTINCT query_id) == 1 - AND count() >= 512 / 64 -- 512 rows inserted, 64 rows per block - AND SUM(ProfileEvents['MergeTreeDataWriterRows']) == 512 - AND SUM(ProfileEvents['MergeTreeDataWriterUncompressedBytes']) > 1024 - AND SUM(ProfileEvents['MergeTreeDataWriterCompressedBytes']) > 1024 - AND SUM(ProfileEvents['MergeTreeDataWriterBlocks']) >= 8 + if(count(DISTINCT query_id) == 1, 'Ok', 'Error: ' || toString(count(DISTINCT query_id))), + if(count() == 512 / 64, 'Ok', 'Error: ' || toString(count())), -- 512 rows inserted, 64 rows per block + if(SUM(ProfileEvents['MergeTreeDataWriterRows']) == 512, 'Ok', 'Error: ' || toString(SUM(ProfileEvents['MergeTreeDataWriterRows']))), + if(SUM(ProfileEvents['MergeTreeDataWriterUncompressedBytes']) >= 1024, 'Ok', 'Error: ' || toString(SUM(ProfileEvents['MergeTreeDataWriterUncompressedBytes']))), + if(SUM(ProfileEvents['MergeTreeDataWriterCompressedBytes']) >= 1024, 'Ok', 'Error: ' || toString(SUM(ProfileEvents['MergeTreeDataWriterCompressedBytes']))), + if(SUM(ProfileEvents['MergeTreeDataWriterBlocks']) >= 8, 'Ok', 'Error: ' || toString(SUM(ProfileEvents['MergeTreeDataWriterBlocks']))) FROM system.part_log WHERE event_time > now() - INTERVAL 10 MINUTE AND database == currentDatabase() AND table == 'test' @@ -24,7 +25,8 @@ OPTIMIZE TABLE test FINAL; SYSTEM FLUSH LOGS; SELECT - count() >= 2 AND SUM(ProfileEvents['MergedRows']) >= 512 + if(count() > 2, 'Ok', 'Error: ' || toString(count())), + if(SUM(ProfileEvents['MergedRows']) >= 512, 'Ok', 'Error: ' || toString(SUM(ProfileEvents['MergedRows']))) FROM system.part_log WHERE event_time > now() - INTERVAL 10 MINUTE AND database == currentDatabase() AND table == 'test' @@ -34,10 +36,11 @@ WHERE event_time > now() - INTERVAL 10 MINUTE ALTER TABLE test UPDATE val = 0 WHERE key % 2 == 0 SETTINGS mutations_sync = 2; SYSTEM FLUSH LOGS; + SELECT - count() == 2 - AND SUM(ProfileEvents['SelectedRows']) == 512 - AND SUM(ProfileEvents['FileOpen']) > 2 + if(count() == 2, 'Ok', 'Error: ' || toString(count())), + if(SUM(ProfileEvents['SelectedRows']) == 512, 'Ok', 'Error: ' || toString(SUM(ProfileEvents['SelectedRows']))), + if(SUM(ProfileEvents['FileOpen']) > 1, 'Ok', 'Error: ' || toString(SUM(ProfileEvents['FileOpen']))) FROM system.part_log WHERE event_time > now() - INTERVAL 10 MINUTE AND database == currentDatabase() AND table == 'test' From fa08f72bba61dcf76dc2af93d18604dfbf23481a Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 31 Jan 2023 12:34:07 +0000 Subject: [PATCH 201/717] Move subthread_profile_events to ThreadStatus --- src/Common/CurrentThread.cpp | 4 +--- src/Common/ProfileEventsScope.cpp | 4 +--- src/Common/ThreadStatus.cpp | 1 - src/Common/ThreadStatus.h | 6 +++++- src/Interpreters/ThreadStatusExt.cpp | 7 ++++++- 5 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/Common/CurrentThread.cpp b/src/Common/CurrentThread.cpp index 14c9f4418e0..a176a19673b 100644 --- a/src/Common/CurrentThread.cpp +++ b/src/Common/CurrentThread.cpp @@ -40,9 +40,7 @@ ThreadStatus & CurrentThread::get() ProfileEvents::Counters & CurrentThread::getProfileEvents() { - if (unlikely(subthread_profile_events)) - return *subthread_profile_events; - return current_thread ? current_thread->performance_counters : ProfileEvents::global_counters; + return current_thread ? *current_thread->current_performance_counters : ProfileEvents::global_counters; } void CurrentThread::updateProgressIn(const Progress & value) diff --git a/src/Common/ProfileEventsScope.cpp b/src/Common/ProfileEventsScope.cpp index d5181692cae..5b996209b76 100644 --- a/src/Common/ProfileEventsScope.cpp +++ b/src/Common/ProfileEventsScope.cpp @@ -3,8 +3,6 @@ namespace DB { -extern thread_local constinit ProfileEvents::Counters * subthread_profile_events; - ProfileEventsScope::ProfileEventsScope() : performance_counters_holder(std::make_unique()) @@ -26,7 +24,7 @@ std::shared_ptr ProfileEventsScope::getSnapsh ProfileEventsScope::~ProfileEventsScope() { - subthread_profile_events = nullptr; + CurrentThread::get().detachProfileCountersScope(); } diff --git a/src/Common/ThreadStatus.cpp b/src/Common/ThreadStatus.cpp index 6949942f745..9f9a78c4036 100644 --- a/src/Common/ThreadStatus.cpp +++ b/src/Common/ThreadStatus.cpp @@ -25,7 +25,6 @@ namespace ErrorCodes } thread_local ThreadStatus constinit * current_thread = nullptr; -thread_local ProfileEvents::Counters constinit * subthread_profile_events = nullptr; #if !defined(SANITIZER) namespace diff --git a/src/Common/ThreadStatus.h b/src/Common/ThreadStatus.h index d44cd57b812..ac53faa2a75 100644 --- a/src/Common/ThreadStatus.h +++ b/src/Common/ThreadStatus.h @@ -107,7 +107,6 @@ using ThreadGroupStatusPtr = std::shared_ptr; * - https://github.com/ClickHouse/ClickHouse/pull/40078 */ extern thread_local constinit ThreadStatus * current_thread; -extern thread_local constinit ProfileEvents::Counters * subthread_profile_events; /** Encapsulates all per-thread info (ProfileEvents, MemoryTracker, query_id, query context, etc.). * The object must be created in thread function and destroyed in the same thread before the exit. @@ -125,6 +124,10 @@ public: /// TODO: merge them into common entity ProfileEvents::Counters performance_counters{VariableContext::Thread}; + + /// Points to performance_counters by default. + /// Could be changed to point to another object to caclulate performance counters for some narrow scope. + ProfileEvents::Counters * current_performance_counters{&performance_counters}; MemoryTracker memory_tracker{VariableContext::Thread}; /// Small amount of untracked memory (per thread atomic-less counter) @@ -247,6 +250,7 @@ public: void attachQuery(const ThreadGroupStatusPtr & thread_group_, bool check_detached = true); void attachProfileCountersScope(ProfileEvents::Counters * performance_counters_scope); + void detachProfileCountersScope(); InternalTextLogsQueuePtr getInternalTextLogsQueue() const { diff --git a/src/Interpreters/ThreadStatusExt.cpp b/src/Interpreters/ThreadStatusExt.cpp index b4f82b2a215..7c1a1fb7bb4 100644 --- a/src/Interpreters/ThreadStatusExt.cpp +++ b/src/Interpreters/ThreadStatusExt.cpp @@ -163,8 +163,13 @@ void ThreadStatus::attachQuery(const ThreadGroupStatusPtr & thread_group_, bool void ThreadStatus::attachProfileCountersScope(ProfileEvents::Counters * performance_counters_scope) { - subthread_profile_events = performance_counters_scope; performance_counters_scope->setParent(&performance_counters); + current_performance_counters = performance_counters_scope; +} + +void ThreadStatus::detachProfileCountersScope() +{ + current_performance_counters = &performance_counters; } void ThreadStatus::initPerformanceCounters() From 7a66c0105a0c2641bd632395d16cc1642b5d6268 Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 31 Jan 2023 12:59:15 +0000 Subject: [PATCH 202/717] Remove unused ProfileEventsExt::dumpToString --- src/Interpreters/ProfileEventsExt.cpp | 16 ---------------- src/Interpreters/ProfileEventsExt.h | 2 -- 2 files changed, 18 deletions(-) diff --git a/src/Interpreters/ProfileEventsExt.cpp b/src/Interpreters/ProfileEventsExt.cpp index b5a49c8f381..0f6b52b2611 100644 --- a/src/Interpreters/ProfileEventsExt.cpp +++ b/src/Interpreters/ProfileEventsExt.cpp @@ -19,22 +19,6 @@ std::shared_ptr TypeEnum = std::make_shared(GAUGE)}, }); -String dumpToString(const Counters::Snapshot & counters, bool nonzero_only) -{ - std::vector ss; - for (Event event = 0; event < Counters::num_counters; ++event) - { - UInt64 value = counters[event]; - - if (nonzero_only && 0 == value) - continue; - - const char * desc = getName(event); - ss.push_back(fmt::format("{}: {}", desc, value)); - } - return fmt::format("[{}]", fmt::join(ss, ", ")); -} - /// Put implementation here to avoid extra linking dependencies for clickhouse_common_io void dumpToMapColumn(const Counters::Snapshot & counters, DB::IColumn * column, bool nonzero_only) { diff --git a/src/Interpreters/ProfileEventsExt.h b/src/Interpreters/ProfileEventsExt.h index be88ca02593..7d9fc512d15 100644 --- a/src/Interpreters/ProfileEventsExt.h +++ b/src/Interpreters/ProfileEventsExt.h @@ -21,8 +21,6 @@ struct ProfileEventsSnapshot using ThreadIdToCountersSnapshot = std::unordered_map; -String dumpToString(const Counters::Snapshot & counters, bool nonzero_only = true); - /// Dumps profile events to columns Map(String, UInt64) void dumpToMapColumn(const Counters::Snapshot & counters, DB::IColumn * column, bool nonzero_only = true); From 4c1aeb797209999136cee0c9497e3ea014f0db07 Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 31 Jan 2023 14:01:12 +0000 Subject: [PATCH 203/717] Use overload of PartLog::addNewPart with profile_events --- src/Common/ThreadStatus.h | 2 +- src/Interpreters/PartLog.cpp | 17 +++------ src/Interpreters/PartLog.h | 8 ++--- src/Storages/MergeTree/MergeTreeData.cpp | 1 - .../MergeTree/ReplicatedMergeTreeSink.cpp | 36 ++++++++++++++----- src/Storages/StorageMergeTree.cpp | 23 ++++++++---- src/Storages/StorageReplicatedMergeTree.cpp | 18 ++++++---- 7 files changed, 64 insertions(+), 41 deletions(-) diff --git a/src/Common/ThreadStatus.h b/src/Common/ThreadStatus.h index ac53faa2a75..531dd669e65 100644 --- a/src/Common/ThreadStatus.h +++ b/src/Common/ThreadStatus.h @@ -126,7 +126,7 @@ public: ProfileEvents::Counters performance_counters{VariableContext::Thread}; /// Points to performance_counters by default. - /// Could be changed to point to another object to caclulate performance counters for some narrow scope. + /// Could be changed to point to another object to calculate performance counters for some narrow scope. ProfileEvents::Counters * current_performance_counters{&performance_counters}; MemoryTracker memory_tracker{VariableContext::Thread}; diff --git a/src/Interpreters/PartLog.cpp b/src/Interpreters/PartLog.cpp index b736fa43e0c..37650196783 100644 --- a/src/Interpreters/PartLog.cpp +++ b/src/Interpreters/PartLog.cpp @@ -258,23 +258,16 @@ bool PartLog::addNewPart(ContextPtr context, const PartLog::PartLogEntry & part, return addNewParts(context, {part}, execution_status); } -bool PartLog::addNewParts(ContextPtr context, const PartLog::MutableDataPartsVector & parts, UInt64 elapsed_ns, - const ExecutionStatus & execution_status) + +PartLog::PartLogEntries PartLog::createPartLogEntries(const MutableDataPartsVector & parts, UInt64 elapsed_ns, ProfileCountersSnapshotPtr profile_counters) { - PartLog::PartLogEntries part_log_entries; + PartLogEntries part_log_entries; part_log_entries.reserve(parts.size()); for (const auto & part : parts) - part_log_entries.emplace_back(part, elapsed_ns); + part_log_entries.emplace_back(part, elapsed_ns, profile_counters); - return addNewParts(context, part_log_entries, execution_status); -} - -bool PartLog::addNewPart(ContextPtr context, const PartLog::MutableDataPartPtr & part, UInt64 elapsed_ns, const ExecutionStatus & execution_status) -{ - PartLog::PartLogEntries part_log_entries; - part_log_entries.emplace_back(part, elapsed_ns); - return addNewParts(context, part_log_entries, execution_status); + return part_log_entries; } } diff --git a/src/Interpreters/PartLog.h b/src/Interpreters/PartLog.h index 57044cba844..843792d03a9 100644 --- a/src/Interpreters/PartLog.h +++ b/src/Interpreters/PartLog.h @@ -133,18 +133,14 @@ public: using PartLogEntries = std::vector; + static PartLogEntries createPartLogEntries(const MutableDataPartsVector & parts, UInt64 elapsed_ns, ProfileCountersSnapshotPtr profile_counters = {}); + /// Add a record about creation of new part. static bool addNewPart(ContextPtr context, const PartLogEntry & part, const ExecutionStatus & execution_status = {}); static bool addNewParts(ContextPtr context, const PartLogEntries & parts, const ExecutionStatus & execution_status = {}); - - /// Add a record about creation of new part. - static bool addNewPart(ContextPtr context, const MutableDataPartPtr & part, UInt64 elapsed_ns, - const ExecutionStatus & execution_status = {}); - static bool addNewParts(ContextPtr context, const MutableDataPartsVector & parts, UInt64 elapsed_ns, - const ExecutionStatus & execution_status = {}); }; } diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 57924331cce..d13452c291e 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -7490,7 +7490,6 @@ bool MergeTreeData::moveParts(const CurrentlyMovingPartsTaggerPtr & moving_tagge MutableDataPartPtr cloned_part; ProfileEventsScope profile_events_scope; - auto write_part_log = [&](const ExecutionStatus & execution_status) { writePartLog( diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index ee192966758..31fd99f0aa1 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -48,14 +49,21 @@ struct ReplicatedMergeTreeSinkImpl::DelayedChunk BlockIDsType block_id; BlockWithPartition block_with_partition; std::unordered_map> block_id_to_offset_idx; + ProfileEvents::Counters part_counters; Partition() = default; - Partition(Poco::Logger * log_, MergeTreeDataWriter::TemporaryPart && temp_part_, UInt64 elapsed_ns_, BlockIDsType && block_id_, BlockWithPartition && block_) + Partition(Poco::Logger * log_, + MergeTreeDataWriter::TemporaryPart && temp_part_, + UInt64 elapsed_ns_, + BlockIDsType && block_id_, + BlockWithPartition && block_, + ProfileEvents::Counters && part_counters_) : log(log_), temp_part(std::move(temp_part_)), elapsed_ns(elapsed_ns_), block_id(std::move(block_id_)), - block_with_partition(std::move(block_)) + block_with_partition(std::move(block_)), + part_counters(std::move(part_counters_)) { initBlockIDMap(); } @@ -186,8 +194,9 @@ std::vector testSelfDeduplicate(std::vector data, std::vector::DelayedChunk::Partition part( - &Poco::Logger::get("testSelfDeduplicate"), MergeTreeDataWriter::TemporaryPart(), 0, std::move(hashes), std::move(block1)); + &Poco::Logger::get("testSelfDeduplicate"), MergeTreeDataWriter::TemporaryPart(), 0, std::move(hashes), std::move(block1), std::move(profile_counters)); part.filterSelfDuplicate(); @@ -411,6 +420,9 @@ void ReplicatedMergeTreeSinkImpl::consume(Chunk chunk) { Stopwatch watch; + ProfileEvents::Counters part_counters; + auto profile_events_scope = std::make_unique(&part_counters); + /// Write part to the filesystem under temporary name. Calculate a checksum. auto temp_part = storage.writer.writeTempPart(current_block, metadata_snapshot, context); @@ -452,6 +464,7 @@ void ReplicatedMergeTreeSinkImpl::consume(Chunk chunk) LOG_DEBUG(log, "Wrote block with {} rows{}", current_block.block.rows(), quorumLogMessage(replicas_num)); } + profile_events_scope.reset(); UInt64 elapsed_ns = watch.elapsed(); size_t max_insert_delayed_streams_for_parallel_write = DEFAULT_DELAYED_STREAMS_FOR_PARALLEL_WRITE; @@ -472,12 +485,14 @@ void ReplicatedMergeTreeSinkImpl::consume(Chunk chunk) partitions = DelayedPartitions{}; } + partitions.emplace_back(DelayedPartition( log, std::move(temp_part), elapsed_ns, std::move(block_id), - std::move(current_block) + std::move(current_block), + std::move(part_counters) /// profile_events_scope must be reset here. )); } @@ -503,6 +518,8 @@ void ReplicatedMergeTreeSinkImpl::finishDelayedChunk(const ZooKeeperWithF for (auto & partition : delayed_chunk->partitions) { + ProfileEventsScope scoped_attach(&partition.part_counters); + partition.temp_part.finalize(); auto & part = partition.temp_part.part; @@ -515,12 +532,14 @@ void ReplicatedMergeTreeSinkImpl::finishDelayedChunk(const ZooKeeperWithF /// Set a special error code if the block is duplicate int error = (deduplicate && part->is_duplicate) ? ErrorCodes::INSERT_WAS_DEDUPLICATED : 0; - PartLog::addNewPart(storage.getContext(), part, partition.elapsed_ns, ExecutionStatus(error)); + auto counters_snapshot = std::make_shared(partition.part_counters.getPartiallyAtomicSnapshot()); + PartLog::addNewPart(storage.getContext(), PartLog::PartLogEntry(part, partition.elapsed_ns, counters_snapshot), ExecutionStatus(error)); storage.incrementInsertedPartsProfileEvent(part->getType()); } catch (...) { - PartLog::addNewPart(storage.getContext(), part, partition.elapsed_ns, ExecutionStatus::fromCurrentException("", true)); + auto counters_snapshot = std::make_shared(partition.part_counters.getPartiallyAtomicSnapshot()); + PartLog::addNewPart(storage.getContext(), PartLog::PartLogEntry(part, partition.elapsed_ns, counters_snapshot), ExecutionStatus::fromCurrentException("", true)); throw; } } @@ -579,16 +598,17 @@ void ReplicatedMergeTreeSinkImpl::writeExistingPart(MergeTreeData: size_t replicas_num = checkQuorumPrecondition(zookeeper); Stopwatch watch; + ProfileEventsScope profile_events_scope; try { part->version.setCreationTID(Tx::PrehistoricTID, nullptr); commitPart(zookeeper, part, BlockIDsType(), replicas_num, true); - PartLog::addNewPart(storage.getContext(), part, watch.elapsed()); + PartLog::addNewPart(storage.getContext(), PartLog::PartLogEntry(part, watch.elapsed(), profile_events_scope.getSnapshot())); } catch (...) { - PartLog::addNewPart(storage.getContext(), part, watch.elapsed(), ExecutionStatus::fromCurrentException("", true)); + PartLog::addNewPart(storage.getContext(), PartLog::PartLogEntry(part, watch.elapsed(), profile_events_scope.getSnapshot()), ExecutionStatus::fromCurrentException("", true)); throw; } } diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 125322281d0..9e98946baeb 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -1608,6 +1609,7 @@ void StorageMergeTree::truncate(const ASTPtr &, const StorageMetadataPtr &, Cont waitForOutdatedPartsToBeLoaded(); Stopwatch watch; + ProfileEventsScope profile_events_scope; auto txn = query_context->getCurrentTransaction(); MergeTreeData::Transaction transaction(*this, txn.get()); @@ -1628,7 +1630,7 @@ void StorageMergeTree::truncate(const ASTPtr &, const StorageMetadataPtr &, Cont auto new_data_parts = createEmptyDataParts(*this, future_parts, txn); renameAndCommitEmptyParts(new_data_parts, transaction); - PartLog::addNewParts(query_context, new_data_parts, watch.elapsed()); + PartLog::addNewParts(query_context, PartLog::createPartLogEntries(new_data_parts, watch.elapsed(), profile_events_scope.getSnapshot())); LOG_INFO(log, "Truncated table with {} parts by replacing them with new empty {} parts. With txn {}", parts.size(), future_parts.size(), @@ -1650,6 +1652,7 @@ void StorageMergeTree::dropPart(const String & part_name, bool detach, ContextPt auto merge_blocker = stopMergesAndWait(); Stopwatch watch; + ProfileEventsScope profile_events_scope; /// It's important to create it outside of lock scope because /// otherwise it can lock parts in destructor and deadlock is possible. @@ -1681,7 +1684,7 @@ void StorageMergeTree::dropPart(const String & part_name, bool detach, ContextPt auto new_data_parts = createEmptyDataParts(*this, future_parts, txn); renameAndCommitEmptyParts(new_data_parts, transaction); - PartLog::addNewParts(query_context, new_data_parts, watch.elapsed()); + PartLog::addNewParts(query_context, PartLog::createPartLogEntries(new_data_parts, watch.elapsed(), profile_events_scope.getSnapshot())); const auto * op = detach ? "Detached" : "Dropped"; LOG_INFO(log, "{} {} part by replacing it with new empty {} part. With txn {}", @@ -1707,6 +1710,7 @@ void StorageMergeTree::dropPartition(const ASTPtr & partition, bool detach, Cont auto merge_blocker = stopMergesAndWait(); Stopwatch watch; + ProfileEventsScope profile_events_scope; /// It's important to create it outside of lock scope because /// otherwise it can lock parts in destructor and deadlock is possible. @@ -1746,7 +1750,7 @@ void StorageMergeTree::dropPartition(const ASTPtr & partition, bool detach, Cont auto new_data_parts = createEmptyDataParts(*this, future_parts, txn); renameAndCommitEmptyParts(new_data_parts, transaction); - PartLog::addNewParts(query_context, new_data_parts, watch.elapsed()); + PartLog::addNewParts(query_context, PartLog::createPartLogEntries(new_data_parts, watch.elapsed(), profile_events_scope.getSnapshot())); const auto * op = detach ? "Detached" : "Dropped"; LOG_INFO(log, "{} partition with {} parts by replacing them with new empty {} parts. With txn {}", @@ -1814,6 +1818,8 @@ void StorageMergeTree::replacePartitionFrom(const StoragePtr & source_table, con auto my_metadata_snapshot = getInMemoryMetadataPtr(); Stopwatch watch; + ProfileEventsScope profile_events_scope; + MergeTreeData & src_data = checkStructureAndGetMergeTreeData(source_table, source_metadata_snapshot, my_metadata_snapshot); String partition_id = getPartitionIDFromQuery(partition, local_context); @@ -1878,11 +1884,12 @@ void StorageMergeTree::replacePartitionFrom(const StoragePtr & source_table, con removePartsInRangeFromWorkingSet(local_context->getCurrentTransaction().get(), drop_range, data_parts_lock); } - PartLog::addNewParts(getContext(), dst_parts, watch.elapsed()); + /// Note: same elapsed time and profile events for all parts is used + PartLog::addNewParts(getContext(), PartLog::createPartLogEntries(dst_parts, watch.elapsed(), profile_events_scope.getSnapshot())); } catch (...) { - PartLog::addNewParts(getContext(), dst_parts, watch.elapsed(), ExecutionStatus::fromCurrentException("", true)); + PartLog::addNewParts(getContext(), PartLog::createPartLogEntries(dst_parts, watch.elapsed()), ExecutionStatus::fromCurrentException("", true)); throw; } } @@ -1909,6 +1916,7 @@ void StorageMergeTree::movePartitionToTable(const StoragePtr & dest_table, const auto dest_metadata_snapshot = dest_table->getInMemoryMetadataPtr(); auto metadata_snapshot = getInMemoryMetadataPtr(); Stopwatch watch; + ProfileEventsScope profile_events_scope; MergeTreeData & src_data = dest_table_storage->checkStructureAndGetMergeTreeData(*this, metadata_snapshot, dest_metadata_snapshot); String partition_id = getPartitionIDFromQuery(partition, local_context); @@ -1961,11 +1969,12 @@ void StorageMergeTree::movePartitionToTable(const StoragePtr & dest_table, const clearOldPartsFromFilesystem(); - PartLog::addNewParts(getContext(), dst_parts, watch.elapsed()); + /// Note: same elapsed time and profile events for all parts is used + PartLog::addNewParts(getContext(), PartLog::createPartLogEntries(dst_parts, watch.elapsed(), profile_events_scope.getSnapshot())); } catch (...) { - PartLog::addNewParts(getContext(), dst_parts, watch.elapsed(), ExecutionStatus::fromCurrentException("", true)); + PartLog::addNewParts(getContext(), PartLog::createPartLogEntries(dst_parts, watch.elapsed()), ExecutionStatus::fromCurrentException("", true)); throw; } } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 8fc2adcad07..e2844431e34 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -1952,6 +1952,8 @@ void StorageReplicatedMergeTree::executeDropRange(const LogEntry & entry) bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) { Stopwatch watch; + ProfileEventsScope profile_events_scope; + auto & entry_replace = *entry.replace_range_entry; LOG_DEBUG(log, "Executing log entry {} to replace parts range {} with {} parts from {}.{}", entry.znode_name, entry_replace.drop_range_part_name, entry_replace.new_part_names.size(), @@ -2344,11 +2346,11 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) } } - PartLog::addNewParts(getContext(), res_parts, watch.elapsed()); + PartLog::addNewParts(getContext(), PartLog::createPartLogEntries(res_parts, watch.elapsed(), profile_events_scope.getSnapshot())); } catch (...) { - PartLog::addNewParts(getContext(), res_parts, watch.elapsed(), ExecutionStatus::fromCurrentException("", true)); + PartLog::addNewParts(getContext(), PartLog::createPartLogEntries(res_parts, watch.elapsed()), ExecutionStatus::fromCurrentException("", true)); for (const auto & res_part : res_parts) unlockSharedData(*res_part); @@ -6887,6 +6889,8 @@ void StorageReplicatedMergeTree::replacePartitionFrom( auto metadata_snapshot = getInMemoryMetadataPtr(); Stopwatch watch; + ProfileEventsScope profile_events_scope; + MergeTreeData & src_data = checkStructureAndGetMergeTreeData(source_table, source_metadata_snapshot, metadata_snapshot); String partition_id = getPartitionIDFromQuery(partition, query_context); @@ -7063,11 +7067,11 @@ void StorageReplicatedMergeTree::replacePartitionFrom( parts_to_remove = removePartsInRangeFromWorkingSetAndGetPartsToRemoveFromZooKeeper(NO_TRANSACTION_RAW, drop_range, data_parts_lock); } - PartLog::addNewParts(getContext(), dst_parts, watch.elapsed()); + PartLog::addNewParts(getContext(), PartLog::createPartLogEntries(dst_parts, watch.elapsed(), profile_events_scope.getSnapshot())); } catch (...) { - PartLog::addNewParts(getContext(), dst_parts, watch.elapsed(), ExecutionStatus::fromCurrentException("", true)); + PartLog::addNewParts(getContext(), PartLog::createPartLogEntries(dst_parts, watch.elapsed()), ExecutionStatus::fromCurrentException("", true)); for (const auto & dst_part : dst_parts) unlockSharedData(*dst_part); @@ -7122,6 +7126,8 @@ void StorageReplicatedMergeTree::movePartitionToTable(const StoragePtr & dest_ta auto metadata_snapshot = getInMemoryMetadataPtr(); Stopwatch watch; + ProfileEventsScope profile_events_scope; + MergeTreeData & src_data = dest_table_storage->checkStructureAndGetMergeTreeData(*this, metadata_snapshot, dest_metadata_snapshot); auto src_data_id = src_data.getStorageID(); String partition_id = getPartitionIDFromQuery(partition, query_context); @@ -7292,11 +7298,11 @@ void StorageReplicatedMergeTree::movePartitionToTable(const StoragePtr & dest_ta transaction.commit(&src_data_parts_lock); } - PartLog::addNewParts(getContext(), dst_parts, watch.elapsed()); + PartLog::addNewParts(getContext(), PartLog::createPartLogEntries(dst_parts, watch.elapsed(), profile_events_scope.getSnapshot())); } catch (...) { - PartLog::addNewParts(getContext(), dst_parts, watch.elapsed(), ExecutionStatus::fromCurrentException("", true)); + PartLog::addNewParts(getContext(), PartLog::createPartLogEntries(dst_parts, watch.elapsed()), ExecutionStatus::fromCurrentException("", true)); for (const auto & dst_part : dst_parts) dest_table_storage->unlockSharedData(*dst_part); From 83dea52affdcd7bea9f19b68c7cf9bbd00b1e3ab Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 1 Feb 2023 12:19:35 +0000 Subject: [PATCH 204/717] add sanity check to ThreadStatus::attachProfileCountersScope --- src/Interpreters/ThreadStatusExt.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/ThreadStatusExt.cpp b/src/Interpreters/ThreadStatusExt.cpp index 7c1a1fb7bb4..d13617b1d93 100644 --- a/src/Interpreters/ThreadStatusExt.cpp +++ b/src/Interpreters/ThreadStatusExt.cpp @@ -163,7 +163,9 @@ void ThreadStatus::attachQuery(const ThreadGroupStatusPtr & thread_group_, bool void ThreadStatus::attachProfileCountersScope(ProfileEvents::Counters * performance_counters_scope) { - performance_counters_scope->setParent(&performance_counters); + if (current_performance_counters != &performance_counters) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Can't attach performance counters scope to the thread, it is already attached to another scope"); + performance_counters_scope->setParent(current_performance_counters); current_performance_counters = performance_counters_scope; } From abd9119583b78ff798bb2341115f03f8ef3d54d1 Mon Sep 17 00:00:00 2001 From: vdimir Date: Thu, 2 Feb 2023 09:46:01 +0000 Subject: [PATCH 205/717] Remove unnecessary ProfileEventsScope --- src/Interpreters/ThreadStatusExt.cpp | 5 +++++ src/Storages/MergeTree/MergeFromLogEntryTask.cpp | 3 --- src/Storages/MergeTree/MutateTask.cpp | 14 -------------- 3 files changed, 5 insertions(+), 17 deletions(-) diff --git a/src/Interpreters/ThreadStatusExt.cpp b/src/Interpreters/ThreadStatusExt.cpp index d13617b1d93..ea2d5c4a125 100644 --- a/src/Interpreters/ThreadStatusExt.cpp +++ b/src/Interpreters/ThreadStatusExt.cpp @@ -163,8 +163,13 @@ void ThreadStatus::attachQuery(const ThreadGroupStatusPtr & thread_group_, bool void ThreadStatus::attachProfileCountersScope(ProfileEvents::Counters * performance_counters_scope) { + if (current_performance_counters == performance_counters_scope) + /// Allow to attach the same scope multiple times + return; + if (current_performance_counters != &performance_counters) throw Exception(ErrorCodes::LOGICAL_ERROR, "Can't attach performance counters scope to the thread, it is already attached to another scope"); + performance_counters_scope->setParent(current_performance_counters); current_performance_counters = performance_counters_scope; } diff --git a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp index 0d8dc7f68a6..a4e5b1f0575 100644 --- a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp @@ -268,7 +268,6 @@ ReplicatedMergeMutateTaskBase::PrepareResult MergeFromLogEntryTask::prepare() transaction_ptr = std::make_unique(storage, NO_TRANSACTION_RAW); stopwatch_ptr = std::make_unique(); - ProfileEventsScope profile_events_scope(&profile_counters); merge_task = storage.merger_mutator.mergePartsToTemporaryPart( future_merged_part, @@ -301,8 +300,6 @@ ReplicatedMergeMutateTaskBase::PrepareResult MergeFromLogEntryTask::prepare() bool MergeFromLogEntryTask::finalize(ReplicatedMergeMutateTaskBase::PartLogWriter write_part_log) { - ProfileEventsScope profile_events_scope(&profile_counters); - part = merge_task->getFuture().get(); storage.merger_mutator.renameMergedTemporaryPart(part, parts, NO_TRANSACTION_PTR, *transaction_ptr); diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index 563137e6a8c..43cf3d950b2 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -815,9 +815,6 @@ public: bool executeStep() override { - /// Metrics will be saved in the local profile_counters. - ProfileEventsScope profile_events_scope(&profile_counters); - auto & current_level_parts = level_parts[current_level]; auto & next_level_parts = level_parts[next_level]; @@ -918,8 +915,6 @@ private: /// TODO(nikitamikhaylov): make this constant a setting static constexpr size_t max_parts_to_merge_in_one_level = 10; - - ProfileEvents::Counters profile_counters; }; @@ -1143,9 +1138,6 @@ public: bool executeStep() override { - /// Metrics will be saved in the local profile_counters. - ProfileEventsScope profile_events_scope(&profile_counters); - switch (state) { case State::NEED_PREPARE: @@ -1261,8 +1253,6 @@ private: MutationContextPtr ctx; std::unique_ptr part_merger_writer_task; - - ProfileEvents::Counters profile_counters; }; @@ -1277,9 +1267,6 @@ public: bool executeStep() override { - /// Metrics will be saved in the local profile_counters. - ProfileEventsScope profile_events_scope(&profile_counters); - switch (state) { case State::NEED_PREPARE: @@ -1471,7 +1458,6 @@ private: MergedColumnOnlyOutputStreamPtr out; std::unique_ptr part_merger_writer_task{nullptr}; - ProfileEvents::Counters profile_counters; }; From 1b715a33882b34b4962d8fca1e8e28aaddbb0c4e Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 8 Feb 2023 11:12:59 +0000 Subject: [PATCH 206/717] Allow scope override in ThreadStatus::attachProfileCountersScope --- src/Common/ProfileEventsScope.cpp | 7 ++++--- src/Common/ProfileEventsScope.h | 1 + src/Common/ThreadStatus.h | 5 +++-- src/Interpreters/ThreadStatusExt.cpp | 16 ++++++---------- 4 files changed, 14 insertions(+), 15 deletions(-) diff --git a/src/Common/ProfileEventsScope.cpp b/src/Common/ProfileEventsScope.cpp index 5b996209b76..92f75f4f5b0 100644 --- a/src/Common/ProfileEventsScope.cpp +++ b/src/Common/ProfileEventsScope.cpp @@ -7,14 +7,14 @@ namespace DB ProfileEventsScope::ProfileEventsScope() : performance_counters_holder(std::make_unique()) , performance_counters_scope(performance_counters_holder.get()) + , previous_counters_scope(CurrentThread::get().attachProfileCountersScope(performance_counters_scope)) { - CurrentThread::get().attachProfileCountersScope(performance_counters_scope); } ProfileEventsScope::ProfileEventsScope(ProfileEvents::Counters * performance_counters_scope_) : performance_counters_scope(performance_counters_scope_) + , previous_counters_scope(CurrentThread::get().attachProfileCountersScope(performance_counters_scope)) { - CurrentThread::get().attachProfileCountersScope(performance_counters_scope); } std::shared_ptr ProfileEventsScope::getSnapshot() @@ -24,7 +24,8 @@ std::shared_ptr ProfileEventsScope::getSnapsh ProfileEventsScope::~ProfileEventsScope() { - CurrentThread::get().detachProfileCountersScope(); + /// Restore previous performance counters + CurrentThread::get().attachProfileCountersScope(previous_counters_scope); } diff --git a/src/Common/ProfileEventsScope.h b/src/Common/ProfileEventsScope.h index 653a1ed55e7..0444531d02b 100644 --- a/src/Common/ProfileEventsScope.h +++ b/src/Common/ProfileEventsScope.h @@ -27,6 +27,7 @@ private: std::unique_ptr performance_counters_holder; ProfileEvents::Counters * performance_counters_scope; + ProfileEvents::Counters * previous_counters_scope; }; diff --git a/src/Common/ThreadStatus.h b/src/Common/ThreadStatus.h index 531dd669e65..20550a63312 100644 --- a/src/Common/ThreadStatus.h +++ b/src/Common/ThreadStatus.h @@ -249,8 +249,9 @@ public: /// Attaches slave thread to existing thread group void attachQuery(const ThreadGroupStatusPtr & thread_group_, bool check_detached = true); - void attachProfileCountersScope(ProfileEvents::Counters * performance_counters_scope); - void detachProfileCountersScope(); + /// Returns pointer to the current profile counters to restore them back. + /// Note: consequent call with new scope will detach previous scope. + ProfileEvents::Counters * attachProfileCountersScope(ProfileEvents::Counters * performance_counters_scope); InternalTextLogsQueuePtr getInternalTextLogsQueue() const { diff --git a/src/Interpreters/ThreadStatusExt.cpp b/src/Interpreters/ThreadStatusExt.cpp index ea2d5c4a125..a770c84a3e7 100644 --- a/src/Interpreters/ThreadStatusExt.cpp +++ b/src/Interpreters/ThreadStatusExt.cpp @@ -161,22 +161,18 @@ void ThreadStatus::attachQuery(const ThreadGroupStatusPtr & thread_group_, bool setupState(thread_group_); } -void ThreadStatus::attachProfileCountersScope(ProfileEvents::Counters * performance_counters_scope) +ProfileEvents::Counters * ThreadStatus::attachProfileCountersScope(ProfileEvents::Counters * performance_counters_scope) { + ProfileEvents::Counters * prev_counters = current_performance_counters; + if (current_performance_counters == performance_counters_scope) /// Allow to attach the same scope multiple times - return; + return prev_counters; - if (current_performance_counters != &performance_counters) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Can't attach performance counters scope to the thread, it is already attached to another scope"); - - performance_counters_scope->setParent(current_performance_counters); + performance_counters_scope->setParent(&performance_counters); current_performance_counters = performance_counters_scope; -} -void ThreadStatus::detachProfileCountersScope() -{ - current_performance_counters = &performance_counters; + return prev_counters; } void ThreadStatus::initPerformanceCounters() From 94928e7445f5b1eb2069e84d7d47fbeb35e9b429 Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 8 Feb 2023 11:16:16 +0000 Subject: [PATCH 207/717] fix 02378_part_log_profile_events --- tests/queries/0_stateless/02378_part_log_profile_events.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02378_part_log_profile_events.sql b/tests/queries/0_stateless/02378_part_log_profile_events.sql index af68360dbbb..38d3f8b4c05 100644 --- a/tests/queries/0_stateless/02378_part_log_profile_events.sql +++ b/tests/queries/0_stateless/02378_part_log_profile_events.sql @@ -39,7 +39,7 @@ SYSTEM FLUSH LOGS; SELECT if(count() == 2, 'Ok', 'Error: ' || toString(count())), - if(SUM(ProfileEvents['SelectedRows']) == 512, 'Ok', 'Error: ' || toString(SUM(ProfileEvents['SelectedRows']))), + if(SUM(ProfileEvents['MergedRows']) == 512, 'Ok', 'Error: ' || toString(SUM(ProfileEvents['MergedRows']))), if(SUM(ProfileEvents['FileOpen']) > 1, 'Ok', 'Error: ' || toString(SUM(ProfileEvents['FileOpen']))) FROM system.part_log WHERE event_time > now() - INTERVAL 10 MINUTE From 850601846927f7968de706232841b8f0d9cde5b7 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Tue, 14 Feb 2023 15:20:39 +0100 Subject: [PATCH 208/717] A test for bug found by fuzzer --- .../02559_multiple_read_steps_in_prewhere_fuzz.reference | 1 + .../02559_multiple_read_steps_in_prewhere_fuzz.sql | 7 +++++++ 2 files changed, 8 insertions(+) create mode 100644 tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere_fuzz.reference create mode 100644 tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere_fuzz.sql diff --git a/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere_fuzz.reference b/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere_fuzz.reference new file mode 100644 index 00000000000..573541ac970 --- /dev/null +++ b/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere_fuzz.reference @@ -0,0 +1 @@ +0 diff --git a/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere_fuzz.sql b/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere_fuzz.sql new file mode 100644 index 00000000000..20f159cbe5d --- /dev/null +++ b/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere_fuzz.sql @@ -0,0 +1,7 @@ +CREATE TABLE test_02559__fuzz_20(`id1` Int16, `id2` Decimal(18, 14)) ENGINE = MergeTree ORDER BY id1; + +INSERT INTO test_02559__fuzz_20 SELECT number, number FROM numbers(10); + +SET enable_multiple_prewhere_read_steps=true, move_all_conditions_to_prewhere=true; + +SELECT count() FROM test_02559__fuzz_20 PREWHERE (id2 >= 104) AND ((-9223372036854775808 OR (inf OR -2147483649 OR NULL) OR NULL) OR 1 OR ignore(ignore(id1) OR NULL, id1)) WHERE ignore(id1) = 0; From 9c25706711ca4872c68d9e2f4fc8b89e116a6e94 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 14 Feb 2023 14:29:25 +0000 Subject: [PATCH 209/717] Update settings history --- src/Core/Settings.h | 2 +- src/Core/SettingsChangesHistory.h | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 631bf18bc8a..73c3a1092e5 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -596,7 +596,7 @@ class IColumn; M(Bool, query_plan_read_in_order, true, "Use query plan for read-in-order optimisation", 0) \ M(Bool, query_plan_aggregation_in_order, true, "Use query plan for aggregation-in-order optimisation", 0) \ M(Bool, query_plan_remove_redundant_sorting, true, "Remove redundant sorting in query plan. For example, sorting steps related to ORDER BY clauses in subqueries", 0) \ - M(Bool, query_plan_remove_redundant_distinct, true, "Remove redundant DISTINCT clauses", 0) \ + M(Bool, query_plan_remove_redundant_distinct, true, "Remove redundant Distinct step in query plan", 0) \ M(UInt64, regexp_max_matches_per_row, 1000, "Max matches of any single regexp per row, used to safeguard 'extractAllGroupsHorizontal' against consuming too much memory with greedy RE.", 0) \ \ M(UInt64, limit, 0, "Limit on read rows from the most 'end' result for select query, default 0 means no limit length", 0) \ diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index 4adb19c68f7..5684e4f3114 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -81,7 +81,9 @@ namespace SettingsChangesHistory static std::map settings_changes_history = { {"23.2", {{"output_format_parquet_fixed_string_as_fixed_byte_array", false, true, "Use Parquet FIXED_LENGTH_BYTE_ARRAY type for FixedString by default"}, - {"output_format_arrow_fixed_string_as_fixed_byte_array", false, true, "Use Arrow FIXED_SIZE_BINARY type for FixedString by default"}}}, + {"output_format_arrow_fixed_string_as_fixed_byte_array", false, true, "Use Arrow FIXED_SIZE_BINARY type for FixedString by default"}, + {"query_plan_remove_redundant_distinct", false, true, "Remove redundant Distinct step in query plan"}, + {"optimize_duplicate_order_by_and_distinct", true, false, "Remove duplicate ORDER BY and DISTINCT if it's possible"}}}, {"23.1", {{"input_format_json_read_objects_as_strings", 0, 1, "Enable reading nested json objects as strings while object type is experimental"}, {"input_format_json_defaults_for_missing_elements_in_named_tuple", false, true, "Allow missing elements in JSON objects while reading named tuples by default"}, {"input_format_csv_detect_header", false, true, "Detect header in CSV format by default"}, From fd2ebd6d7714f38c9fcc16ad2ab400e2b97c5370 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 14 Feb 2023 14:30:33 +0000 Subject: [PATCH 210/717] Add tests for GROUP BY WITH ROLLUP/CUBE --- ...e_redundant_distinct_distributed.reference | 0 ..._remove_redundant_distinct_distributed.sql | 23 +++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 tests/queries/0_stateless/02500_remove_redundant_distinct_distributed.reference create mode 100644 tests/queries/0_stateless/02500_remove_redundant_distinct_distributed.sql diff --git a/tests/queries/0_stateless/02500_remove_redundant_distinct_distributed.reference b/tests/queries/0_stateless/02500_remove_redundant_distinct_distributed.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02500_remove_redundant_distinct_distributed.sql b/tests/queries/0_stateless/02500_remove_redundant_distinct_distributed.sql new file mode 100644 index 00000000000..350779fa459 --- /dev/null +++ b/tests/queries/0_stateless/02500_remove_redundant_distinct_distributed.sql @@ -0,0 +1,23 @@ +-- Tags: distributed + +SET allow_experimental_analyzer=0; +set optimize_duplicate_order_by_and_distinct = 0; + +SET distributed_group_by_no_merge = 0; +SELECT DISTINCT number +FROM +( + SELECT DISTINCT number + FROM remote('127.0.0.{1,2}', system.numbers) + LIMIT 1 + SETTINGS distributed_group_by_no_merge = 1 +); + +SET distributed_group_by_no_merge = 1; +SELECT DISTINCT number +FROM +( + SELECT DISTINCT number + FROM remote('127.0.0.{1,2}', system.numbers) + LIMIT 1 +); From 7ee94d6a900b71c6af94b6e74c2ca131b8d5c088 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Tue, 14 Feb 2023 15:34:04 +0100 Subject: [PATCH 211/717] Fix for 'Result column is not empty' bug found by fuzzer --- src/Interpreters/ExpressionActions.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Interpreters/ExpressionActions.cpp b/src/Interpreters/ExpressionActions.cpp index 97555feb426..9931ae97286 100644 --- a/src/Interpreters/ExpressionActions.cpp +++ b/src/Interpreters/ExpressionActions.cpp @@ -584,6 +584,12 @@ static void executeAction(const ExpressionActions::Action & action, ExecutionCon { /// Do not execute function if it's result is already known. res_column.column = action.node->column->cloneResized(num_rows); + /// But still need to remove unused arguments. + for (const auto & argument : action.arguments) + { + if (!argument.needed_later) + columns[argument.pos] = {}; + } break; } From d4518281bbefe830c2d964f4dc35133c536d5850 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Tue, 14 Feb 2023 17:03:04 +0000 Subject: [PATCH 212/717] test more headers --- .../02564_query_id_header.reference | 27 +++++++++++++++---- .../0_stateless/02564_query_id_header.sh | 27 +++++++++++++++---- 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/tests/queries/0_stateless/02564_query_id_header.reference b/tests/queries/0_stateless/02564_query_id_header.reference index 655f9fa9eaa..413e8929f36 100644 --- a/tests/queries/0_stateless/02564_query_id_header.reference +++ b/tests/queries/0_stateless/02564_query_id_header.reference @@ -1,5 +1,22 @@ -X-ClickHouse-Query-Id -X-ClickHouse-Query-Id -X-ClickHouse-Query-Id -X-ClickHouse-Query-Id -X-ClickHouse-Query-Id +CREATE TABLE t_query_id_header (a UInt64) ENGINE = Memory +< Content-Type: text/plain; charset=UTF-8 +< X-ClickHouse-Query-Id: query_id +< X-ClickHouse-Timezone: timezone +INSERT INTO t_query_id_header VALUES (1) +< Content-Type: text/plain; charset=UTF-8 +< X-ClickHouse-Query-Id: query_id +< X-ClickHouse-Timezone: timezone +EXISTS TABLE t_query_id_header +< Content-Type: text/tab-separated-values; charset=UTF-8 +< X-ClickHouse-Format: TabSeparated +< X-ClickHouse-Query-Id: query_id +< X-ClickHouse-Timezone: timezone +SELECT * FROM t_query_id_header +< Content-Type: text/tab-separated-values; charset=UTF-8 +< X-ClickHouse-Format: TabSeparated +< X-ClickHouse-Query-Id: query_id +< X-ClickHouse-Timezone: timezone +DROP TABLE t_query_id_header +< Content-Type: text/plain; charset=UTF-8 +< X-ClickHouse-Query-Id: query_id +< X-ClickHouse-Timezone: timezone diff --git a/tests/queries/0_stateless/02564_query_id_header.sh b/tests/queries/0_stateless/02564_query_id_header.sh index 440182cd243..67ddbcfcc46 100755 --- a/tests/queries/0_stateless/02564_query_id_header.sh +++ b/tests/queries/0_stateless/02564_query_id_header.sh @@ -4,10 +4,27 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +CLICKHOUSE_TIMEZONE_ESCAPED=$($CLICKHOUSE_CLIENT --query="SELECT timezone()" | sed 's/[]\/$*.^+:()[]/\\&/g') + +function run_and_check_headers() +{ + query=$1 + query_id="${CLICKHOUSE_DATABASE}_${RANDOM}" + + echo "$query" + + ${CLICKHOUSE_CURL} -sS -v "${CLICKHOUSE_URL}&query_id=$query_id" -d "$1" 2>&1 \ + | grep -e "< X-ClickHouse-Query-Id" -e "< X-ClickHouse-Timezone" -e "< X-ClickHouse-Format" -e "< Content-Type" \ + | sed "s/$CLICKHOUSE_TIMEZONE_ESCAPED/timezone/" \ + | sed "s/$query_id/query_id/" \ + | sed "s/\r$//" \ + | sort +} + ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS t_query_id_header" -${CLICKHOUSE_CURL} -sS -v "${CLICKHOUSE_URL}" -d "CREATE TABLE t_query_id_header (a UInt64) ENGINE = Memory" 2>&1 | grep -o "X-ClickHouse-Query-Id" -${CLICKHOUSE_CURL} -sS -v "${CLICKHOUSE_URL}" -d "INSERT INTO t_query_id_header VALUES (1)" 2>&1 | grep -o "X-ClickHouse-Query-Id" -${CLICKHOUSE_CURL} -sS -v "${CLICKHOUSE_URL}" -d "EXISTS TABLE t_query_id_header" 2>&1 | grep -o "X-ClickHouse-Query-Id" -${CLICKHOUSE_CURL} -sS -v "${CLICKHOUSE_URL}" -d "SELECT * FROM t_query_id_header" 2>&1 | grep -o "X-ClickHouse-Query-Id" -${CLICKHOUSE_CURL} -sS -v "${CLICKHOUSE_URL}" -d "DROP TABLE t_query_id_header" 2>&1 | grep -o "X-ClickHouse-Query-Id" +run_and_check_headers "CREATE TABLE t_query_id_header (a UInt64) ENGINE = Memory" +run_and_check_headers "INSERT INTO t_query_id_header VALUES (1)" +run_and_check_headers "EXISTS TABLE t_query_id_header" +run_and_check_headers "SELECT * FROM t_query_id_header" +run_and_check_headers "DROP TABLE t_query_id_header" From c9b8e79684f6c2ea38437661a2a145ca90e2ff66 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 14 Feb 2023 19:45:57 +0000 Subject: [PATCH 213/717] Fix tests --- .../02500_remove_redundant_distinct.reference | 130 ++++++++++++++++++ .../02500_remove_redundant_distinct.sh | 72 ++++++++++ ...e_redundant_distinct_distributed.reference | 0 ..._remove_redundant_distinct_distributed.sql | 23 ---- 4 files changed, 202 insertions(+), 23 deletions(-) delete mode 100644 tests/queries/0_stateless/02500_remove_redundant_distinct_distributed.reference delete mode 100644 tests/queries/0_stateless/02500_remove_redundant_distinct_distributed.sql diff --git a/tests/queries/0_stateless/02500_remove_redundant_distinct.reference b/tests/queries/0_stateless/02500_remove_redundant_distinct.reference index ccce269f694..91f89b8dd9a 100644 --- a/tests/queries/0_stateless/02500_remove_redundant_distinct.reference +++ b/tests/queries/0_stateless/02500_remove_redundant_distinct.reference @@ -269,3 +269,133 @@ Expression (Project names) ReadFromStorage (SystemNumbers) -- execute 12 +-- GROUP BY WITH ROLLUP before DISTINCT with on different columns => do _not_ remove DISTINCT +-- query +SELECT DISTINCT c +FROM +( + SELECT + a, + sum(b) AS c + FROM + ( + SELECT + x.number AS a, + y.number AS b + FROM numbers(3) AS x, numbers(3, 3) AS y + ) + GROUP BY a WITH ROLLUP +) +-- explain +Expression (Project names) + Distinct (DISTINCT) + Distinct (Preliminary DISTINCT) + Expression ((Projection + (Change column names to column identifiers + (Project names + Projection)))) + Rollup + Aggregating + Expression ((Before GROUP BY + (Change column names to column identifiers + (Project names + (Projection + DROP unused columns after JOIN))))) + Join (JOIN FillRightFirst) + Expression (Change column names to column identifiers) + ReadFromStorage (SystemNumbers) + Expression (Change column names to column identifiers) + ReadFromStorage (SystemNumbers) +-- execute +12 +36 +-- GROUP BY WITH ROLLUP before DISTINCT with on the same columns => remove DISTINCT +-- query +SELECT DISTINCT a +FROM +( + SELECT + a, + sum(b) AS c + FROM + ( + SELECT + x.number AS a, + y.number AS b + FROM numbers(3) AS x, numbers(3, 3) AS y + ) + GROUP BY a WITH ROLLUP +) +-- explain +Expression ((Project names + (Projection + (Change column names to column identifiers + (Project names + Projection))))) + Rollup + Aggregating + Expression ((Before GROUP BY + (Change column names to column identifiers + (Project names + (Projection + DROP unused columns after JOIN))))) + Join (JOIN FillRightFirst) + Expression (Change column names to column identifiers) + ReadFromStorage (SystemNumbers) + Expression (Change column names to column identifiers) + ReadFromStorage (SystemNumbers) +-- execute +0 +2 +1 +0 +-- GROUP BY WITH CUBE before DISTINCT with on different columns => do _not_ remove DISTINCT +-- query +SELECT DISTINCT c +FROM +( + SELECT + a, + sum(b) AS c + FROM + ( + SELECT + x.number AS a, + y.number AS b + FROM numbers(3) AS x, numbers(3, 3) AS y + ) + GROUP BY a WITH CUBE +) +-- explain +Expression (Project names) + Distinct (DISTINCT) + Distinct (Preliminary DISTINCT) + Expression ((Projection + (Change column names to column identifiers + (Project names + Projection)))) + Cube + Aggregating + Expression ((Before GROUP BY + (Change column names to column identifiers + (Project names + (Projection + DROP unused columns after JOIN))))) + Join (JOIN FillRightFirst) + Expression (Change column names to column identifiers) + ReadFromStorage (SystemNumbers) + Expression (Change column names to column identifiers) + ReadFromStorage (SystemNumbers) +-- execute +12 +36 +-- GROUP BY WITH CUBE before DISTINCT with on the same columns => remove DISTINCT +-- query +SELECT DISTINCT a +FROM +( + SELECT + a, + sum(b) AS c + FROM + ( + SELECT + x.number AS a, + y.number AS b + FROM numbers(3) AS x, numbers(3, 3) AS y + ) + GROUP BY a WITH CUBE +) +-- explain +Expression ((Project names + (Projection + (Change column names to column identifiers + (Project names + Projection))))) + Cube + Aggregating + Expression ((Before GROUP BY + (Change column names to column identifiers + (Project names + (Projection + DROP unused columns after JOIN))))) + Join (JOIN FillRightFirst) + Expression (Change column names to column identifiers) + ReadFromStorage (SystemNumbers) + Expression (Change column names to column identifiers) + ReadFromStorage (SystemNumbers) +-- execute +0 +2 +1 +0 diff --git a/tests/queries/0_stateless/02500_remove_redundant_distinct.sh b/tests/queries/0_stateless/02500_remove_redundant_distinct.sh index 9faf32abc16..db2d3588844 100755 --- a/tests/queries/0_stateless/02500_remove_redundant_distinct.sh +++ b/tests/queries/0_stateless/02500_remove_redundant_distinct.sh @@ -148,3 +148,75 @@ FROM GROUP BY a )" run_query "$query" + +echo "-- GROUP BY WITH ROLLUP before DISTINCT with on different columns => do _not_ remove DISTINCT" +query="SELECT DISTINCT c +FROM +( + SELECT + a, + sum(b) AS c + FROM + ( + SELECT + x.number AS a, + y.number AS b + FROM numbers(3) AS x, numbers(3, 3) AS y + ) + GROUP BY a WITH ROLLUP +)" +run_query "$query" + +echo "-- GROUP BY WITH ROLLUP before DISTINCT with on the same columns => remove DISTINCT" +query="SELECT DISTINCT a +FROM +( + SELECT + a, + sum(b) AS c + FROM + ( + SELECT + x.number AS a, + y.number AS b + FROM numbers(3) AS x, numbers(3, 3) AS y + ) + GROUP BY a WITH ROLLUP +)" +run_query "$query" + +echo "-- GROUP BY WITH CUBE before DISTINCT with on different columns => do _not_ remove DISTINCT" +query="SELECT DISTINCT c +FROM +( + SELECT + a, + sum(b) AS c + FROM + ( + SELECT + x.number AS a, + y.number AS b + FROM numbers(3) AS x, numbers(3, 3) AS y + ) + GROUP BY a WITH CUBE +)" +run_query "$query" + +echo "-- GROUP BY WITH CUBE before DISTINCT with on the same columns => remove DISTINCT" +query="SELECT DISTINCT a +FROM +( + SELECT + a, + sum(b) AS c + FROM + ( + SELECT + x.number AS a, + y.number AS b + FROM numbers(3) AS x, numbers(3, 3) AS y + ) + GROUP BY a WITH CUBE +)" +run_query "$query" diff --git a/tests/queries/0_stateless/02500_remove_redundant_distinct_distributed.reference b/tests/queries/0_stateless/02500_remove_redundant_distinct_distributed.reference deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/queries/0_stateless/02500_remove_redundant_distinct_distributed.sql b/tests/queries/0_stateless/02500_remove_redundant_distinct_distributed.sql deleted file mode 100644 index 350779fa459..00000000000 --- a/tests/queries/0_stateless/02500_remove_redundant_distinct_distributed.sql +++ /dev/null @@ -1,23 +0,0 @@ --- Tags: distributed - -SET allow_experimental_analyzer=0; -set optimize_duplicate_order_by_and_distinct = 0; - -SET distributed_group_by_no_merge = 0; -SELECT DISTINCT number -FROM -( - SELECT DISTINCT number - FROM remote('127.0.0.{1,2}', system.numbers) - LIMIT 1 - SETTINGS distributed_group_by_no_merge = 1 -); - -SET distributed_group_by_no_merge = 1; -SELECT DISTINCT number -FROM -( - SELECT DISTINCT number - FROM remote('127.0.0.{1,2}', system.numbers) - LIMIT 1 -); From d1157a70595d448686bf964a09300359944afaa1 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 14 Feb 2023 20:36:12 +0000 Subject: [PATCH 214/717] Disable new analyzer in tests --- .../02500_remove_redundant_distinct.reference | 182 +++++++++--------- .../02500_remove_redundant_distinct.sh | 8 +- 2 files changed, 94 insertions(+), 96 deletions(-) diff --git a/tests/queries/0_stateless/02500_remove_redundant_distinct.reference b/tests/queries/0_stateless/02500_remove_redundant_distinct.reference index 91f89b8dd9a..574340b0c44 100644 --- a/tests/queries/0_stateless/02500_remove_redundant_distinct.reference +++ b/tests/queries/0_stateless/02500_remove_redundant_distinct.reference @@ -1,14 +1,14 @@ -- Disabled query_plan_remove_redundant_distinct -Expression (Project names) - Distinct (DISTINCT) +Expression (Projection) + Distinct Distinct (Preliminary DISTINCT) - Expression ((Projection + (Change column names to column identifiers + Project names))) - Distinct (DISTINCT) + Expression ((Before ORDER BY + Projection)) + Distinct Distinct (Preliminary DISTINCT) - Expression ((Projection + (Change column names to column identifiers + Project names))) - Distinct (DISTINCT) + Expression ((Before ORDER BY + Projection)) + Distinct Distinct (Preliminary DISTINCT) - Expression ((Projection + Change column names to column identifiers)) + Expression (Before ORDER BY) ReadFromStorage (SystemNumbers) -- Enabled query_plan_remove_redundant_distinct -- DISTINCT is only in most inner subquery @@ -24,10 +24,10 @@ FROM ) ) -- explain -Expression ((Project names + (Projection + (Change column names to column identifiers + (Project names + (Projection + (Change column names to column identifiers + Project names))))))) - Distinct (DISTINCT) +Expression ((Projection + (Before ORDER BY + (Projection + (Before ORDER BY + Projection))))) + Distinct Distinct (Preliminary DISTINCT) - Expression ((Projection + Change column names to column identifiers)) + Expression (Before ORDER BY) ReadFromStorage (SystemNumbers) -- execute 0 @@ -43,22 +43,21 @@ SELECT DISTINCT number FROM ) ORDER BY number -- explain -Expression (Project names) - Distinct (DISTINCT) +Expression (Projection) + Distinct Sorting (Sorting for ORDER BY) - Expression (Before ORDER BY) - Distinct (Preliminary DISTINCT) - Union - Expression ((Projection + (Change column names to column identifiers + Project names))) - Distinct (DISTINCT) - Distinct (Preliminary DISTINCT) - Expression ((Projection + Change column names to column identifiers)) - ReadFromStorage (SystemNumbers) - Expression (( + ( + Project names))) - Distinct (DISTINCT) - Distinct (Preliminary DISTINCT) - Expression ((Projection + Change column names to column identifiers)) - ReadFromStorage (SystemNumbers) + Distinct (Preliminary DISTINCT) + Union + Expression ((Before ORDER BY + Projection)) + Distinct + Distinct (Preliminary DISTINCT) + Expression (Before ORDER BY) + ReadFromStorage (SystemNumbers) + Expression (( + Projection)) + Distinct + Distinct (Preliminary DISTINCT) + Expression (Before ORDER BY) + ReadFromStorage (SystemNumbers) -- execute 0 1 @@ -69,26 +68,26 @@ FROM ( SELECT DISTINCT number AS n FROM numbers(2) -), +) as x, ( SELECT DISTINCT number AS n FROM numbers(2) -) SETTINGS joined_subquery_requires_alias=0 +) as y -- explain -Expression (Project names) - Distinct (DISTINCT) +Expression (Projection) + Distinct Distinct (Preliminary DISTINCT) - Expression ((Projection + DROP unused columns after JOIN)) + Expression (Before ORDER BY) Join (JOIN FillRightFirst) - Expression ((Change column names to column identifiers + Project names)) - Distinct (DISTINCT) + Expression ((Before JOIN + Projection)) + Distinct Distinct (Preliminary DISTINCT) - Expression ((Projection + Change column names to column identifiers)) + Expression (Before ORDER BY) ReadFromStorage (SystemNumbers) - Expression ((Change column names to column identifiers + Project names)) - Distinct (DISTINCT) + Expression ((Joined actions + (Rename joined columns + Projection))) + Distinct Distinct (Preliminary DISTINCT) - Expression ((Projection + Change column names to column identifiers)) + Expression (Before ORDER BY) ReadFromStorage (SystemNumbers) -- execute 0 0 @@ -108,10 +107,10 @@ FROM ) ) -- explain -Expression ((Project names + (Projection + (Change column names to column identifiers + (Project names + (Projection + (Change column names to column identifiers + Project names))))))) - Distinct (DISTINCT) +Expression ((Projection + (Before ORDER BY + (Projection + (Before ORDER BY + Projection))))) + Distinct Distinct (Preliminary DISTINCT) - Expression ((Projection + Change column names to column identifiers)) + Expression (Before ORDER BY) ReadFromStorage (SystemNumbers) -- execute 0 0 @@ -130,10 +129,10 @@ FROM ) ) -- explain -Expression ((Project names + (Projection + (Change column names to column identifiers + (Project names + (Projection + (Change column names to column identifiers + Project names))))))) - Distinct (DISTINCT) +Expression ((Projection + (Before ORDER BY + (Projection + (Before ORDER BY + Projection))))) + Distinct Distinct (Preliminary DISTINCT) - Expression ((Projection + Change column names to column identifiers)) + Expression (Before ORDER BY) ReadFromStorage (SystemNumbers) -- execute 2 0 0 @@ -149,15 +148,15 @@ FROM ) AS words ARRAY JOIN [0, 1] AS arr -- explain -Expression (Project names) - Distinct (DISTINCT) +Expression (Projection) + Distinct Distinct (Preliminary DISTINCT) - Expression (Projection) + Expression (Before ORDER BY) ArrayJoin (ARRAY JOIN) - Expression ((ARRAY JOIN actions + (Change column names to column identifiers + Project names))) - Distinct (DISTINCT) + Expression ((Before ARRAY JOIN + Projection)) + Distinct Distinct (Preliminary DISTINCT) - Expression ((Projection + Change column names to column identifiers)) + Expression (Before ORDER BY) ReadFromStorage (Values) -- execute Hello @@ -173,17 +172,16 @@ FROM ORDER BY id ASC WITH FILL ) -- explain -Expression (Project names) - Distinct (DISTINCT) +Expression (Projection) + Distinct Distinct (Preliminary DISTINCT) - Expression ((Projection + (Change column names to column identifiers + Project names))) + Expression ((Before ORDER BY + Projection)) Filling - Distinct (DISTINCT) + Distinct Sorting (Sorting for ORDER BY) - Expression (Before ORDER BY) - Distinct (Preliminary DISTINCT) - Expression ((Projection + Change column names to column identifiers)) - ReadFromStorage (Values) + Distinct (Preliminary DISTINCT) + Expression (Before ORDER BY) + ReadFromStorage (Values) -- execute 0 1 @@ -197,14 +195,14 @@ FROM ) WHERE arrayJoin(cities) IN ['Berlin', 'Bensheim'] -- explain -Expression (Project names) - Distinct (DISTINCT) +Expression (( + Projection)) + Distinct Distinct (Preliminary DISTINCT) - Expression (Projection) - Filter ((WHERE + (Change column names to column identifiers + Project names))) - Distinct (DISTINCT) + Expression (Before ORDER BY) + Filter ((WHERE + Projection)) + Distinct Distinct (Preliminary DISTINCT) - Expression ((Projection + Change column names to column identifiers)) + Expression (Before ORDER BY) ReadFromStorage (SystemOne) -- execute ['Istanbul','Berlin','Bensheim'] @@ -226,13 +224,13 @@ FROM GROUP BY a ) -- explain -Expression ((Project names + (Projection + (Change column names to column identifiers + (Project names + Projection))))) +Expression ((Projection + (Before ORDER BY + (Projection + Before ORDER BY)))) Aggregating - Expression ((Before GROUP BY + (Change column names to column identifiers + (Project names + (Projection + DROP unused columns after JOIN))))) + Expression ((Before GROUP BY + (Projection + Before ORDER BY))) Join (JOIN FillRightFirst) - Expression (Change column names to column identifiers) + Expression (Before JOIN) ReadFromStorage (SystemNumbers) - Expression (Change column names to column identifiers) + Expression ((Joined actions + (Rename joined columns + (Projection + Before ORDER BY)))) ReadFromStorage (SystemNumbers) -- execute 0 @@ -256,16 +254,16 @@ FROM GROUP BY a ) -- explain -Expression (Project names) - Distinct (DISTINCT) +Expression (Projection) + Distinct Distinct (Preliminary DISTINCT) - Expression ((Projection + (Change column names to column identifiers + (Project names + Projection)))) + Expression ((Before ORDER BY + (Projection + Before ORDER BY))) Aggregating - Expression ((Before GROUP BY + (Change column names to column identifiers + (Project names + (Projection + DROP unused columns after JOIN))))) + Expression ((Before GROUP BY + (Projection + Before ORDER BY))) Join (JOIN FillRightFirst) - Expression (Change column names to column identifiers) + Expression (Before JOIN) ReadFromStorage (SystemNumbers) - Expression (Change column names to column identifiers) + Expression ((Joined actions + (Rename joined columns + (Projection + Before ORDER BY)))) ReadFromStorage (SystemNumbers) -- execute 12 @@ -287,17 +285,17 @@ FROM GROUP BY a WITH ROLLUP ) -- explain -Expression (Project names) - Distinct (DISTINCT) +Expression (Projection) + Distinct Distinct (Preliminary DISTINCT) - Expression ((Projection + (Change column names to column identifiers + (Project names + Projection)))) + Expression ((Before ORDER BY + (Projection + Before ORDER BY))) Rollup Aggregating - Expression ((Before GROUP BY + (Change column names to column identifiers + (Project names + (Projection + DROP unused columns after JOIN))))) + Expression ((Before GROUP BY + (Projection + Before ORDER BY))) Join (JOIN FillRightFirst) - Expression (Change column names to column identifiers) + Expression (Before JOIN) ReadFromStorage (SystemNumbers) - Expression (Change column names to column identifiers) + Expression ((Joined actions + (Rename joined columns + (Projection + Before ORDER BY)))) ReadFromStorage (SystemNumbers) -- execute 12 @@ -320,14 +318,14 @@ FROM GROUP BY a WITH ROLLUP ) -- explain -Expression ((Project names + (Projection + (Change column names to column identifiers + (Project names + Projection))))) +Expression ((Projection + (Before ORDER BY + (Projection + Before ORDER BY)))) Rollup Aggregating - Expression ((Before GROUP BY + (Change column names to column identifiers + (Project names + (Projection + DROP unused columns after JOIN))))) + Expression ((Before GROUP BY + (Projection + Before ORDER BY))) Join (JOIN FillRightFirst) - Expression (Change column names to column identifiers) + Expression (Before JOIN) ReadFromStorage (SystemNumbers) - Expression (Change column names to column identifiers) + Expression ((Joined actions + (Rename joined columns + (Projection + Before ORDER BY)))) ReadFromStorage (SystemNumbers) -- execute 0 @@ -352,17 +350,17 @@ FROM GROUP BY a WITH CUBE ) -- explain -Expression (Project names) - Distinct (DISTINCT) +Expression (Projection) + Distinct Distinct (Preliminary DISTINCT) - Expression ((Projection + (Change column names to column identifiers + (Project names + Projection)))) + Expression ((Before ORDER BY + (Projection + Before ORDER BY))) Cube Aggregating - Expression ((Before GROUP BY + (Change column names to column identifiers + (Project names + (Projection + DROP unused columns after JOIN))))) + Expression ((Before GROUP BY + (Projection + Before ORDER BY))) Join (JOIN FillRightFirst) - Expression (Change column names to column identifiers) + Expression (Before JOIN) ReadFromStorage (SystemNumbers) - Expression (Change column names to column identifiers) + Expression ((Joined actions + (Rename joined columns + (Projection + Before ORDER BY)))) ReadFromStorage (SystemNumbers) -- execute 12 @@ -385,14 +383,14 @@ FROM GROUP BY a WITH CUBE ) -- explain -Expression ((Project names + (Projection + (Change column names to column identifiers + (Project names + Projection))))) +Expression ((Projection + (Before ORDER BY + (Projection + Before ORDER BY)))) Cube Aggregating - Expression ((Before GROUP BY + (Change column names to column identifiers + (Project names + (Projection + DROP unused columns after JOIN))))) + Expression ((Before GROUP BY + (Projection + Before ORDER BY))) Join (JOIN FillRightFirst) - Expression (Change column names to column identifiers) + Expression (Before JOIN) ReadFromStorage (SystemNumbers) - Expression (Change column names to column identifiers) + Expression ((Joined actions + (Rename joined columns + (Projection + Before ORDER BY)))) ReadFromStorage (SystemNumbers) -- execute 0 diff --git a/tests/queries/0_stateless/02500_remove_redundant_distinct.sh b/tests/queries/0_stateless/02500_remove_redundant_distinct.sh index db2d3588844..b7b764f82c4 100755 --- a/tests/queries/0_stateless/02500_remove_redundant_distinct.sh +++ b/tests/queries/0_stateless/02500_remove_redundant_distinct.sh @@ -5,8 +5,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh OPTIMIZATION_SETTING="query_plan_remove_redundant_distinct" -DISABLE_OPTIMIZATION="SET allow_experimental_analyzer=1;SET $OPTIMIZATION_SETTING=0;SET optimize_duplicate_order_by_and_distinct=0" -ENABLE_OPTIMIZATION="SET allow_experimental_analyzer=1;SET $OPTIMIZATION_SETTING=1;SET optimize_duplicate_order_by_and_distinct=0" +DISABLE_OPTIMIZATION="SET $OPTIMIZATION_SETTING=0;SET optimize_duplicate_order_by_and_distinct=0" +ENABLE_OPTIMIZATION="SET $OPTIMIZATION_SETTING=1;SET optimize_duplicate_order_by_and_distinct=0" echo "-- Disabled $OPTIMIZATION_SETTING" query="SELECT DISTINCT * @@ -51,11 +51,11 @@ FROM ( SELECT DISTINCT number AS n FROM numbers(2) -), +) as x, ( SELECT DISTINCT number AS n FROM numbers(2) -) SETTINGS joined_subquery_requires_alias=0" +) as y" run_query "$query" echo "-- DISTINCT duplicates with several columns" From 425b99afcade915d4d575bf08bcdd8233bd120e0 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 14 Feb 2023 21:14:41 +0000 Subject: [PATCH 215/717] GROUP BY WITH TOTALS --- .../Optimizations/removeRedundantDistinct.cpp | 42 ++++++++++-- .../02500_remove_redundant_distinct.reference | 67 +++++++++++++++++++ .../02500_remove_redundant_distinct.sh | 36 ++++++++++ 3 files changed, 138 insertions(+), 7 deletions(-) diff --git a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp index 25afd1cf2d4..0c8a9de4b45 100644 --- a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp +++ b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp @@ -26,18 +26,28 @@ namespace DB::QueryPlanOptimizations namespace { constexpr bool debug_logging_enabled = true; + + template + void logDebug(String key, const T & value, const char * separator = " : ") + { + if constexpr (debug_logging_enabled) + { + WriteBufferFromOwnString ss; + if constexpr (std::is_pointer_v) + ss << *value; + else + ss << value; + + LOG_DEBUG(&Poco::Logger::get("redundantDistinct"), "{}{}{}", key, separator, ss.str()); + } + } + void logActionsDAG(const String & prefix, const ActionsDAGPtr & actions) { if constexpr (debug_logging_enabled) LOG_DEBUG(&Poco::Logger::get("redundantDistinct"), "{} :\n{}", prefix, actions->dumpDAG()); } - void logDebug(String key, String value) - { - if constexpr (debug_logging_enabled) - LOG_DEBUG(&Poco::Logger::get("redundantDistinct"), "{} : {}", key, value); - } - using DistinctColumns = std::set; DistinctColumns getDistinctColumns(const DistinctStep * distinct) { @@ -59,14 +69,17 @@ namespace const ActionsDAG::Node * output_alias = nullptr; for (const auto * node : actions->getOutputs()) { - if (node->result_name == output_name && node->type == ActionsDAG::ActionType::ALIAS) + if (node->result_name == output_name) { output_alias = node; break; } } if (!output_alias) + { + logDebug("getOriginalNodeForOutputAlias: no output alias found", output_name); return nullptr; + } /// find original(non alias) node it refers to const ActionsDAG::Node * node = output_alias; @@ -84,18 +97,29 @@ namespace bool compareAggregationKeysWithDistinctColumns( const Names & aggregation_keys, const DistinctColumns & distinct_columns, const ActionsDAGPtr & path_actions) { + logDebug("aggregation_keys", aggregation_keys); + logDebug("aggregation_keys size", aggregation_keys.size()); + logDebug("distinct_columns size", distinct_columns.size()); if (aggregation_keys.size() != distinct_columns.size()) return false; /// compare columns of two DISTINCTs for (const auto & column : distinct_columns) { + logDebug("distinct column name", column); const auto * alias_node = getOriginalNodeForOutputAlias(path_actions, String(column)); if (!alias_node) + { + logDebug("original name for alias is not found for", column); return false; + } + logDebug("alias result name", alias_node->result_name); if (std::find(cbegin(aggregation_keys), cend(aggregation_keys), alias_node->result_name) == aggregation_keys.cend()) + { + logDebug("alias result name is not found in aggregation keys", alias_node->result_name); return false; + } } return true; } @@ -187,6 +211,10 @@ namespace if (const auto * rollup_step = typeid_cast(aggregation_before_distinct); rollup_step) return compareAggregationKeysWithDistinctColumns(rollup_step->getParams().keys, distinct_columns, actions); + + if (const auto * totals_step = typeid_cast(aggregation_before_distinct); totals_step) + return compareAggregationKeysWithDistinctColumns( + totals_step->getOutputStream().header.getNames(), distinct_columns, actions); } return false; diff --git a/tests/queries/0_stateless/02500_remove_redundant_distinct.reference b/tests/queries/0_stateless/02500_remove_redundant_distinct.reference index 574340b0c44..32ddab4886c 100644 --- a/tests/queries/0_stateless/02500_remove_redundant_distinct.reference +++ b/tests/queries/0_stateless/02500_remove_redundant_distinct.reference @@ -397,3 +397,70 @@ Expression ((Projection + (Before ORDER BY + (Projection + Before ORDER BY)))) 2 1 0 +-- GROUP BY WITH TOTALS before DISTINCT with on different columns => do _not_ remove DISTINCT +-- query +SELECT DISTINCT c +FROM +( + SELECT + a, + sum(b) AS c + FROM + ( + SELECT + x.number AS a, + y.number AS b + FROM numbers(3) AS x, numbers(3, 3) AS y + ) + GROUP BY a WITH TOTALS +) +-- explain +Expression (Projection) + Distinct + Distinct (Preliminary DISTINCT) + Expression ((Before ORDER BY + (Projection + Before ORDER BY))) + TotalsHaving + Aggregating + Expression ((Before GROUP BY + (Projection + Before ORDER BY))) + Join (JOIN FillRightFirst) + Expression (Before JOIN) + ReadFromStorage (SystemNumbers) + Expression ((Joined actions + (Rename joined columns + (Projection + Before ORDER BY)))) + ReadFromStorage (SystemNumbers) +-- execute +12 + +36 +-- GROUP BY WITH TOTALS before DISTINCT with on the same columns => remove DISTINCT +-- query +SELECT DISTINCT a +FROM +( + SELECT + a, + sum(b) AS c + FROM + ( + SELECT + x.number AS a, + y.number AS b + FROM numbers(3) AS x, numbers(3, 3) AS y + ) + GROUP BY a WITH TOTALS +) +-- explain +Expression ((Projection + (Before ORDER BY + (Projection + Before ORDER BY)))) + TotalsHaving + Aggregating + Expression ((Before GROUP BY + (Projection + Before ORDER BY))) + Join (JOIN FillRightFirst) + Expression (Before JOIN) + ReadFromStorage (SystemNumbers) + Expression ((Joined actions + (Rename joined columns + (Projection + Before ORDER BY)))) + ReadFromStorage (SystemNumbers) +-- execute +0 +2 +1 + +0 diff --git a/tests/queries/0_stateless/02500_remove_redundant_distinct.sh b/tests/queries/0_stateless/02500_remove_redundant_distinct.sh index b7b764f82c4..879cc776fe1 100755 --- a/tests/queries/0_stateless/02500_remove_redundant_distinct.sh +++ b/tests/queries/0_stateless/02500_remove_redundant_distinct.sh @@ -220,3 +220,39 @@ FROM GROUP BY a WITH CUBE )" run_query "$query" + +echo "-- GROUP BY WITH TOTALS before DISTINCT with on different columns => do _not_ remove DISTINCT" +query="SELECT DISTINCT c +FROM +( + SELECT + a, + sum(b) AS c + FROM + ( + SELECT + x.number AS a, + y.number AS b + FROM numbers(3) AS x, numbers(3, 3) AS y + ) + GROUP BY a WITH TOTALS +)" +run_query "$query" + +echo "-- GROUP BY WITH TOTALS before DISTINCT with on the same columns => remove DISTINCT" +query="SELECT DISTINCT a +FROM +( + SELECT + a, + sum(b) AS c + FROM + ( + SELECT + x.number AS a, + y.number AS b + FROM numbers(3) AS x, numbers(3, 3) AS y + ) + GROUP BY a WITH TOTALS +)" +run_query "$query" From 3eef3bf64319d6d34c07189a7a9904f4970b62f5 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 14 Feb 2023 21:49:56 +0000 Subject: [PATCH 216/717] Simplify aggregation pass always checking AggregatingStep and skipping Rollup, Cube, TotalsHaving --- .../Optimizations/removeRedundantDistinct.cpp | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp index 0c8a9de4b45..c0d80309db4 100644 --- a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp +++ b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp @@ -139,6 +139,10 @@ namespace || typeid_cast(step)) return true; + /// those steps can be only after AggregatingStep, so we skip them here but check AggregatingStep separately + if (typeid_cast(step) || typeid_cast(step) || typeid_cast(step)) + return true; + return false; } @@ -172,8 +176,7 @@ namespace while (!node->children.empty()) { const IQueryPlanStep * current_step = node->step.get(); - if (typeid_cast(current_step) || typeid_cast(current_step) - || typeid_cast(current_step) || typeid_cast(current_step)) + if (typeid_cast(current_step)) { aggregation_before_distinct = current_step; break; @@ -205,16 +208,6 @@ namespace if (const auto * aggregating_step = typeid_cast(aggregation_before_distinct); aggregating_step) return compareAggregationKeysWithDistinctColumns(aggregating_step->getParams().keys, distinct_columns, actions); - - if (const auto * cube_step = typeid_cast(aggregation_before_distinct); cube_step) - return compareAggregationKeysWithDistinctColumns(cube_step->getParams().keys, distinct_columns, actions); - - if (const auto * rollup_step = typeid_cast(aggregation_before_distinct); rollup_step) - return compareAggregationKeysWithDistinctColumns(rollup_step->getParams().keys, distinct_columns, actions); - - if (const auto * totals_step = typeid_cast(aggregation_before_distinct); totals_step) - return compareAggregationKeysWithDistinctColumns( - totals_step->getOutputStream().header.getNames(), distinct_columns, actions); } return false; From 144005319d569cfd18d935a294ae247245e4e571 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 14 Feb 2023 22:05:27 +0000 Subject: [PATCH 217/717] Remove MSVC support from poco --- base/poco/Crypto/include/Poco/Crypto/Crypto.h | 70 ------------------- base/poco/Crypto/src/OpenSSLInitializer.cpp | 12 ---- .../Data/ODBC/include/Poco/Data/ODBC/ODBC.h | 5 -- base/poco/Data/include/Poco/Data/Data.h | 5 -- base/poco/Foundation/include/Poco/Alignment.h | 53 -------------- .../Poco/BufferedBidirectionalStreamBuf.h | 3 - .../include/Poco/BufferedStreamBuf.h | 3 - base/poco/Foundation/include/Poco/ByteOrder.h | 9 +-- .../poco/Foundation/include/Poco/Foundation.h | 27 ------- base/poco/Foundation/include/Poco/Platform.h | 2 - .../Foundation/include/Poco/Platform_WIN32.h | 47 ------------- .../poco/Foundation/include/Poco/StreamUtil.h | 5 +- base/poco/Foundation/include/Poco/Tuple.h | 4 -- base/poco/Foundation/include/Poco/Types.h | 23 +----- .../include/Poco/UnbufferedStreamBuf.h | 3 - base/poco/Foundation/src/gzguts.h | 5 -- base/poco/Foundation/src/zutil.h | 13 ---- base/poco/JSON/include/Poco/JSON/JSON.h | 5 -- .../MongoDB/include/Poco/MongoDB/MongoDB.h | 5 -- base/poco/Net/include/Poco/Net/Net.h | 5 -- base/poco/Net/src/IPAddress.cpp | 2 - base/poco/Net/src/SocketAddress.cpp | 2 - .../NetSSL_OpenSSL/include/Poco/Net/NetSSL.h | 7 -- base/poco/Redis/include/Poco/Redis/Redis.h | 5 -- base/poco/Util/include/Poco/Util/Util.h | 5 -- base/poco/XML/include/Poco/XML/XML.h | 5 -- .../XML/include/Poco/XML/expat_external.h | 4 +- base/poco/XML/src/xmlparse.cpp | 3 - 28 files changed, 4 insertions(+), 333 deletions(-) diff --git a/base/poco/Crypto/include/Poco/Crypto/Crypto.h b/base/poco/Crypto/include/Poco/Crypto/Crypto.h index a6b8a5cf22f..90df2fed128 100644 --- a/base/poco/Crypto/include/Poco/Crypto/Crypto.h +++ b/base/poco/Crypto/include/Poco/Crypto/Crypto.h @@ -89,76 +89,6 @@ enum RSAPaddingMode // // Automatically link Crypto and OpenSSL libraries. // -#if defined(_MSC_VER) -# if !defined(POCO_NO_AUTOMATIC_LIBS) -# if defined(POCO_INTERNAL_OPENSSL_MSVC_VER) -# if defined(POCO_EXTERNAL_OPENSSL) -# pragma message("External OpenSSL defined but internal headers used - possible mismatch!") -# endif // POCO_EXTERNAL_OPENSSL -# if !defined(_DEBUG) -# define POCO_DEBUG_SUFFIX "" -# if !defined(_DLL) -# define POCO_STATIC_SUFFIX "mt" -# else // _DLL -# define POCO_STATIC_SUFFIX "" -# endif -# else // _DEBUG -# define POCO_DEBUG_SUFFIX "d" -# if !defined(_DLL) -# define POCO_STATIC_SUFFIX "mt" -# else // _DLL -# define POCO_STATIC_SUFFIX "" -# endif -# endif -# pragma comment(lib, "libcrypto" POCO_STATIC_SUFFIX POCO_DEBUG_SUFFIX ".lib") -# pragma comment(lib, "libssl" POCO_STATIC_SUFFIX POCO_DEBUG_SUFFIX ".lib") -# if !defined(_WIN64) && !defined(_DLL) && (POCO_INTERNAL_OPENSSL_MSVC_VER == 120) \ - && (POCO_MSVC_VERSION < POCO_INTERNAL_OPENSSL_MSVC_VER) -# pragma comment(lib, "libPreVS2013CRT" POCO_STATIC_SUFFIX POCO_DEBUG_SUFFIX ".lib") -# endif -# if !defined(_DLL) && (POCO_MSVS_VERSION >= 2015) -# pragma comment(lib, "legacy_stdio_definitions.lib") -# pragma comment(lib, "legacy_stdio_wide_specifiers.lib") -# endif -# elif defined(POCO_EXTERNAL_OPENSSL) -# if POCO_EXTERNAL_OPENSSL == POCO_EXTERNAL_OPENSSL_SLPRO -# if defined(POCO_DLL) -# if OPENSSL_VERSION_PREREQ(1, 1) -# pragma comment(lib, "libcrypto.lib") -# pragma comment(lib, "libssl.lib") -# else -# pragma comment(lib, "libeay32.lib") -# pragma comment(lib, "ssleay32.lib") -# endif -# else -# if OPENSSL_VERSION_PREREQ(1, 1) -# if defined(_WIN64) -# pragma comment(lib, "libcrypto64" POCO_LIB_SUFFIX) -# pragma comment(lib, "libssl64" POCO_LIB_SUFFIX) -# else -# pragma comment(lib, "libcrypto32" POCO_LIB_SUFFIX) -# pragma comment(lib, "libssl32" POCO_LIB_SUFFIX) -# endif -# else -# pragma comment(lib, "libeay32" POCO_LIB_SUFFIX) -# pragma comment(lib, "ssleay32" POCO_LIB_SUFFIX) -# endif -# endif -# elif POCO_EXTERNAL_OPENSSL == POCO_EXTERNAL_OPENSSL_DEFAULT -# if OPENSSL_VERSION_PREREQ(1, 1) -# pragma comment(lib, "libcrypto.lib") -# pragma comment(lib, "libssl.lib") -# else -# pragma comment(lib, "libeay32.lib") -# pragma comment(lib, "ssleay32.lib") -# endif -# endif -# endif // POCO_INTERNAL_OPENSSL_MSVC_VER -# if !defined(Crypto_EXPORTS) -# pragma comment(lib, "PocoCrypto" POCO_LIB_SUFFIX) -# endif -# endif // POCO_NO_AUTOMATIC_LIBS -#endif namespace Poco diff --git a/base/poco/Crypto/src/OpenSSLInitializer.cpp b/base/poco/Crypto/src/OpenSSLInitializer.cpp index a3bc1d01431..9aa44c2d976 100644 --- a/base/poco/Crypto/src/OpenSSLInitializer.cpp +++ b/base/poco/Crypto/src/OpenSSLInitializer.cpp @@ -40,18 +40,6 @@ using Poco::RandomInputStream; using Poco::Thread; -#if defined(_MSC_VER) && !defined(_DLL) && defined(POCO_INTERNAL_OPENSSL_MSVC_VER) - - #if (POCO_MSVS_VERSION >= 2015) - FILE _iob[] = { *stdin, *stdout, *stderr }; - extern "C" FILE * __cdecl __iob_func(void) { return _iob; } - #endif // (POCO_MSVS_VERSION >= 2015) - - #if (POCO_MSVS_VERSION < 2012) - extern "C" __declspec(noreturn) void __cdecl __report_rangecheckfailure(void) { ::ExitProcess(1); } - #endif // (POCO_MSVS_VERSION < 2012) - -#endif // _MSC_VER && _MT && !POCO_EXTERNAL_OPENSSL && (POCO_MSVS_VERSION < 2013) namespace Poco { diff --git a/base/poco/Data/ODBC/include/Poco/Data/ODBC/ODBC.h b/base/poco/Data/ODBC/include/Poco/Data/ODBC/ODBC.h index 8a6fa6560f9..9cf91df6da5 100644 --- a/base/poco/Data/ODBC/include/Poco/Data/ODBC/ODBC.h +++ b/base/poco/Data/ODBC/include/Poco/Data/ODBC/ODBC.h @@ -58,11 +58,6 @@ // // Automatically link Data library. // -#if defined(_MSC_VER) -# if !defined(POCO_NO_AUTOMATIC_LIBS) && !defined(ODBC_EXPORTS) -# pragma comment(lib, "PocoDataODBC" POCO_LIB_SUFFIX) -# endif -#endif #endif // ODBC_ODBC_INCLUDED diff --git a/base/poco/Data/include/Poco/Data/Data.h b/base/poco/Data/include/Poco/Data/Data.h index 7243d36c091..e45def7ea24 100644 --- a/base/poco/Data/include/Poco/Data/Data.h +++ b/base/poco/Data/include/Poco/Data/Data.h @@ -52,11 +52,6 @@ // // Automatically link Data library. // -#if defined(_MSC_VER) -# if !defined(POCO_NO_AUTOMATIC_LIBS) && !defined(Data_EXPORTS) -# pragma comment(lib, "PocoData" POCO_LIB_SUFFIX) -# endif -#endif #endif // Data_Data_INCLUDED diff --git a/base/poco/Foundation/include/Poco/Alignment.h b/base/poco/Foundation/include/Poco/Alignment.h index 72218910dd2..b1d48ffd62d 100644 --- a/base/poco/Foundation/include/Poco/Alignment.h +++ b/base/poco/Foundation/include/Poco/Alignment.h @@ -135,7 +135,6 @@ struct AlignedCharArrayImpl; // MSVC requires special handling here. -# ifndef _MSC_VER # ifdef POCO_COMPILER_CLANG @@ -179,58 +178,6 @@ POCO_ALIGNEDCHARARRAY_TEMPLATE_ALIGNMENT(8192); # undef POCO_ALIGNEDCHARARRAY_TEMPLATE_ALIGNMENT # endif // POCO_HAVE_ALIGNMENT -# else // _MSC_VER - -// We provide special variations of this template for the most common -// alignments because __declspec(align(...)) doesn't actually work when it is -// a member of a by-value function argument in MSVC, even if the alignment -// request is something reasonably like 8-byte or 16-byte. -template <> -struct AlignedCharArrayImpl<1> -{ - char aligned; -}; -template <> -struct AlignedCharArrayImpl<2> -{ - short aligned; -}; -template <> -struct AlignedCharArrayImpl<4> -{ - int aligned; -}; -template <> -struct AlignedCharArrayImpl<8> -{ - double aligned; -}; - -# define POCO_ALIGNEDCHARARRAY_TEMPLATE_ALIGNMENT(x) \ - template <> \ - struct AlignedCharArrayImpl \ - { \ - __declspec(align(x)) char aligned; \ - } - -POCO_ALIGNEDCHARARRAY_TEMPLATE_ALIGNMENT(16); -POCO_ALIGNEDCHARARRAY_TEMPLATE_ALIGNMENT(32); -POCO_ALIGNEDCHARARRAY_TEMPLATE_ALIGNMENT(64); -POCO_ALIGNEDCHARARRAY_TEMPLATE_ALIGNMENT(128); - -# if (_MSC_VER > 1600) // MSVC 2010 complains on alignment larger than 128 -POCO_ALIGNEDCHARARRAY_TEMPLATE_ALIGNMENT(512); -POCO_ALIGNEDCHARARRAY_TEMPLATE_ALIGNMENT(1024); -POCO_ALIGNEDCHARARRAY_TEMPLATE_ALIGNMENT(2048); -POCO_ALIGNEDCHARARRAY_TEMPLATE_ALIGNMENT(4096); -POCO_ALIGNEDCHARARRAY_TEMPLATE_ALIGNMENT(8192); -# endif // _MSC_VER > 1600 - -// Any larger and MSVC complains. -# undef POCO_ALIGNEDCHARARRAY_TEMPLATE_ALIGNMENT - -# define POCO_HAVE_ALIGNMENT -# endif // _MSC_VER // POCO_HAVE_ALIGNMENT will be defined on the pre-C++11 platforms/compilers where // it can be reliably determined and used. Uncomment the line below to explicitly diff --git a/base/poco/Foundation/include/Poco/BufferedBidirectionalStreamBuf.h b/base/poco/Foundation/include/Poco/BufferedBidirectionalStreamBuf.h index ec00d61eeb6..05d2f81b16d 100644 --- a/base/poco/Foundation/include/Poco/BufferedBidirectionalStreamBuf.h +++ b/base/poco/Foundation/include/Poco/BufferedBidirectionalStreamBuf.h @@ -161,9 +161,6 @@ private: // instantiation - to avoid duplicate symbols due to multiple // instantiations in different libraries. // -#if defined(_MSC_VER) && defined(POCO_DLL) && !defined(Foundation_EXPORTS) -template class Foundation_API BasicBufferedBidirectionalStreamBuf>; -#endif typedef BasicBufferedBidirectionalStreamBuf> BufferedBidirectionalStreamBuf; diff --git a/base/poco/Foundation/include/Poco/BufferedStreamBuf.h b/base/poco/Foundation/include/Poco/BufferedStreamBuf.h index ea81e5fafa8..9f4cbd4e4d8 100644 --- a/base/poco/Foundation/include/Poco/BufferedStreamBuf.h +++ b/base/poco/Foundation/include/Poco/BufferedStreamBuf.h @@ -151,9 +151,6 @@ private: // instantiation - to avoid duplicate symbols due to multiple // instantiations in different libraries. // -#if defined(_MSC_VER) && defined(POCO_DLL) && !defined(Foundation_EXPORTS) -template class Foundation_API BasicBufferedStreamBuf>; -#endif typedef BasicBufferedStreamBuf> BufferedStreamBuf; diff --git a/base/poco/Foundation/include/Poco/ByteOrder.h b/base/poco/Foundation/include/Poco/ByteOrder.h index e5ce630c03a..4f2644ddf4e 100644 --- a/base/poco/Foundation/include/Poco/ByteOrder.h +++ b/base/poco/Foundation/include/Poco/ByteOrder.h @@ -18,9 +18,6 @@ #include "Poco/Foundation.h" #include "Poco/Types.h" -#if defined(_MSC_VER) -# include // builtins -#endif namespace Poco @@ -99,11 +96,7 @@ public: #if !defined(POCO_NO_BYTESWAP_BUILTINS) -# if defined(_MSC_VER) -# if (POCO_MSVC_VERSION > 71) -# define POCO_HAVE_MSC_BYTESWAP 1 -# endif -# elif defined(__clang__) +# if defined(__clang__) # if __has_builtin(__builtin_bswap32) # define POCO_HAVE_GCC_BYTESWAP 1 # endif diff --git a/base/poco/Foundation/include/Poco/Foundation.h b/base/poco/Foundation/include/Poco/Foundation.h index 51bcc4e509d..23755b8e8b9 100644 --- a/base/poco/Foundation/include/Poco/Foundation.h +++ b/base/poco/Foundation/include/Poco/Foundation.h @@ -65,31 +65,6 @@ // // Automatically link Foundation library. // -#if defined(_MSC_VER) -# if defined(POCO_DLL) -# if defined(_DEBUG) -# define POCO_LIB_SUFFIX "d.lib" -# else -# define POCO_LIB_SUFFIX ".lib" -# endif -# elif defined(_DLL) -# if defined(_DEBUG) -# define POCO_LIB_SUFFIX "mdd.lib" -# else -# define POCO_LIB_SUFFIX "md.lib" -# endif -# else -# if defined(_DEBUG) -# define POCO_LIB_SUFFIX "mtd.lib" -# else -# define POCO_LIB_SUFFIX "mt.lib" -# endif -# endif - -# if !defined(POCO_NO_AUTOMATIC_LIBS) && !defined(Foundation_EXPORTS) -# pragma comment(lib, "PocoFoundation" POCO_LIB_SUFFIX) -# endif -#endif // @@ -150,8 +125,6 @@ # define POCO_DEPRECATED __attribute__((deprecated)) #elif defined(__clang__) # define POCO_DEPRECATED __attribute__((deprecated)) -#elif defined(_MSC_VER) -# define POCO_DEPRECATED __declspec(deprecated) #else # define POCO_DEPRECATED #endif diff --git a/base/poco/Foundation/include/Poco/Platform.h b/base/poco/Foundation/include/Poco/Platform.h index fea9f14352b..6623b25781e 100644 --- a/base/poco/Foundation/include/Poco/Platform.h +++ b/base/poco/Foundation/include/Poco/Platform.h @@ -232,8 +232,6 @@ #if defined(__clang__) # define POCO_COMPILER_CLANG -#elif defined(_MSC_VER) -# define POCO_COMPILER_MSVC #elif defined(__GNUC__) # define POCO_COMPILER_GCC #elif defined(__MINGW32__) || defined(__MINGW64__) diff --git a/base/poco/Foundation/include/Poco/Platform_WIN32.h b/base/poco/Foundation/include/Poco/Platform_WIN32.h index 9a893591f13..0d5ec7ec65d 100644 --- a/base/poco/Foundation/include/Poco/Platform_WIN32.h +++ b/base/poco/Foundation/include/Poco/Platform_WIN32.h @@ -129,16 +129,10 @@ #endif // POCO_FORCE_MIN_WINDOWS_OS_SUPPORT -#if defined(_MSC_VER) && !defined(POCO_MSVC_SECURE_WARNINGS) && !defined(_CRT_SECURE_NO_DEPRECATE) -# define _CRT_SECURE_NO_DEPRECATE -#endif // Verify that we're built with the multithreaded // versions of the runtime libraries -#if defined(_MSC_VER) && !defined(_MT) -# error Must compile with /MD, /MDd, /MT or /MTd -#endif // Check debug/release settings consistency @@ -147,31 +141,6 @@ #endif -#if (_MSC_VER >= 1300) && (_MSC_VER < 1400) // Visual Studio 2003, MSVC++ 7.1 -# define POCO_MSVS_VERSION 2003 -# define POCO_MSVC_VERSION 71 -#elif (_MSC_VER >= 1400) && (_MSC_VER < 1500) // Visual Studio 2005, MSVC++ 8.0 -# define POCO_MSVS_VERSION 2005 -# define POCO_MSVC_VERSION 80 -#elif (_MSC_VER >= 1500) && (_MSC_VER < 1600) // Visual Studio 2008, MSVC++ 9.0 -# define POCO_MSVS_VERSION 2008 -# define POCO_MSVC_VERSION 90 -#elif (_MSC_VER >= 1600) && (_MSC_VER < 1700) // Visual Studio 2010, MSVC++ 10.0 -# define POCO_MSVS_VERSION 2010 -# define POCO_MSVC_VERSION 100 -#elif (_MSC_VER >= 1700) && (_MSC_VER < 1800) // Visual Studio 2012, MSVC++ 11.0 -# define POCO_MSVS_VERSION 2012 -# define POCO_MSVC_VERSION 110 -#elif (_MSC_VER >= 1800) && (_MSC_VER < 1900) // Visual Studio 2013, MSVC++ 12.0 -# define POCO_MSVS_VERSION 2013 -# define POCO_MSVC_VERSION 120 -#elif (_MSC_VER >= 1900) && (_MSC_VER < 1910) // Visual Studio 2015, MSVC++ 14.0 -# define POCO_MSVS_VERSION 2015 -# define POCO_MSVC_VERSION 140 -#elif (_MSC_VER >= 1910) && (_MSC_VER < 2000) // Visual Studio 2017, MSVC++ 14.1 -# define POCO_MSVS_VERSION 2017 -# define POCO_MSVC_VERSION 141 -#endif // Unicode Support @@ -186,25 +155,9 @@ // Turn off some annoying warnings -#if defined(_MSC_VER) -# pragma warning(disable : 4018) // signed/unsigned comparison -# pragma warning(disable : 4250) // VC++ 11.0: inheriting from std stream classes produces C4250 warning; \ - // see -# pragma warning(disable : 4251) // ... needs to have dll-interface warning -# pragma warning(disable : 4275) // non dll-interface class 'std::exception' used as base for dll-interface class 'Poco::Exception' -# pragma warning( \ - disable : 4344) // behavior change: use of explicit template arguments results in call to '...' but '...' is a better match -# pragma warning(disable : 4351) // new behavior: elements of array '...' will be default initialized -# pragma warning(disable : 4355) // 'this' : used in base member initializer list -# pragma warning(disable : 4675) // resolved overload was found by argument-dependent lookup -# pragma warning(disable : 4996) // VC++ 8.0 deprecation warnings -#endif // Enable C++11 support for VS 2010 and newer -#if defined(_MSC_VER) && (_MSC_VER >= 1700) && !defined(POCO_ENABLE_CPP11) -# define POCO_ENABLE_CPP11 -#endif #if defined(__INTEL_COMPILER) diff --git a/base/poco/Foundation/include/Poco/StreamUtil.h b/base/poco/Foundation/include/Poco/StreamUtil.h index e6c7cb3fa57..7668ec4555d 100644 --- a/base/poco/Foundation/include/Poco/StreamUtil.h +++ b/base/poco/Foundation/include/Poco/StreamUtil.h @@ -73,10 +73,7 @@ #if !defined(POCO_IOS_INIT_HACK) // Microsoft Visual Studio with Dinkumware STL (but not STLport) -# if defined(_MSC_VER) && (!defined(_STLP_MSVC) || defined(_STLP_NO_OWN_IOSTREAMS)) -# define POCO_IOS_INIT_HACK 1 -// QNX with Dinkumware but not GNU C++ Library -# elif defined(__QNX__) && !defined(__GLIBCPP__) +# if defined(__QNX__) && !defined(__GLIBCPP__) # define POCO_IOS_INIT_HACK 1 # endif #endif diff --git a/base/poco/Foundation/include/Poco/Tuple.h b/base/poco/Foundation/include/Poco/Tuple.h index d79df65bc0f..07a379ff10a 100644 --- a/base/poco/Foundation/include/Poco/Tuple.h +++ b/base/poco/Foundation/include/Poco/Tuple.h @@ -26,11 +26,7 @@ namespace Poco { -#if defined(_MSC_VER) -# define POCO_TYPEWRAPPER_DEFAULTVALUE(T) TypeWrapper::TYPE() -#else # define POCO_TYPEWRAPPER_DEFAULTVALUE(T) typename TypeWrapper::TYPE() -#endif template < diff --git a/base/poco/Foundation/include/Poco/Types.h b/base/poco/Foundation/include/Poco/Types.h index d9a3e5bd26e..85256e8dcea 100644 --- a/base/poco/Foundation/include/Poco/Types.h +++ b/base/poco/Foundation/include/Poco/Types.h @@ -25,28 +25,7 @@ namespace Poco { -#if defined(_MSC_VER) -// -// Windows/Visual C++ -// -typedef signed char Int8; -typedef unsigned char UInt8; -typedef signed short Int16; -typedef unsigned short UInt16; -typedef signed int Int32; -typedef unsigned int UInt32; -typedef signed __int64 Int64; -typedef unsigned __int64 UInt64; -# if defined(_WIN64) -# define POCO_PTR_IS_64_BIT 1 -typedef signed __int64 IntPtr; -typedef unsigned __int64 UIntPtr; -# else -typedef signed long IntPtr; -typedef unsigned long UIntPtr; -# endif -# define POCO_HAVE_INT64 1 -#elif defined(__GNUC__) || defined(__clang__) +#if defined(__GNUC__) || defined(__clang__) // // Unix/GCC/Clang // diff --git a/base/poco/Foundation/include/Poco/UnbufferedStreamBuf.h b/base/poco/Foundation/include/Poco/UnbufferedStreamBuf.h index 9c867ed0869..56c1d148af2 100644 --- a/base/poco/Foundation/include/Poco/UnbufferedStreamBuf.h +++ b/base/poco/Foundation/include/Poco/UnbufferedStreamBuf.h @@ -156,9 +156,6 @@ private: // instantiation - to avoid duplicate symbols due to multiple // instantiations in different libraries. // -#if defined(_MSC_VER) && defined(POCO_DLL) && !defined(Foundation_EXPORTS) -template class Foundation_API BasicUnbufferedStreamBuf>; -#endif typedef BasicUnbufferedStreamBuf> UnbufferedStreamBuf; diff --git a/base/poco/Foundation/src/gzguts.h b/base/poco/Foundation/src/gzguts.h index 2cbe1e4baec..488d328c5cd 100644 --- a/base/poco/Foundation/src/gzguts.h +++ b/base/poco/Foundation/src/gzguts.h @@ -74,9 +74,7 @@ # ifdef WIN32 /* In Win32, vsnprintf is available as the "non-ANSI" _vsnprintf. */ # if !defined(vsnprintf) && !defined(NO_vsnprintf) -# if !defined(_MSC_VER) || (defined(_MSC_VER) && _MSC_VER < 1500) # define vsnprintf _vsnprintf -# endif # endif # endif # ifdef __SASC @@ -96,9 +94,6 @@ /* unlike snprintf (which is required in C99), _snprintf does not guarantee null termination of the result -- however this is only used in gzlib.c where the result is assured to fit in the space provided */ -#if defined(_MSC_VER) && _MSC_VER < 1900 -# define snprintf _snprintf -#endif #ifndef local # define local static diff --git a/base/poco/Foundation/src/zutil.h b/base/poco/Foundation/src/zutil.h index 48dd5f69293..38223d36b6c 100644 --- a/base/poco/Foundation/src/zutil.h +++ b/base/poco/Foundation/src/zutil.h @@ -22,9 +22,7 @@ #include "zlib.h" #if defined(STDC) && !defined(Z_SOLO) -# if !(defined(_WIN32_WCE) && defined(_MSC_VER)) # include -# endif # include # include #endif @@ -130,17 +128,6 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ # define fdopen(fd, mode) NULL /* No fdopen() */ #endif -#if (defined(_MSC_VER) && (_MSC_VER > 600)) && !defined __INTERIX -# if defined(_WIN32_WCE) && _WIN32_WCE < 0x800 -# define fdopen(fd, mode) NULL /* No fdopen() */ -# ifndef _PTRDIFF_T_DEFINED -typedef int ptrdiff_t; -# define _PTRDIFF_T_DEFINED -# endif -# else -# define fdopen(fd, type) _fdopen(fd, type) -# endif -#endif /* provide prototypes for these when building zlib without LFS */ #if !defined(_WIN32) && (!defined(_LARGEFILE64_SOURCE) || _LFS64_LARGEFILE - 0 == 0) diff --git a/base/poco/JSON/include/Poco/JSON/JSON.h b/base/poco/JSON/include/Poco/JSON/JSON.h index f0bef787be0..2e1717a040d 100644 --- a/base/poco/JSON/include/Poco/JSON/JSON.h +++ b/base/poco/JSON/include/Poco/JSON/JSON.h @@ -52,11 +52,6 @@ // // Automatically link JSON library. // -#if defined(_MSC_VER) -# if !defined(POCO_NO_AUTOMATIC_LIBS) && !defined(JSON_EXPORTS) -# pragma comment(lib, "PocoJSON" POCO_LIB_SUFFIX) -# endif -#endif #endif // JSON_JSON_INCLUDED diff --git a/base/poco/MongoDB/include/Poco/MongoDB/MongoDB.h b/base/poco/MongoDB/include/Poco/MongoDB/MongoDB.h index de246ddc9dd..d336c1a8478 100644 --- a/base/poco/MongoDB/include/Poco/MongoDB/MongoDB.h +++ b/base/poco/MongoDB/include/Poco/MongoDB/MongoDB.h @@ -54,11 +54,6 @@ // // Automatically link MongoDB library. // -#if defined(_MSC_VER) -# if !defined(POCO_NO_AUTOMATIC_LIBS) && !defined(MongoDB_EXPORTS) -# pragma comment(lib, "PocoMongoDB" POCO_LIB_SUFFIX) -# endif -#endif #endif // MongoDBMongoDB_INCLUDED diff --git a/base/poco/Net/include/Poco/Net/Net.h b/base/poco/Net/include/Poco/Net/Net.h index 6d9a6f6b4ac..cc89471ae06 100644 --- a/base/poco/Net/include/Poco/Net/Net.h +++ b/base/poco/Net/include/Poco/Net/Net.h @@ -52,11 +52,6 @@ // // Automatically link Net library. // -#if defined(_MSC_VER) -# if !defined(POCO_NO_AUTOMATIC_LIBS) && !defined(Net_EXPORTS) -# pragma comment(lib, "PocoNet" POCO_LIB_SUFFIX) -# endif -#endif // Default to enabled IPv6 support if not explicitly disabled diff --git a/base/poco/Net/src/IPAddress.cpp b/base/poco/Net/src/IPAddress.cpp index e34a02181b2..90d9400304a 100644 --- a/base/poco/Net/src/IPAddress.cpp +++ b/base/poco/Net/src/IPAddress.cpp @@ -42,14 +42,12 @@ namespace Poco { namespace Net { -#if !defined(_MSC_VER) || defined(__STDC__) // Go home MSVC, you're drunk... // See http://stackoverflow.com/questions/5899857/multiple-definition-error-for-static-const-class-members const IPAddress::Family IPAddress::IPv4; #if defined(POCO_HAVE_IPv6) const IPAddress::Family IPAddress::IPv6; #endif -#endif IPAddress::IPAddress() diff --git a/base/poco/Net/src/SocketAddress.cpp b/base/poco/Net/src/SocketAddress.cpp index 0782ca3eea3..42ea75e6c18 100644 --- a/base/poco/Net/src/SocketAddress.cpp +++ b/base/poco/Net/src/SocketAddress.cpp @@ -56,7 +56,6 @@ struct AFLT // -#if !defined(_MSC_VER) || defined(__STDC__) // Go home MSVC, you're drunk... // See http://stackoverflow.com/questions/5899857/multiple-definition-error-for-static-const-class-members const SocketAddress::Family SocketAddress::IPv4; @@ -66,7 +65,6 @@ const SocketAddress::Family SocketAddress::IPv6; #if defined(POCO_OS_FAMILY_UNIX) const SocketAddress::Family SocketAddress::UNIX_LOCAL; #endif -#endif SocketAddress::SocketAddress() diff --git a/base/poco/NetSSL_OpenSSL/include/Poco/Net/NetSSL.h b/base/poco/NetSSL_OpenSSL/include/Poco/Net/NetSSL.h index abf9ca5c604..d6c7eba1a3d 100644 --- a/base/poco/NetSSL_OpenSSL/include/Poco/Net/NetSSL.h +++ b/base/poco/NetSSL_OpenSSL/include/Poco/Net/NetSSL.h @@ -53,13 +53,6 @@ // // Automatically link NetSSL and OpenSSL libraries. // -#if defined(_MSC_VER) -# if !defined(POCO_NO_AUTOMATIC_LIBS) -# if !defined(NetSSL_EXPORTS) -# pragma comment(lib, "PocoNetSSL" POCO_LIB_SUFFIX) -# endif -# endif // POCO_NO_AUTOMATIC_LIBS -#endif namespace Poco diff --git a/base/poco/Redis/include/Poco/Redis/Redis.h b/base/poco/Redis/include/Poco/Redis/Redis.h index 3ba1660cebf..ef9400fbfd7 100644 --- a/base/poco/Redis/include/Poco/Redis/Redis.h +++ b/base/poco/Redis/include/Poco/Redis/Redis.h @@ -52,11 +52,6 @@ // // Automatically link Redis library. // -#if defined(_MSC_VER) -# if !defined(POCO_NO_AUTOMATIC_LIBS) && !defined(Redis_EXPORTS) -# pragma comment(lib, "PocoRedis" POCO_LIB_SUFFIX) -# endif -#endif #endif // RedisRedis_INCLUDED diff --git a/base/poco/Util/include/Poco/Util/Util.h b/base/poco/Util/include/Poco/Util/Util.h index 0116b06c2a5..ea59e9192f1 100644 --- a/base/poco/Util/include/Poco/Util/Util.h +++ b/base/poco/Util/include/Poco/Util/Util.h @@ -52,11 +52,6 @@ // // Automatically link Util library. // -#if defined(_MSC_VER) -# if !defined(POCO_NO_AUTOMATIC_LIBS) && !defined(Util_EXPORTS) -# pragma comment(lib, "PocoUtil" POCO_LIB_SUFFIX) -# endif -#endif #endif // Util_Util_INCLUDED diff --git a/base/poco/XML/include/Poco/XML/XML.h b/base/poco/XML/include/Poco/XML/XML.h index a5b611a76dd..f527d1a9e41 100644 --- a/base/poco/XML/include/Poco/XML/XML.h +++ b/base/poco/XML/include/Poco/XML/XML.h @@ -52,11 +52,6 @@ // // Automatically link XML library. // -#if defined(_MSC_VER) -# if !defined(POCO_NO_AUTOMATIC_LIBS) && !defined(XML_EXPORTS) -# pragma comment(lib, "PocoXML" POCO_LIB_SUFFIX) -# endif -#endif #endif // XML_XML_INCLUDED diff --git a/base/poco/XML/include/Poco/XML/expat_external.h b/base/poco/XML/include/Poco/XML/expat_external.h index cb5905c7b04..6a846ba9e00 100644 --- a/base/poco/XML/include/Poco/XML/expat_external.h +++ b/base/poco/XML/include/Poco/XML/expat_external.h @@ -65,9 +65,7 @@ system headers may assume the cdecl convention. */ #ifndef XMLCALL -# if defined(_MSC_VER) -# define XMLCALL __cdecl -# elif defined(__GNUC__) && defined(__i386) && !defined(__INTEL_COMPILER) +# if defined(__GNUC__) && defined(__i386) && !defined(__INTEL_COMPILER) # define XMLCALL __attribute__((cdecl)) # else /* For any platform which uses this definition and supports more than diff --git a/base/poco/XML/src/xmlparse.cpp b/base/poco/XML/src/xmlparse.cpp index 3207ac3ddbd..1ac894ff25c 100644 --- a/base/poco/XML/src/xmlparse.cpp +++ b/base/poco/XML/src/xmlparse.cpp @@ -68,9 +68,6 @@ # endif #endif -#if defined(_MSC_VER) && !defined(_CRT_SECURE_NO_WARNINGS) -#define _CRT_SECURE_NO_WARNINGS -#endif #include #include /* memset(), memcpy() */ From 1942ea6f294d30a4919daa75d34fadbf3720ade9 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 14 Feb 2023 22:07:38 +0000 Subject: [PATCH 218/717] Remove further legacy compiler support from poco --- base/poco/Foundation/include/Poco/Platform.h | 10 -- .../Foundation/include/Poco/Platform_POSIX.h | 3 - base/poco/Foundation/include/Poco/String.h | 6 - base/poco/Foundation/include/Poco/Types.h | 117 ------------------ 4 files changed, 136 deletions(-) diff --git a/base/poco/Foundation/include/Poco/Platform.h b/base/poco/Foundation/include/Poco/Platform.h index 6623b25781e..87fa440fa5f 100644 --- a/base/poco/Foundation/include/Poco/Platform.h +++ b/base/poco/Foundation/include/Poco/Platform.h @@ -238,24 +238,14 @@ # define POCO_COMPILER_MINGW #elif defined(__INTEL_COMPILER) || defined(__ICC) || defined(__ECC) || defined(__ICL) # define POCO_COMPILER_INTEL -#elif defined(__SUNPRO_CC) -# define POCO_COMPILER_SUN #elif defined(__MWERKS__) || defined(__CWCC__) # define POCO_COMPILER_CODEWARRIOR #elif defined(__sgi) || defined(sgi) # define POCO_COMPILER_SGI -#elif defined(__HP_aCC) -# define POCO_COMPILER_HP_ACC #elif defined(__BORLANDC__) || defined(__CODEGEARC__) # define POCO_COMPILER_CBUILDER #elif defined(__DMC__) # define POCO_COMPILER_DMARS -#elif defined(__DECCXX) -# define POCO_COMPILER_COMPAC -#elif (defined(__xlc__) || defined(__xlC__)) && defined(__IBMCPP__) -# define POCO_COMPILER_IBM_XLC // IBM XL C++ -#elif defined(__IBMCPP__) && defined(__COMPILER_VER__) -# define POCO_COMPILER_IBM_XLC_ZOS // IBM z/OS C++ #endif diff --git a/base/poco/Foundation/include/Poco/Platform_POSIX.h b/base/poco/Foundation/include/Poco/Platform_POSIX.h index cc2667bdac8..96f0c32cb9e 100644 --- a/base/poco/Foundation/include/Poco/Platform_POSIX.h +++ b/base/poco/Foundation/include/Poco/Platform_POSIX.h @@ -25,9 +25,6 @@ #if defined(hpux) || defined(_hpux) # if defined(__hppa) || defined(__hppa__) # define POCO_NO_SYS_SELECT_H 1 -# if defined(__HP_aCC) -# define POCO_NO_TEMPLATE_ICOMPARE 1 -# endif # endif #endif diff --git a/base/poco/Foundation/include/Poco/String.h b/base/poco/Foundation/include/Poco/String.h index 90168ede895..2ac03d38aed 100644 --- a/base/poco/Foundation/include/Poco/String.h +++ b/base/poco/Foundation/include/Poco/String.h @@ -429,13 +429,7 @@ S translateInPlace(S & str, const typename S::value_type * from, const typename poco_check_ptr(from); poco_check_ptr(to); str = translate(str, S(from), S(to)); -#if defined(__SUNPRO_CC) - // Fix around the RVO bug in SunStudio 12.4 - S ret(str); - return ret; -#else return str; -#endif } diff --git a/base/poco/Foundation/include/Poco/Types.h b/base/poco/Foundation/include/Poco/Types.h index 85256e8dcea..327aea02286 100644 --- a/base/poco/Foundation/include/Poco/Types.h +++ b/base/poco/Foundation/include/Poco/Types.h @@ -55,123 +55,6 @@ typedef unsigned long long UInt64; # endif # endif # define POCO_HAVE_INT64 1 -#elif defined(__DECCXX) -// -// Compaq C++ -// -typedef signed char Int8; -typedef unsigned char UInt8; -typedef signed short Int16; -typedef unsigned short UInt16; -typedef signed int Int32; -typedef unsigned int UInt32; -typedef signed __int64 Int64; -typedef unsigned __int64 UInt64; -typedef signed long IntPtr; -typedef unsigned long UIntPtr; -# define POCO_PTR_IS_64_BIT 1 -# define POCO_LONG_IS_64_BIT 1 -# define POCO_HAVE_INT64 1 -#elif defined(__HP_aCC) -// -// HP Ansi C++ -// -typedef signed char Int8; -typedef unsigned char UInt8; -typedef signed short Int16; -typedef unsigned short UInt16; -typedef signed int Int32; -typedef unsigned int UInt32; -typedef signed long IntPtr; -typedef unsigned long UIntPtr; -# if defined(__LP64__) -# define POCO_PTR_IS_64_BIT 1 -# define POCO_LONG_IS_64_BIT 1 -typedef signed long Int64; -typedef unsigned long UInt64; -# else -typedef signed long long Int64; -typedef unsigned long long UInt64; -# endif -# define POCO_HAVE_INT64 1 -#elif defined(__SUNPRO_CC) -// -// SUN Forte C++ -// -typedef signed char Int8; -typedef unsigned char UInt8; -typedef signed short Int16; -typedef unsigned short UInt16; -typedef signed int Int32; -typedef unsigned int UInt32; -typedef signed long IntPtr; -typedef unsigned long UIntPtr; -# if defined(__sparcv9) -# define POCO_PTR_IS_64_BIT 1 -# define POCO_LONG_IS_64_BIT 1 -typedef signed long Int64; -typedef unsigned long UInt64; -# else -typedef signed long long Int64; -typedef unsigned long long UInt64; -# endif -# define POCO_HAVE_INT64 1 -#elif defined(__IBMCPP__) -// -// IBM XL C++ -// -typedef signed char Int8; -typedef unsigned char UInt8; -typedef signed short Int16; -typedef unsigned short UInt16; -typedef signed int Int32; -typedef unsigned int UInt32; -typedef signed long IntPtr; -typedef unsigned long UIntPtr; -# if defined(__64BIT__) -# define POCO_PTR_IS_64_BIT 1 -# define POCO_LONG_IS_64_BIT 1 -typedef signed long Int64; -typedef unsigned long UInt64; -# else -typedef signed long long Int64; -typedef unsigned long long UInt64; -# endif -# define POCO_HAVE_INT64 1 -#elif defined(__sgi) -// -// MIPSpro C++ -// -typedef signed char Int8; -typedef unsigned char UInt8; -typedef signed short Int16; -typedef unsigned short UInt16; -typedef signed int Int32; -typedef unsigned int UInt32; -typedef signed long IntPtr; -typedef unsigned long UIntPtr; -# if _MIPS_SZLONG == 64 -# define POCO_PTR_IS_64_BIT 1 -# define POCO_LONG_IS_64_BIT 1 -typedef signed long Int64; -typedef unsigned long UInt64; -# else -typedef signed long long Int64; -typedef unsigned long long UInt64; -# endif -# define POCO_HAVE_INT64 1 -#elif defined(_DIAB_TOOL) -typedef signed char Int8; -typedef unsigned char UInt8; -typedef signed short Int16; -typedef unsigned short UInt16; -typedef signed int Int32; -typedef unsigned int UInt32; -typedef signed long IntPtr; -typedef unsigned long UIntPtr; -typedef signed long long Int64; -typedef unsigned long long UInt64; -# define POCO_HAVE_INT64 1 #endif From 20eb1d83baf44b11aef6104801fb8119f56bd0bb Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 14 Feb 2023 22:09:22 +0000 Subject: [PATCH 219/717] Remove cygwin --- base/poco/Foundation/include/Poco/Platform.h | 3 --- base/poco/Foundation/src/Environment_UNIX.cpp | 4 ---- base/poco/Foundation/src/SharedLibrary_UNIX.cpp | 9 --------- base/poco/Foundation/src/Timezone_UNIX.cpp | 3 --- base/poco/Foundation/src/gzguts.h | 5 ----- base/poco/Foundation/src/zutil.h | 2 -- 6 files changed, 26 deletions(-) diff --git a/base/poco/Foundation/include/Poco/Platform.h b/base/poco/Foundation/include/Poco/Platform.h index 87fa440fa5f..f7afbcea42f 100644 --- a/base/poco/Foundation/include/Poco/Platform.h +++ b/base/poco/Foundation/include/Poco/Platform.h @@ -89,9 +89,6 @@ #elif defined(__QNX__) # define POCO_OS_FAMILY_UNIX 1 # define POCO_OS POCO_OS_QNX -#elif defined(__CYGWIN__) -# define POCO_OS_FAMILY_UNIX 1 -# define POCO_OS POCO_OS_CYGWIN #elif defined(POCO_VXWORKS) # define POCO_OS_FAMILY_UNIX 1 # define POCO_OS POCO_OS_VXWORKS diff --git a/base/poco/Foundation/src/Environment_UNIX.cpp b/base/poco/Foundation/src/Environment_UNIX.cpp index 04622dd1874..202e5d88f83 100644 --- a/base/poco/Foundation/src/Environment_UNIX.cpp +++ b/base/poco/Foundation/src/Environment_UNIX.cpp @@ -185,11 +185,7 @@ void EnvironmentImpl::nodeIdImpl(NodeId& id) /// #include #include #include -#ifndef __CYGWIN__ #include -#else // workaround for Cygwin, which does not have if_arp.h -#define ARPHRD_ETHER 1 /* Ethernet 10Mbps */ -#endif #include /// #include #include diff --git a/base/poco/Foundation/src/SharedLibrary_UNIX.cpp b/base/poco/Foundation/src/SharedLibrary_UNIX.cpp index 4a3dc2ffb0b..9dee3577e2f 100644 --- a/base/poco/Foundation/src/SharedLibrary_UNIX.cpp +++ b/base/poco/Foundation/src/SharedLibrary_UNIX.cpp @@ -18,9 +18,6 @@ // Note: cygwin is missing RTLD_LOCAL, set it to 0 -#if defined(__CYGWIN__) && !defined(RTLD_LOCAL) -#define RTLD_LOCAL 0 -#endif namespace Poco { @@ -111,12 +108,6 @@ std::string SharedLibraryImpl::suffixImpl() #else return ".sl"; #endif -#elif defined(__CYGWIN__) - #if defined(_DEBUG) && !defined(POCO_NO_SHARED_LIBRARY_DEBUG_SUFFIX) - return "d.dll"; - #else - return ".dll"; - #endif #else #if defined(_DEBUG) && !defined(POCO_NO_SHARED_LIBRARY_DEBUG_SUFFIX) return "d.so"; diff --git a/base/poco/Foundation/src/Timezone_UNIX.cpp b/base/poco/Foundation/src/Timezone_UNIX.cpp index 118b43465ec..152c91121ec 100644 --- a/base/poco/Foundation/src/Timezone_UNIX.cpp +++ b/base/poco/Foundation/src/Timezone_UNIX.cpp @@ -39,9 +39,6 @@ public: gmtime_r(&now, &t); std::time_t utc = std::mktime(&t); return now - utc; - #elif defined(__CYGWIN__) - tzset(); - return -_timezone; #else tzset(); return -timezone; diff --git a/base/poco/Foundation/src/gzguts.h b/base/poco/Foundation/src/gzguts.h index 488d328c5cd..834e7c8a7e1 100644 --- a/base/poco/Foundation/src/gzguts.h +++ b/base/poco/Foundation/src/gzguts.h @@ -61,11 +61,6 @@ # endif #endif -#if defined(__CYGWIN__) -# ifndef HAVE_VSNPRINTF -# define HAVE_VSNPRINTF -# endif -#endif #ifndef HAVE_VSNPRINTF # ifdef __TURBOC__ diff --git a/base/poco/Foundation/src/zutil.h b/base/poco/Foundation/src/zutil.h index 38223d36b6c..4deb4ec0db1 100644 --- a/base/poco/Foundation/src/zutil.h +++ b/base/poco/Foundation/src/zutil.h @@ -115,9 +115,7 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ #endif #ifdef WIN32 -# ifndef __CYGWIN__ /* Cygwin is Unix, not Win32 */ # define OS_CODE 0x0b -# endif #endif #ifdef __50SERIES /* Prime/PRIMOS */ From 1780bba532c2905f3f88ceb6133a1a7352110dda Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 14 Feb 2023 22:09:58 +0000 Subject: [PATCH 220/717] Remove vxworks --- base/poco/Foundation/include/Poco/Event.h | 2 - base/poco/Foundation/include/Poco/File.h | 2 - .../poco/Foundation/include/Poco/Foundation.h | 2 - base/poco/Foundation/include/Poco/Mutex.h | 2 - base/poco/Foundation/include/Poco/Platform.h | 3 - base/poco/Foundation/include/Poco/Process.h | 2 - base/poco/Foundation/include/Poco/RWLock.h | 2 - base/poco/Foundation/include/Poco/Semaphore.h | 2 - .../Foundation/include/Poco/SharedLibrary.h | 2 - base/poco/Foundation/include/Poco/Thread.h | 2 - .../Foundation/include/Poco/Thread_POSIX.h | 8 --- base/poco/Foundation/src/Clock.cpp | 35 ------------ base/poco/Foundation/src/Debugger.cpp | 6 -- .../Foundation/src/DirectoryIterator_UNIX.cpp | 8 --- base/poco/Foundation/src/Environment.cpp | 4 +- base/poco/Foundation/src/Event.cpp | 2 - base/poco/Foundation/src/Event_POSIX.cpp | 12 ---- base/poco/Foundation/src/File.cpp | 2 - base/poco/Foundation/src/FileStream_POSIX.cpp | 4 -- base/poco/Foundation/src/LocalDateTime.cpp | 5 -- base/poco/Foundation/src/Message.cpp | 6 -- base/poco/Foundation/src/Mutex.cpp | 2 - base/poco/Foundation/src/Mutex_POSIX.cpp | 31 +---------- base/poco/Foundation/src/Path_UNIX.cpp | 29 ---------- base/poco/Foundation/src/Process.cpp | 2 - base/poco/Foundation/src/RWLock.cpp | 2 - base/poco/Foundation/src/RWLock_VX.cpp | 7 --- base/poco/Foundation/src/Semaphore.cpp | 2 - base/poco/Foundation/src/Semaphore_POSIX.cpp | 12 ---- base/poco/Foundation/src/SharedLibrary.cpp | 2 - base/poco/Foundation/src/TemporaryFile.cpp | 6 -- base/poco/Foundation/src/Thread.cpp | 2 - base/poco/Foundation/src/Timestamp.cpp | 4 -- base/poco/Foundation/src/Timezone.cpp | 2 - base/poco/Net/include/Poco/Net/HostEntry.h | 3 - base/poco/Net/include/Poco/Net/SocketDefs.h | 54 ------------------ base/poco/Net/src/DNS.cpp | 14 ----- base/poco/Net/src/HostEntry.cpp | 11 ---- base/poco/Net/src/ICMPv4PacketImpl.cpp | 10 ---- base/poco/Net/src/NetworkInterface.cpp | 55 ------------------- base/poco/Net/src/SocketAddress.cpp | 4 -- base/poco/Net/src/SocketImpl.cpp | 28 ---------- .../poco/Util/include/Poco/Util/Application.h | 30 ---------- .../include/Poco/Util/ServerApplication.h | 30 +--------- .../include/Poco/Util/SystemConfiguration.h | 2 - base/poco/Util/src/ServerApplication.cpp | 48 ---------------- base/poco/Util/src/SystemConfiguration.cpp | 8 --- base/poco/XML/src/expat_config.h | 2 - 48 files changed, 4 insertions(+), 511 deletions(-) diff --git a/base/poco/Foundation/include/Poco/Event.h b/base/poco/Foundation/include/Poco/Event.h index 26c26d295a4..1f5ea71f1f1 100644 --- a/base/poco/Foundation/include/Poco/Event.h +++ b/base/poco/Foundation/include/Poco/Event.h @@ -24,8 +24,6 @@ #if defined(POCO_OS_FAMILY_WINDOWS) # include "Poco/Event_WIN32.h" -#elif defined(POCO_VXWORKS) -# include "Poco/Event_VX.h" #else # include "Poco/Event_POSIX.h" #endif diff --git a/base/poco/Foundation/include/Poco/File.h b/base/poco/Foundation/include/Poco/File.h index b2a61cedf75..cfc2c1f7e47 100644 --- a/base/poco/Foundation/include/Poco/File.h +++ b/base/poco/Foundation/include/Poco/File.h @@ -31,8 +31,6 @@ # endif #elif defined(POCO_OS_FAMILY_WINDOWS) # include "Poco/File_WIN32.h" -#elif defined(POCO_VXWORKS) -# include "Poco/File_VX.h" #elif defined(POCO_OS_FAMILY_UNIX) # include "Poco/File_UNIX.h" #endif diff --git a/base/poco/Foundation/include/Poco/Foundation.h b/base/poco/Foundation/include/Poco/Foundation.h index 23755b8e8b9..3001a5d4ccd 100644 --- a/base/poco/Foundation/include/Poco/Foundation.h +++ b/base/poco/Foundation/include/Poco/Foundation.h @@ -73,8 +73,6 @@ #include "Poco/Platform.h" #if defined(_WIN32) # include "Poco/Platform_WIN32.h" -#elif defined(POCO_VXWORKS) -# include "Poco/Platform_VX.h" #elif defined(POCO_OS_FAMILY_UNIX) # include "Poco/Platform_POSIX.h" #endif diff --git a/base/poco/Foundation/include/Poco/Mutex.h b/base/poco/Foundation/include/Poco/Mutex.h index de1bf2fbbaf..d976c8af2ba 100644 --- a/base/poco/Foundation/include/Poco/Mutex.h +++ b/base/poco/Foundation/include/Poco/Mutex.h @@ -29,8 +29,6 @@ # else # include "Poco/Mutex_WIN32.h" # endif -#elif defined(POCO_VXWORKS) -# include "Poco/Mutex_VX.h" #else # include "Poco/Mutex_POSIX.h" #endif diff --git a/base/poco/Foundation/include/Poco/Platform.h b/base/poco/Foundation/include/Poco/Platform.h index f7afbcea42f..081c8474046 100644 --- a/base/poco/Foundation/include/Poco/Platform.h +++ b/base/poco/Foundation/include/Poco/Platform.h @@ -89,9 +89,6 @@ #elif defined(__QNX__) # define POCO_OS_FAMILY_UNIX 1 # define POCO_OS POCO_OS_QNX -#elif defined(POCO_VXWORKS) -# define POCO_OS_FAMILY_UNIX 1 -# define POCO_OS POCO_OS_VXWORKS #elif defined(unix) || defined(__unix) || defined(__unix__) # define POCO_OS_FAMILY_UNIX 1 # define POCO_OS POCO_OS_UNKNOWN_UNIX diff --git a/base/poco/Foundation/include/Poco/Process.h b/base/poco/Foundation/include/Poco/Process.h index 6d4680335ff..c930a2240fd 100644 --- a/base/poco/Foundation/include/Poco/Process.h +++ b/base/poco/Foundation/include/Poco/Process.h @@ -29,8 +29,6 @@ # endif #elif defined(POCO_OS_FAMILY_WINDOWS) # include "Poco/Process_WIN32.h" -#elif defined(POCO_VXWORKS) -# include "Poco/Process_VX.h" #elif defined(POCO_OS_FAMILY_UNIX) # include "Poco/Process_UNIX.h" #endif diff --git a/base/poco/Foundation/include/Poco/RWLock.h b/base/poco/Foundation/include/Poco/RWLock.h index 1a4eb3efde2..292ff6afcbe 100644 --- a/base/poco/Foundation/include/Poco/RWLock.h +++ b/base/poco/Foundation/include/Poco/RWLock.h @@ -30,8 +30,6 @@ # endif #elif POCO_OS == POCO_OS_ANDROID # include "Poco/RWLock_Android.h" -#elif defined(POCO_VXWORKS) -# include "Poco/RWLock_VX.h" #else # include "Poco/RWLock_POSIX.h" #endif diff --git a/base/poco/Foundation/include/Poco/Semaphore.h b/base/poco/Foundation/include/Poco/Semaphore.h index 1382b532523..1e23ff37848 100644 --- a/base/poco/Foundation/include/Poco/Semaphore.h +++ b/base/poco/Foundation/include/Poco/Semaphore.h @@ -24,8 +24,6 @@ #if defined(POCO_OS_FAMILY_WINDOWS) # include "Poco/Semaphore_WIN32.h" -#elif defined(POCO_VXWORKS) -# include "Poco/Semaphore_VX.h" #else # include "Poco/Semaphore_POSIX.h" #endif diff --git a/base/poco/Foundation/include/Poco/SharedLibrary.h b/base/poco/Foundation/include/Poco/SharedLibrary.h index dd9f73bb5c0..387a9793f45 100644 --- a/base/poco/Foundation/include/Poco/SharedLibrary.h +++ b/base/poco/Foundation/include/Poco/SharedLibrary.h @@ -23,8 +23,6 @@ #if defined(hpux) || defined(_hpux) # include "Poco/SharedLibrary_HPUX.h" -#elif defined(POCO_VXWORKS) -# include "Poco/SharedLibrary_VX.h" #elif defined(POCO_OS_FAMILY_UNIX) # include "Poco/SharedLibrary_UNIX.h" #elif defined(POCO_OS_FAMILY_WINDOWS) && defined(POCO_WIN32_UTF8) diff --git a/base/poco/Foundation/include/Poco/Thread.h b/base/poco/Foundation/include/Poco/Thread.h index 49f82cc3713..1534b09b788 100644 --- a/base/poco/Foundation/include/Poco/Thread.h +++ b/base/poco/Foundation/include/Poco/Thread.h @@ -29,8 +29,6 @@ # else # include "Poco/Thread_WIN32.h" # endif -#elif defined(POCO_VXWORKS) -# include "Poco/Thread_VX.h" #else # include "Poco/Thread_POSIX.h" #endif diff --git a/base/poco/Foundation/include/Poco/Thread_POSIX.h b/base/poco/Foundation/include/Poco/Thread_POSIX.h index fb812bdad39..8dd107d554e 100644 --- a/base/poco/Foundation/include/Poco/Thread_POSIX.h +++ b/base/poco/Foundation/include/Poco/Thread_POSIX.h @@ -32,9 +32,6 @@ # include #endif #include -#if defined(POCO_VXWORKS) -# include -#endif namespace Poco @@ -116,11 +113,6 @@ private: , started(false) , joined(false) { -#if defined(POCO_VXWORKS) - // This workaround is for VxWorks 5.x where - // pthread_init() won't properly initialize the thread. - std::memset(&thread, 0, sizeof(thread)); -#endif } SharedPtr pRunnableTarget; diff --git a/base/poco/Foundation/src/Clock.cpp b/base/poco/Foundation/src/Clock.cpp index e52cd5db453..169f2a09805 100644 --- a/base/poco/Foundation/src/Clock.cpp +++ b/base/poco/Foundation/src/Clock.cpp @@ -21,8 +21,6 @@ #elif defined(POCO_OS_FAMILY_UNIX) #include #include -#elif defined(POCO_VXWORKS) -#include #elif defined(POCO_OS_FAMILY_WINDOWS) #include "Poco/UnWindows.h" #endif @@ -115,18 +113,6 @@ void Clock::update() _clock = ClockVal(ts.tv_sec)*resolution() + ts.tv_nsec/1000; -#elif defined(POCO_VXWORKS) - - struct timespec ts; -#if defined(CLOCK_MONOTONIC) // should be in VxWorks 6.x - if (clock_gettime(CLOCK_MONOTONIC, &ts)) - throw SystemException("cannot get system clock"); -#else - if (clock_gettime(CLOCK_REALTIME, &ts)) - throw SystemException("cannot get system clock"); -#endif - _clock = ClockVal(ts.tv_sec)*resolution() + ts.tv_nsec/1000; - #elif defined(POCO_HAVE_CLOCK_GETTIME) struct timespec ts; @@ -168,19 +154,6 @@ Clock::ClockDiff Clock::accuracy() ClockVal acc = nanosecs/1000; return acc > 0 ? acc : 1; -#elif defined(POCO_VXWORKS) - - struct timespec ts; -#if defined(CLOCK_MONOTONIC) // should be in VxWorks 6.x - if (clock_getres(CLOCK_MONOTONIC, &ts)) - throw SystemException("cannot get system clock"); -#else - if (clock_getres(CLOCK_REALTIME, &ts)) - throw SystemException("cannot get system clock"); -#endif - ClockVal acc = ClockVal(ts.tv_sec)*resolution() + ts.tv_nsec/1000; - return acc > 0 ? acc : 1; - #elif defined(_POSIX_TIMERS) && defined(_POSIX_MONOTONIC_CLOCK) struct timespec ts; @@ -208,14 +181,6 @@ bool Clock::monotonic() return true; -#elif defined(POCO_VXWORKS) - -#if defined(CLOCK_MONOTONIC) // should be in VxWorks 6.x - return true; -#else - return false; -#endif - #elif defined(_POSIX_TIMERS) && defined(_POSIX_MONOTONIC_CLOCK) return true; diff --git a/base/poco/Foundation/src/Debugger.cpp b/base/poco/Foundation/src/Debugger.cpp index c9efe5eac1f..16cc146d638 100644 --- a/base/poco/Foundation/src/Debugger.cpp +++ b/base/poco/Foundation/src/Debugger.cpp @@ -53,8 +53,6 @@ bool Debugger::isAvailable() #else return IsDebuggerPresent() ? true : false; #endif - #elif defined(POCO_VXWORKS) - return false; #elif defined(POCO_OS_FAMILY_UNIX) return std::getenv("POCO_ENABLE_DEBUGGER") ? true : false; #endif @@ -106,10 +104,6 @@ void Debugger::enter() { DebugBreak(); } - #elif defined(POCO_VXWORKS) - { - // not supported - } #elif defined(POCO_OS_FAMILY_UNIX) if (isAvailable()) { diff --git a/base/poco/Foundation/src/DirectoryIterator_UNIX.cpp b/base/poco/Foundation/src/DirectoryIterator_UNIX.cpp index 16be5051eab..6f5e3fd8175 100644 --- a/base/poco/Foundation/src/DirectoryIterator_UNIX.cpp +++ b/base/poco/Foundation/src/DirectoryIterator_UNIX.cpp @@ -13,11 +13,7 @@ #include "Poco/DirectoryIterator_UNIX.h" -#if defined(POCO_VXWORKS) -#include "Poco/File_VX.h" -#else #include "Poco/File_UNIX.h" -#endif #include "Poco/Path.h" @@ -29,11 +25,7 @@ DirectoryIteratorImpl::DirectoryIteratorImpl(const std::string& path): _pDir(0), Path p(path); p.makeFile(); -#if defined(POCO_VXWORKS) - _pDir = opendir(const_cast(p.toString().c_str())); -#else _pDir = opendir(p.toString().c_str()); -#endif if (!_pDir) File::handleLastError(path); next(); diff --git a/base/poco/Foundation/src/Environment.cpp b/base/poco/Foundation/src/Environment.cpp index 32753efc5ab..50a3469820d 100644 --- a/base/poco/Foundation/src/Environment.cpp +++ b/base/poco/Foundation/src/Environment.cpp @@ -18,9 +18,7 @@ #include // sprintf() -#if defined(POCO_VXWORKS) -#include "Environment_VX.cpp" -#elif defined(POCO_OS_FAMILY_UNIX) +#if defined(POCO_OS_FAMILY_UNIX) #include "Environment_UNIX.cpp" #elif defined(POCO_OS_FAMILY_WINDOWS) #if defined(_WIN32_WCE) diff --git a/base/poco/Foundation/src/Event.cpp b/base/poco/Foundation/src/Event.cpp index c8e8fa0b9e1..e042f53be97 100644 --- a/base/poco/Foundation/src/Event.cpp +++ b/base/poco/Foundation/src/Event.cpp @@ -17,8 +17,6 @@ #if defined(POCO_OS_FAMILY_WINDOWS) #include "Event_WIN32.cpp" -#elif defined(POCO_VXWORKS) -#include "Event_VX.cpp" #else #include "Event_POSIX.cpp" #endif diff --git a/base/poco/Foundation/src/Event_POSIX.cpp b/base/poco/Foundation/src/Event_POSIX.cpp index 50fb6f78a82..62047a50b3d 100644 --- a/base/poco/Foundation/src/Event_POSIX.cpp +++ b/base/poco/Foundation/src/Event_POSIX.cpp @@ -13,13 +13,8 @@ #include "Poco/Event_POSIX.h" -#if defined(POCO_VXWORKS) -#include -#include -#else #include #include -#endif // @@ -49,13 +44,6 @@ namespace Poco { EventImpl::EventImpl(bool autoReset): _auto(autoReset), _state(false) { -#if defined(POCO_VXWORKS) - // This workaround is for VxWorks 5.x where - // pthread_mutex_init() won't properly initialize the mutex - // resulting in a subsequent freeze in pthread_mutex_destroy() - // if the mutex has never been used. - std::memset(&_mutex, 0, sizeof(_mutex)); -#endif if (pthread_mutex_init(&_mutex, NULL)) throw SystemException("cannot create event (mutex)"); diff --git a/base/poco/Foundation/src/File.cpp b/base/poco/Foundation/src/File.cpp index ec198d13909..fe719338d4d 100644 --- a/base/poco/Foundation/src/File.cpp +++ b/base/poco/Foundation/src/File.cpp @@ -25,8 +25,6 @@ #endif #elif defined(POCO_OS_FAMILY_WINDOWS) #include "File_WIN32.cpp" -#elif defined(POCO_VXWORKS) -#include "File_VX.cpp" #elif defined(POCO_OS_FAMILY_UNIX) #include "File_UNIX.cpp" #endif diff --git a/base/poco/Foundation/src/FileStream_POSIX.cpp b/base/poco/Foundation/src/FileStream_POSIX.cpp index 848d7c367e2..1cec74e2ceb 100644 --- a/base/poco/Foundation/src/FileStream_POSIX.cpp +++ b/base/poco/Foundation/src/FileStream_POSIX.cpp @@ -89,11 +89,7 @@ int FileStreamBuf::writeToDevice(const char* buffer, std::streamsize length) { if (_fd == -1) return -1; -#if defined(POCO_VXWORKS) - int n = write(_fd, const_cast(buffer), length); -#else int n = write(_fd, buffer, length); -#endif if (n == -1) File::handleLastError(_path); _pos += n; diff --git a/base/poco/Foundation/src/LocalDateTime.cpp b/base/poco/Foundation/src/LocalDateTime.cpp index 720f41a4d27..29093290f76 100644 --- a/base/poco/Foundation/src/LocalDateTime.cpp +++ b/base/poco/Foundation/src/LocalDateTime.cpp @@ -270,13 +270,8 @@ void LocalDateTime::determineTzd(bool adjust) _tzd = (Timezone::utcOffset() + ((broken->tm_isdst == 1) ? 3600 : 0)); #else std::tm broken; -#if defined(POCO_VXWORKS) - if (localtime_r(&epochTime, &broken) != OK) - throw Poco::SystemException("cannot get local time"); -#else if (!localtime_r(&epochTime, &broken)) throw Poco::SystemException("cannot get local time"); -#endif _tzd = (Timezone::utcOffset() + ((broken.tm_isdst == 1) ? 3600 : 0)); #endif adjustForTzd(); diff --git a/base/poco/Foundation/src/Message.cpp b/base/poco/Foundation/src/Message.cpp index e7e86c4b34a..0dfe4323134 100644 --- a/base/poco/Foundation/src/Message.cpp +++ b/base/poco/Foundation/src/Message.cpp @@ -14,9 +14,7 @@ #include "Poco/Message.h" #include "Poco/Exception.h" -#if !defined(POCO_VXWORKS) #include "Poco/Process.h" -#endif #include "Poco/Thread.h" #include @@ -109,9 +107,7 @@ Message::~Message() void Message::init() { -#if !defined(POCO_VXWORKS) _pid = Process::id(); -#endif Thread* pThread = Thread::current(); if (pThread) { @@ -247,10 +243,8 @@ const std::string& Message::get(const std::string& param, const std::string& def long Message::getPid() const { -#if !defined(POCO_VXWORKS) if (_pid < 0) _pid = Process::id(); -#endif return _pid; } diff --git a/base/poco/Foundation/src/Mutex.cpp b/base/poco/Foundation/src/Mutex.cpp index cb101150880..55619846d35 100644 --- a/base/poco/Foundation/src/Mutex.cpp +++ b/base/poco/Foundation/src/Mutex.cpp @@ -21,8 +21,6 @@ #else #include "Mutex_WIN32.cpp" #endif -#elif defined(POCO_VXWORKS) -#include "Mutex_VX.cpp" #else #include "Mutex_POSIX.cpp" #endif diff --git a/base/poco/Foundation/src/Mutex_POSIX.cpp b/base/poco/Foundation/src/Mutex_POSIX.cpp index 56132ef5d58..183dbdffe9b 100644 --- a/base/poco/Foundation/src/Mutex_POSIX.cpp +++ b/base/poco/Foundation/src/Mutex_POSIX.cpp @@ -18,12 +18,7 @@ #include #endif #include -#if defined(POCO_VXWORKS) -#include -#include -#else #include -#endif #if defined(_POSIX_TIMEOUTS) && (_POSIX_TIMEOUTS - 200112L) >= 0L @@ -47,18 +42,11 @@ namespace Poco { MutexImpl::MutexImpl() { -#if defined(POCO_VXWORKS) - // This workaround is for VxWorks 5.x where - // pthread_mutex_init() won't properly initialize the mutex - // resulting in a subsequent freeze in pthread_mutex_destroy() - // if the mutex has never been used. - std::memset(&_mutex, 0, sizeof(_mutex)); -#endif pthread_mutexattr_t attr; pthread_mutexattr_init(&attr); #if defined(PTHREAD_MUTEX_RECURSIVE_NP) pthread_mutexattr_settype_np(&attr, PTHREAD_MUTEX_RECURSIVE_NP); -#elif !defined(POCO_VXWORKS) +#else pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); #endif if (pthread_mutex_init(&_mutex, &attr)) @@ -72,18 +60,11 @@ MutexImpl::MutexImpl() MutexImpl::MutexImpl(bool fast) { -#if defined(POCO_VXWORKS) - // This workaround is for VxWorks 5.x where - // pthread_mutex_init() won't properly initialize the mutex - // resulting in a subsequent freeze in pthread_mutex_destroy() - // if the mutex has never been used. - std::memset(&_mutex, 0, sizeof(_mutex)); -#endif pthread_mutexattr_t attr; pthread_mutexattr_init(&attr); #if defined(PTHREAD_MUTEX_RECURSIVE_NP) pthread_mutexattr_settype_np(&attr, fast ? PTHREAD_MUTEX_NORMAL_NP : PTHREAD_MUTEX_RECURSIVE_NP); -#elif !defined(POCO_VXWORKS) +#else pthread_mutexattr_settype(&attr, fast ? PTHREAD_MUTEX_NORMAL : PTHREAD_MUTEX_RECURSIVE); #endif if (pthread_mutex_init(&_mutex, &attr)) @@ -143,18 +124,10 @@ bool MutexImpl::tryLockImpl(long milliseconds) return true; else if (rc != EBUSY) throw SystemException("cannot lock mutex"); -#if defined(POCO_VXWORKS) - struct timespec ts; - ts.tv_sec = 0; - ts.tv_nsec = sleepMillis*1000000; - nanosleep(&ts, NULL); - -#else struct timeval tv; tv.tv_sec = 0; tv.tv_usec = sleepMillis * 1000; select(0, NULL, NULL, NULL, &tv); -#endif } while (!now.isElapsed(diff)); return false; diff --git a/base/poco/Foundation/src/Path_UNIX.cpp b/base/poco/Foundation/src/Path_UNIX.cpp index a6096f89d89..957a62db180 100644 --- a/base/poco/Foundation/src/Path_UNIX.cpp +++ b/base/poco/Foundation/src/Path_UNIX.cpp @@ -19,9 +19,7 @@ #include #include #include -#if !defined(POCO_VXWORKS) #include -#endif #include @@ -49,12 +47,6 @@ std::string PathImpl::currentImpl() std::string PathImpl::homeImpl() { -#if defined(POCO_VXWORKS) - if (EnvironmentImpl::hasImpl("HOME")) - return EnvironmentImpl::getImpl("HOME"); - else - return "/"; -#else std::string path; #if defined(_POSIX_C_SOURCE) || defined(_BSD_SOURCE) || defined(_POSIX_C_SOURCE) size_t buf_size = 1024; // Same as glibc use for getpwuid @@ -83,15 +75,11 @@ std::string PathImpl::homeImpl() std::string::size_type n = path.size(); if (n > 0 && path[n - 1] != '/') path.append("/"); return path; -#endif } std::string PathImpl::configHomeImpl() { -#if defined(POCO_VXWORKS) - return PathImpl::homeImpl(); -#else std::string path = PathImpl::homeImpl(); std::string::size_type n = path.size(); if (n > 0 && path[n - 1] == '/') @@ -102,15 +90,11 @@ std::string PathImpl::configHomeImpl() #endif return path; -#endif } std::string PathImpl::dataHomeImpl() { -#if defined(POCO_VXWORKS) - return PathImpl::homeImpl(); -#else std::string path = PathImpl::homeImpl(); std::string::size_type n = path.size(); if (n > 0 && path[n - 1] == '/') @@ -121,15 +105,11 @@ std::string PathImpl::dataHomeImpl() #endif return path; -#endif } std::string PathImpl::cacheHomeImpl() { -#if defined(POCO_VXWORKS) - return PathImpl::tempImpl(); -#else std::string path = PathImpl::homeImpl(); std::string::size_type n = path.size(); if (n > 0 && path[n - 1] == '/') @@ -140,15 +120,11 @@ std::string PathImpl::cacheHomeImpl() #endif return path; -#endif } std::string PathImpl::tempHomeImpl() { -#if defined(POCO_VXWORKS) - return PathImpl::tempImpl(); -#else std::string path = PathImpl::homeImpl(); std::string::size_type n = path.size(); if (n > 0 && path[n - 1] == '/') @@ -159,7 +135,6 @@ std::string PathImpl::tempHomeImpl() #endif return path; -#endif } @@ -196,11 +171,7 @@ std::string PathImpl::configImpl() std::string PathImpl::nullImpl() { -#if defined(POCO_VXWORKS) - return "/null"; -#else return "/dev/null"; -#endif } diff --git a/base/poco/Foundation/src/Process.cpp b/base/poco/Foundation/src/Process.cpp index 6cc0b7e6b9d..b864f030d5d 100644 --- a/base/poco/Foundation/src/Process.cpp +++ b/base/poco/Foundation/src/Process.cpp @@ -55,8 +55,6 @@ namespace #endif #elif defined(POCO_OS_FAMILY_WINDOWS) #include "Process_WIN32.cpp" -#elif defined(POCO_VXWORKS) -#include "Process_VX.cpp" #elif defined(POCO_OS_FAMILY_UNIX) #include "Process_UNIX.cpp" #endif diff --git a/base/poco/Foundation/src/RWLock.cpp b/base/poco/Foundation/src/RWLock.cpp index ba3f7d14253..da679bf1b51 100644 --- a/base/poco/Foundation/src/RWLock.cpp +++ b/base/poco/Foundation/src/RWLock.cpp @@ -23,8 +23,6 @@ #endif #elif POCO_OS == POCO_OS_ANDROID #include "RWLock_Android.cpp" -#elif defined(POCO_VXWORKS) -#include "RWLock_VX.cpp" #else #include "RWLock_POSIX.cpp" #endif diff --git a/base/poco/Foundation/src/RWLock_VX.cpp b/base/poco/Foundation/src/RWLock_VX.cpp index 44167d97e64..6b7eb166b79 100644 --- a/base/poco/Foundation/src/RWLock_VX.cpp +++ b/base/poco/Foundation/src/RWLock_VX.cpp @@ -21,13 +21,6 @@ namespace Poco { RWLockImpl::RWLockImpl() { -#if defined(POCO_VXWORKS) - // This workaround is for VxWorks 5.x where - // pthread_mutex_init() won't properly initialize the mutex - // resulting in a subsequent freeze in pthread_mutex_destroy() - // if the mutex has never been used. - std::memset(&_mutex, 0, sizeof(_mutex)); -#endif pthread_mutexattr_t attr; pthread_mutexattr_init(&attr); if (pthread_mutex_init(&_mutex, &attr)) diff --git a/base/poco/Foundation/src/Semaphore.cpp b/base/poco/Foundation/src/Semaphore.cpp index 3403125bdcc..85782cb821c 100644 --- a/base/poco/Foundation/src/Semaphore.cpp +++ b/base/poco/Foundation/src/Semaphore.cpp @@ -17,8 +17,6 @@ #if defined(POCO_OS_FAMILY_WINDOWS) #include "Semaphore_WIN32.cpp" -#elif defined(POCO_VXWORKS) -#include "Semaphore_VX.cpp" #else #include "Semaphore_POSIX.cpp" #endif diff --git a/base/poco/Foundation/src/Semaphore_POSIX.cpp b/base/poco/Foundation/src/Semaphore_POSIX.cpp index f4818af2004..966aa369ed7 100644 --- a/base/poco/Foundation/src/Semaphore_POSIX.cpp +++ b/base/poco/Foundation/src/Semaphore_POSIX.cpp @@ -13,13 +13,8 @@ #include "Poco/Semaphore_POSIX.h" -#if defined(POCO_VXWORKS) -#include -#include -#else #include #include -#endif // @@ -51,13 +46,6 @@ SemaphoreImpl::SemaphoreImpl(int n, int max): _n(n), _max(max) { poco_assert (n >= 0 && max > 0 && n <= max); -#if defined(POCO_VXWORKS) - // This workaround is for VxWorks 5.x where - // pthread_mutex_init() won't properly initialize the mutex - // resulting in a subsequent freeze in pthread_mutex_destroy() - // if the mutex has never been used. - std::memset(&_mutex, 0, sizeof(_mutex)); -#endif if (pthread_mutex_init(&_mutex, NULL)) throw SystemException("cannot create semaphore (mutex)"); diff --git a/base/poco/Foundation/src/SharedLibrary.cpp b/base/poco/Foundation/src/SharedLibrary.cpp index 542f9e81029..478bbbaf831 100644 --- a/base/poco/Foundation/src/SharedLibrary.cpp +++ b/base/poco/Foundation/src/SharedLibrary.cpp @@ -18,8 +18,6 @@ #if defined(hpux) || defined(_hpux) #include "SharedLibrary_HPUX.cpp" -#elif defined(POCO_VXWORKS) -#include "SharedLibrary_VX.cpp" #elif defined(POCO_OS_FAMILY_UNIX) #include "SharedLibrary_UNIX.cpp" #elif defined(POCO_OS_FAMILY_WINDOWS) && defined(POCO_WIN32_UTF8) diff --git a/base/poco/Foundation/src/TemporaryFile.cpp b/base/poco/Foundation/src/TemporaryFile.cpp index e89c970f337..da0e1aab6ad 100644 --- a/base/poco/Foundation/src/TemporaryFile.cpp +++ b/base/poco/Foundation/src/TemporaryFile.cpp @@ -15,9 +15,7 @@ #include "Poco/TemporaryFile.h" #include "Poco/Path.h" #include "Poco/Exception.h" -#if !defined(POCO_VXWORKS) #include "Poco/Process.h" -#endif #include "Poco/Mutex.h" #include #include @@ -150,11 +148,7 @@ std::string TemporaryFile::tempName(const std::string& tempDir) { name << Path::separator(); } -#if defined(POCO_VXWORKS) - name << "tmp"; -#else name << "tmp" << Process::id(); -#endif for (int i = 0; i < 6; ++i) { name << char('a' + (n % 26)); diff --git a/base/poco/Foundation/src/Thread.cpp b/base/poco/Foundation/src/Thread.cpp index 2f16bbed7f6..c3523bf7994 100644 --- a/base/poco/Foundation/src/Thread.cpp +++ b/base/poco/Foundation/src/Thread.cpp @@ -26,8 +26,6 @@ #else #include "Thread_WIN32.cpp" #endif -#elif defined(POCO_VXWORKS) -#include "Thread_VX.cpp" #else #include "Thread_POSIX.cpp" #endif diff --git a/base/poco/Foundation/src/Timestamp.cpp b/base/poco/Foundation/src/Timestamp.cpp index aca1e3494a3..a95964f65ba 100644 --- a/base/poco/Foundation/src/Timestamp.cpp +++ b/base/poco/Foundation/src/Timestamp.cpp @@ -22,12 +22,8 @@ #if defined(POCO_OS_FAMILY_UNIX) #include #include -#if defined(POCO_VXWORKS) -#include -#else #include #include -#endif #elif defined(POCO_OS_FAMILY_WINDOWS) #include "Poco/UnWindows.h" #if defined(_WIN32_WCE) diff --git a/base/poco/Foundation/src/Timezone.cpp b/base/poco/Foundation/src/Timezone.cpp index fa229cbb221..2e062dcc6b5 100644 --- a/base/poco/Foundation/src/Timezone.cpp +++ b/base/poco/Foundation/src/Timezone.cpp @@ -22,8 +22,6 @@ #else #include "Timezone_WIN32.cpp" #endif -#elif defined(POCO_VXWORKS) -#include "Timezone_VX.cpp" #else #include "Timezone_UNIX.cpp" #endif diff --git a/base/poco/Net/include/Poco/Net/HostEntry.h b/base/poco/Net/include/Poco/Net/HostEntry.h index 98a441e56bd..f2b8a6c09c0 100644 --- a/base/poco/Net/include/Poco/Net/HostEntry.h +++ b/base/poco/Net/include/Poco/Net/HostEntry.h @@ -50,9 +50,6 @@ namespace Net /// Creates the HostEntry from the data in an addrinfo structure. #endif -#if defined(POCO_VXWORKS) - HostEntry(const std::string & name, const IPAddress & addr); -#endif HostEntry(const HostEntry & entry); /// Creates the HostEntry by copying another one. diff --git a/base/poco/Net/include/Poco/Net/SocketDefs.h b/base/poco/Net/include/Poco/Net/SocketDefs.h index 83694ffba8c..0dc7e0e6012 100644 --- a/base/poco/Net/include/Poco/Net/SocketDefs.h +++ b/base/poco/Net/include/Poco/Net/SocketDefs.h @@ -73,60 +73,6 @@ # ifndef ADDRESS_FAMILY # define ADDRESS_FAMILY USHORT # endif -#elif defined(POCO_VXWORKS) -# include -# include -# include -# include -# include -# include -# include -# include -# define POCO_INVALID_SOCKET -1 -# define poco_socket_t int -# define poco_socklen_t int -# define poco_ioctl_request_t int -# define poco_closesocket(s) ::close(s) -# define POCO_EINTR EINTR -# define POCO_EACCES EACCES -# define POCO_EFAULT EFAULT -# define POCO_EINVAL EINVAL -# define POCO_EMFILE EMFILE -# define POCO_EAGAIN EAGAIN -# define POCO_EWOULDBLOCK EWOULDBLOCK -# define POCO_EINPROGRESS EINPROGRESS -# define POCO_EALREADY EALREADY -# define POCO_ENOTSOCK ENOTSOCK -# define POCO_EDESTADDRREQ EDESTADDRREQ -# define POCO_EMSGSIZE EMSGSIZE -# define POCO_EPROTOTYPE EPROTOTYPE -# define POCO_ENOPROTOOPT ENOPROTOOPT -# define POCO_EPROTONOSUPPORT EPROTONOSUPPORT -# define POCO_ESOCKTNOSUPPORT ESOCKTNOSUPPORT -# define POCO_ENOTSUP ENOTSUP -# define POCO_EPFNOSUPPORT EPFNOSUPPORT -# define POCO_EAFNOSUPPORT EAFNOSUPPORT -# define POCO_EADDRINUSE EADDRINUSE -# define POCO_EADDRNOTAVAIL EADDRNOTAVAIL -# define POCO_ENETDOWN ENETDOWN -# define POCO_ENETUNREACH ENETUNREACH -# define POCO_ENETRESET ENETRESET -# define POCO_ECONNABORTED ECONNABORTED -# define POCO_ECONNRESET ECONNRESET -# define POCO_ENOBUFS ENOBUFS -# define POCO_EISCONN EISCONN -# define POCO_ENOTCONN ENOTCONN -# define POCO_ESHUTDOWN ESHUTDOWN -# define POCO_ETIMEDOUT ETIMEDOUT -# define POCO_ECONNREFUSED ECONNREFUSED -# define POCO_EHOSTDOWN EHOSTDOWN -# define POCO_EHOSTUNREACH EHOSTUNREACH -# define POCO_ESYSNOTREADY -4 -# define POCO_ENOTINIT -5 -# define POCO_HOST_NOT_FOUND HOST_NOT_FOUND -# define POCO_TRY_AGAIN TRY_AGAIN -# define POCO_NO_RECOVERY NO_RECOVERY -# define POCO_NO_DATA NO_DATA #elif defined(POCO_OS_FAMILY_UNIX) # include # include diff --git a/base/poco/Net/src/DNS.cpp b/base/poco/Net/src/DNS.cpp index 9ed5a49cef2..a74292e40fd 100644 --- a/base/poco/Net/src/DNS.cpp +++ b/base/poco/Net/src/DNS.cpp @@ -87,12 +87,6 @@ HostEntry DNS::hostByName(const std::string& hostname, unsigned { aierror(rc, hostname); } -#elif defined(POCO_VXWORKS) - int addr = hostGetByName(const_cast(hostname.c_str())); - if (addr != ERROR) - { - return HostEntry(hostname, IPAddress(&addr, sizeof(addr))); - } #else struct hostent* he = gethostbyname(hostname.c_str()); if (he) @@ -141,12 +135,6 @@ HostEntry DNS::hostByAddress(const IPAddress& address, unsigned { aierror(rc, address.toString()); } -#elif defined(POCO_VXWORKS) - char name[MAXHOSTNAMELEN + 1]; - if (hostGetByAddr(*reinterpret_cast(address.addr()), name) == OK) - { - return HostEntry(std::string(name), address); - } #else struct hostent* he = gethostbyaddr(reinterpret_cast(address.addr()), address.length(), address.af()); if (he) @@ -332,8 +320,6 @@ int DNS::lastError() { #if defined(_WIN32) return GetLastError(); -#elif defined(POCO_VXWORKS) - return errno; #else return h_errno; #endif diff --git a/base/poco/Net/src/HostEntry.cpp b/base/poco/Net/src/HostEntry.cpp index 3257c703113..19d8d2ed73d 100644 --- a/base/poco/Net/src/HostEntry.cpp +++ b/base/poco/Net/src/HostEntry.cpp @@ -86,17 +86,6 @@ HostEntry::HostEntry(struct addrinfo* ainfo) #endif // POCO_HAVE_IPv6 -#if defined(POCO_VXWORKS) - - -HostEntry::HostEntry(const std::string& name, const IPAddress& addr): - _name(name) -{ - _addresses.push_back(addr); -} - - -#endif // POCO_VXWORKS HostEntry::HostEntry(const HostEntry& entry): diff --git a/base/poco/Net/src/ICMPv4PacketImpl.cpp b/base/poco/Net/src/ICMPv4PacketImpl.cpp index 7df987a6fdd..f38e4ad1ea0 100644 --- a/base/poco/Net/src/ICMPv4PacketImpl.cpp +++ b/base/poco/Net/src/ICMPv4PacketImpl.cpp @@ -17,9 +17,7 @@ #include "Poco/Timestamp.h" #include "Poco/Timespan.h" #include "Poco/NumberFormatter.h" -#if !defined(POCO_VXWORKS) #include "Poco/Process.h" -#endif #include @@ -131,11 +129,7 @@ void ICMPv4PacketImpl::initPacket() icp->code = 0; icp->checksum = 0; icp->seq = ++_seq; -#if defined(POCO_VXWORKS) - icp->id = 0; -#else icp->id = static_cast(Poco::Process::id()); -#endif struct timeval* ptp = (struct timeval *) (icp + 1); *ptp = time(); @@ -185,11 +179,7 @@ Poco::UInt8* ICMPv4PacketImpl::data(Poco::UInt8* buffer, int length) const bool ICMPv4PacketImpl::validReplyID(Poco::UInt8* buffer, int length) const { Header *icp = header(buffer, length); -#if defined(POCO_VXWORKS) - return icp && icp->id == 0; -#else return icp && (static_cast(Process::id()) == icp->id); -#endif } diff --git a/base/poco/Net/src/NetworkInterface.cpp b/base/poco/Net/src/NetworkInterface.cpp index e6279040e6e..e744de22277 100644 --- a/base/poco/Net/src/NetworkInterface.cpp +++ b/base/poco/Net/src/NetworkInterface.cpp @@ -1288,61 +1288,6 @@ NetworkInterface::Map NetworkInterface::map(bool ipOnly, bool upOnly) } } // namespace Poco::Net -#elif defined(POCO_VXWORKS) -// -// VxWorks -// - -#error TODO - -/* -namespace Poco { -namespace Net { - - -NetworkInterface::NetworkInterfaceList NetworkInterface::list() -{ - FastMutex::ScopedLock lock(_mutex); - NetworkInterfaceList result; - - int ifIndex = 1; - char ifName[32]; - char ifAddr[INET_ADDR_LEN]; - - for (;;) - { - if (ifIndexToIfName(ifIndex, ifName) == OK) - { - std::string name(ifName); - IPAddress addr; - IPAddress mask; - IPAddress bcst; - if (ifAddrGet(ifName, ifAddr) == OK) - { - addr = IPAddress(std::string(ifAddr)); - } - int ifMask; - if (ifMaskGet(ifName, &ifMask) == OK) - { - mask = IPAddress(&ifMask, sizeof(ifMask)); - } - if (ifBroadcastGet(ifName, ifAddr) == OK) - { - bcst = IPAddress(std::string(ifAddr)); - } - result.push_back(NetworkInterface(name, name, name, addr, mask, bcst)); - ifIndex++; - } - else break; - } - - return result; -} - - -} } // namespace Poco::Net -*/ - #elif defined(POCO_OS_FAMILY_BSD) || (POCO_OS == POCO_OS_QNX) || (POCO_OS == POCO_OS_SOLARIS) // // BSD variants, QNX(?) and Solaris diff --git a/base/poco/Net/src/SocketAddress.cpp b/base/poco/Net/src/SocketAddress.cpp index 42ea75e6c18..fb7c39373d6 100644 --- a/base/poco/Net/src/SocketAddress.cpp +++ b/base/poco/Net/src/SocketAddress.cpp @@ -391,15 +391,11 @@ Poco::UInt16 SocketAddress::resolveService(const std::string& service) } else { -#if defined(POCO_VXWORKS) - throw ServiceNotFoundException(service); -#else struct servent* se = getservbyname(service.c_str(), NULL); if (se) return ntohs(se->s_port); else throw ServiceNotFoundException(service); -#endif } } diff --git a/base/poco/Net/src/SocketImpl.cpp b/base/poco/Net/src/SocketImpl.cpp index 160ff6bd62b..6fdcbc1af7d 100644 --- a/base/poco/Net/src/SocketImpl.cpp +++ b/base/poco/Net/src/SocketImpl.cpp @@ -130,11 +130,7 @@ void SocketImpl::connect(const SocketAddress& address) int rc; do { -#if defined(POCO_VXWORKS) - rc = ::connect(_sockfd, (sockaddr*) address.addr(), address.length()); -#else rc = ::connect(_sockfd, address.addr(), address.length()); -#endif } while (rc != 0 && lastError() == POCO_EINTR); if (rc != 0) @@ -154,11 +150,7 @@ void SocketImpl::connect(const SocketAddress& address, const Poco::Timespan& tim setBlocking(false); try { -#if defined(POCO_VXWORKS) - int rc = ::connect(_sockfd, (sockaddr*) address.addr(), address.length()); -#else int rc = ::connect(_sockfd, address.addr(), address.length()); -#endif if (rc != 0) { int err = lastError(); @@ -186,11 +178,7 @@ void SocketImpl::connectNB(const SocketAddress& address) init(address.af()); } setBlocking(false); -#if defined(POCO_VXWORKS) - int rc = ::connect(_sockfd, (sockaddr*) address.addr(), address.length()); -#else int rc = ::connect(_sockfd, address.addr(), address.length()); -#endif if (rc != 0) { int err = lastError(); @@ -216,11 +204,7 @@ void SocketImpl::bind(const SocketAddress& address, bool reuseAddress, bool reus setReuseAddress(true); if (reusePort) setReusePort(true); -#if defined(POCO_VXWORKS) - int rc = ::bind(_sockfd, (sockaddr*) address.addr(), address.length()); -#else int rc = ::bind(_sockfd, address.addr(), address.length()); -#endif if (rc != 0) error(address.toString()); } @@ -373,11 +357,7 @@ int SocketImpl::sendTo(const void* buffer, int length, const SocketAddress& addr do { if (_sockfd == POCO_INVALID_SOCKET) throw InvalidSocketException(); -#if defined(POCO_VXWORKS) - rc = ::sendto(_sockfd, (char*) buffer, length, flags, (sockaddr*) address.addr(), address.length()); -#else rc = ::sendto(_sockfd, reinterpret_cast(buffer), length, flags, address.addr(), address.length()); -#endif } while (_blocking && rc < 0 && lastError() == POCO_EINTR); if (rc < 0) error(); @@ -694,11 +674,7 @@ void SocketImpl::setRawOption(int level, int option, const void* value, poco_soc { if (_sockfd == POCO_INVALID_SOCKET) throw InvalidSocketException(); -#if defined(POCO_VXWORKS) - int rc = ::setsockopt(_sockfd, level, option, (char*) value, length); -#else int rc = ::setsockopt(_sockfd, level, option, reinterpret_cast(value), length); -#endif if (rc == -1) error(); } @@ -931,8 +907,6 @@ void SocketImpl::ioctl(poco_ioctl_request_t request, int& arg) { #if defined(_WIN32) int rc = ioctlsocket(_sockfd, request, reinterpret_cast(&arg)); -#elif defined(POCO_VXWORKS) - int rc = ::ioctl(_sockfd, request, (int) &arg); #else int rc = ::ioctl(_sockfd, request, &arg); #endif @@ -944,8 +918,6 @@ void SocketImpl::ioctl(poco_ioctl_request_t request, void* arg) { #if defined(_WIN32) int rc = ioctlsocket(_sockfd, request, reinterpret_cast(arg)); -#elif defined(POCO_VXWORKS) - int rc = ::ioctl(_sockfd, request, (int) arg); #else int rc = ::ioctl(_sockfd, request, arg); #endif diff --git a/base/poco/Util/include/Poco/Util/Application.h b/base/poco/Util/include/Poco/Util/Application.h index e9a82f359c5..0991fff7cb5 100644 --- a/base/poco/Util/include/Poco/Util/Application.h +++ b/base/poco/Util/include/Poco/Util/Application.h @@ -27,9 +27,6 @@ #include "Poco/Util/OptionSet.h" #include "Poco/Util/Subsystem.h" #include "Poco/Util/Util.h" -#if defined(POCO_VXWORKS) -# include -#endif #include #include @@ -501,33 +498,6 @@ namespace Util } \ return pApp->run(); \ } -#elif defined(POCO_VXWORKS) -# define POCO_APP_MAIN(App) \ - int pocoAppMain(const char * appName, ...) \ - { \ - std::vector args; \ - args.push_back(std::string(appName)); \ - va_list vargs; \ - va_start(vargs, appName); \ - const char * arg = va_arg(vargs, const char *); \ - while (arg) \ - { \ - args.push_back(std::string(arg)); \ - arg = va_arg(vargs, const char *); \ - } \ - va_end(vargs); \ - Poco::AutoPtr pApp = new App; \ - try \ - { \ - pApp->init(args); \ - } \ - catch (Poco::Exception & exc) \ - { \ - pApp->logger().log(exc); \ - return Poco::Util::Application::EXIT_CONFIG; \ - } \ - return pApp->run(); \ - } #else # define POCO_APP_MAIN(App) \ int main(int argc, char ** argv) \ diff --git a/base/poco/Util/include/Poco/Util/ServerApplication.h b/base/poco/Util/include/Poco/Util/ServerApplication.h index b125cc1901b..15294733078 100644 --- a/base/poco/Util/include/Poco/Util/ServerApplication.h +++ b/base/poco/Util/include/Poco/Util/ServerApplication.h @@ -169,9 +169,7 @@ namespace Util #endif private: -#if defined(POCO_VXWORKS) - static Poco::Event _terminate; -#elif defined(POCO_OS_FAMILY_UNIX) +#if defined(POCO_OS_FAMILY_UNIX) void handleDaemon(const std::string & name, const std::string & value); void handleUMask(const std::string & name, const std::string & value); void handlePidFile(const std::string & name, const std::string & value); @@ -243,32 +241,6 @@ namespace Util return Poco::Util::Application::EXIT_SOFTWARE; \ } \ } -#elif defined(POCO_VXWORKS) -# define POCO_SERVER_MAIN(App) \ - int pocoSrvMain(const char * appName, ...) \ - { \ - std::vector args; \ - args.push_back(std::string(appName)); \ - va_list vargs; \ - va_start(vargs, appName); \ - const char * arg = va_arg(vargs, const char *); \ - while (arg) \ - { \ - args.push_back(std::string(arg)); \ - arg = va_arg(vargs, const char *); \ - } \ - va_end(vargs); \ - try \ - { \ - App app; \ - return app.run(args); \ - } \ - catch (Poco::Exception & exc) \ - { \ - std::cerr << exc.displayText() << std::endl; \ - return Poco::Util::Application::EXIT_SOFTWARE; \ - } \ - } #else # define POCO_SERVER_MAIN(App) \ int main(int argc, char ** argv) \ diff --git a/base/poco/Util/include/Poco/Util/SystemConfiguration.h b/base/poco/Util/include/Poco/Util/SystemConfiguration.h index 3ebd7f80a13..a61a325b886 100644 --- a/base/poco/Util/include/Poco/Util/SystemConfiguration.h +++ b/base/poco/Util/include/Poco/Util/SystemConfiguration.h @@ -88,9 +88,7 @@ namespace Util static const std::string TEMPDIR; static const std::string CONFIGDIR; static const std::string DATETIME; -#if !defined(POCO_VXWORKS) static const std::string PID; -#endif static const std::string ENV; }; diff --git a/base/poco/Util/src/ServerApplication.cpp b/base/poco/Util/src/ServerApplication.cpp index dbfa5d703d9..9da5d457d2f 100644 --- a/base/poco/Util/src/ServerApplication.cpp +++ b/base/poco/Util/src/ServerApplication.cpp @@ -18,10 +18,8 @@ #include "Poco/Util/OptionException.h" #include "Poco/FileStream.h" #include "Poco/Exception.h" -#if !defined(POCO_VXWORKS) #include "Poco/Process.h" #include "Poco/NamedEvent.h" -#endif #include "Poco/NumberFormatter.h" #include "Poco/Logger.h" #include "Poco/String.h" @@ -515,52 +513,6 @@ int ServerApplication::run(int argc, wchar_t** argv) #endif // _WIN32_WCE -#elif defined(POCO_VXWORKS) -// -// VxWorks specific code -// -void ServerApplication::waitForTerminationRequest() -{ - _terminate.wait(); -} - - -int ServerApplication::run(int argc, char** argv) -{ - try - { - init(argc, argv); - } - catch (Exception& exc) - { - logger().log(exc); - return EXIT_CONFIG; - } - return run(); -} - - -int ServerApplication::run(const std::vector& args) -{ - try - { - init(args); - } - catch (Exception& exc) - { - logger().log(exc); - return EXIT_CONFIG; - } - return run(); -} - - -void ServerApplication::defineOptions(OptionSet& options) -{ - Application::defineOptions(options); -} - - #elif defined(POCO_OS_FAMILY_UNIX) diff --git a/base/poco/Util/src/SystemConfiguration.cpp b/base/poco/Util/src/SystemConfiguration.cpp index c53952ada11..04b36878a94 100644 --- a/base/poco/Util/src/SystemConfiguration.cpp +++ b/base/poco/Util/src/SystemConfiguration.cpp @@ -19,9 +19,7 @@ #include "Poco/DateTimeFormatter.h" #include "Poco/DateTimeFormat.h" #include "Poco/NumberFormatter.h" -#if !defined(POCO_VXWORKS) #include "Poco/Process.h" -#endif #include "Poco/Exception.h" #include @@ -48,9 +46,7 @@ const std::string SystemConfiguration::TEMPHOMEDIR = "system.tempHomeDir"; const std::string SystemConfiguration::TEMPDIR = "system.tempDir"; const std::string SystemConfiguration::CONFIGDIR = "system.configDir"; const std::string SystemConfiguration::DATETIME = "system.dateTime"; -#if !defined(POCO_VXWORKS) const std::string SystemConfiguration::PID = "system.pid"; -#endif const std::string SystemConfiguration::ENV = "system.env."; @@ -142,13 +138,11 @@ bool SystemConfiguration::getRaw(const std::string& key, std::string& value) con { value = Poco::DateTimeFormatter::format(Poco::DateTime(), Poco::DateTimeFormat::ISO8601_FORMAT); } -#if !defined(POCO_VXWORKS) else if (key == PID) { value = "0"; value = Poco::NumberFormatter::format(Poco::Process::id()); } -#endif else if (key.compare(0, ENV.size(), ENV) == 0) { return getEnv(key.substr(ENV.size()), value); @@ -186,9 +180,7 @@ void SystemConfiguration::enumerate(const std::string& key, Keys& range) const range.push_back("tempDir"); range.push_back("configDir"); range.push_back("dateTime"); -#if !defined(POCO_VXWORKS) range.push_back("pid"); -#endif range.push_back("env"); } } diff --git a/base/poco/XML/src/expat_config.h b/base/poco/XML/src/expat_config.h index 14383d0b261..bab3747f66b 100644 --- a/base/poco/XML/src/expat_config.h +++ b/base/poco/XML/src/expat_config.h @@ -17,9 +17,7 @@ #include "Poco/Platform.h" -#if !defined(POCO_VXWORKS) # include -#endif #include From cd7f6b7f0e25b5eb7b9d196adf0e9f44014e46fe Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 14 Feb 2023 22:10:22 +0000 Subject: [PATCH 221/717] Remove qnx --- .../Foundation/include/Poco/FPEnvironment.h | 2 - base/poco/Foundation/include/Poco/Platform.h | 3 - .../poco/Foundation/include/Poco/StreamUtil.h | 3 - base/poco/Foundation/src/FPEnvironment.cpp | 2 - base/poco/Foundation/src/Process_UNIX.cpp | 57 ------------------- 5 files changed, 67 deletions(-) diff --git a/base/poco/Foundation/include/Poco/FPEnvironment.h b/base/poco/Foundation/include/Poco/FPEnvironment.h index 0cc40e21041..62f96b76730 100644 --- a/base/poco/Foundation/include/Poco/FPEnvironment.h +++ b/base/poco/Foundation/include/Poco/FPEnvironment.h @@ -27,8 +27,6 @@ # include "Poco/FPEnvironment_DEC.h" #elif defined(sun) || defined(__sun) # include "Poco/FPEnvironment_SUN.h" -#elif defined(__QNX__) -# include "Poco/FPEnvironment_QNX.h" #elif defined(POCO_OS_FAMILY_UNIX) # include "Poco/FPEnvironment_C99.h" #elif defined(POCO_OS_FAMILY_WINDOWS) diff --git a/base/poco/Foundation/include/Poco/Platform.h b/base/poco/Foundation/include/Poco/Platform.h index 081c8474046..a0d88d096c9 100644 --- a/base/poco/Foundation/include/Poco/Platform.h +++ b/base/poco/Foundation/include/Poco/Platform.h @@ -86,9 +86,6 @@ #elif defined(sun) || defined(__sun) # define POCO_OS_FAMILY_UNIX 1 # define POCO_OS POCO_OS_SOLARIS -#elif defined(__QNX__) -# define POCO_OS_FAMILY_UNIX 1 -# define POCO_OS POCO_OS_QNX #elif defined(unix) || defined(__unix) || defined(__unix__) # define POCO_OS_FAMILY_UNIX 1 # define POCO_OS POCO_OS_UNKNOWN_UNIX diff --git a/base/poco/Foundation/include/Poco/StreamUtil.h b/base/poco/Foundation/include/Poco/StreamUtil.h index 7668ec4555d..fa1814a0f2e 100644 --- a/base/poco/Foundation/include/Poco/StreamUtil.h +++ b/base/poco/Foundation/include/Poco/StreamUtil.h @@ -73,9 +73,6 @@ #if !defined(POCO_IOS_INIT_HACK) // Microsoft Visual Studio with Dinkumware STL (but not STLport) -# if defined(__QNX__) && !defined(__GLIBCPP__) -# define POCO_IOS_INIT_HACK 1 -# endif #endif diff --git a/base/poco/Foundation/src/FPEnvironment.cpp b/base/poco/Foundation/src/FPEnvironment.cpp index 94a837f9cd6..70c8e68af5a 100644 --- a/base/poco/Foundation/src/FPEnvironment.cpp +++ b/base/poco/Foundation/src/FPEnvironment.cpp @@ -23,8 +23,6 @@ #include "FPEnvironment_DEC.cpp" #elif defined(sun) || defined(__sun) #include "FPEnvironment_SUN.cpp" -#elif defined(__QNX__) -#include "FPEnvironment_QNX.cpp" #elif defined(POCO_OS_FAMILY_UNIX) #include "FPEnvironment_C99.cpp" #elif defined(POCO_OS_FAMILY_WINDOWS) diff --git a/base/poco/Foundation/src/Process_UNIX.cpp b/base/poco/Foundation/src/Process_UNIX.cpp index 4671278e1d8..c470edee65f 100644 --- a/base/poco/Foundation/src/Process_UNIX.cpp +++ b/base/poco/Foundation/src/Process_UNIX.cpp @@ -25,11 +25,6 @@ #include -#if defined(__QNX__) -#include -#include -#include -#endif namespace Poco { @@ -90,59 +85,7 @@ void ProcessImpl::timesImpl(long& userTime, long& kernelTime) ProcessHandleImpl* ProcessImpl::launchImpl(const std::string& command, const ArgsImpl& args, const std::string& initialDirectory, Pipe* inPipe, Pipe* outPipe, Pipe* errPipe, const EnvImpl& env) { -#if defined(__QNX__) - if (initialDirectory.empty()) - { - /// use QNX's spawn system call which is more efficient than fork/exec. - char** argv = new char*[args.size() + 2]; - int i = 0; - argv[i++] = const_cast(command.c_str()); - for (ArgsImpl::const_iterator it = args.begin(); it != args.end(); ++it) - argv[i++] = const_cast(it->c_str()); - argv[i] = NULL; - struct inheritance inherit; - std::memset(&inherit, 0, sizeof(inherit)); - inherit.flags = SPAWN_ALIGN_DEFAULT | SPAWN_CHECK_SCRIPT | SPAWN_SEARCH_PATH; - int fdmap[3]; - fdmap[0] = inPipe ? inPipe->readHandle() : 0; - fdmap[1] = outPipe ? outPipe->writeHandle() : 1; - fdmap[2] = errPipe ? errPipe->writeHandle() : 2; - - char** envPtr = 0; - std::vector envChars; - std::vector envPtrs; - if (!env.empty()) - { - envChars = getEnvironmentVariablesBuffer(env); - envPtrs.reserve(env.size() + 1); - char* p = &envChars[0]; - while (*p) - { - envPtrs.push_back(p); - while (*p) ++p; - ++p; - } - envPtrs.push_back(0); - envPtr = &envPtrs[0]; - } - - int pid = spawn(command.c_str(), 3, fdmap, &inherit, argv, envPtr); - delete [] argv; - if (pid == -1) - throw SystemException("cannot spawn", command); - - if (inPipe) inPipe->close(Pipe::CLOSE_READ); - if (outPipe) outPipe->close(Pipe::CLOSE_WRITE); - if (errPipe) errPipe->close(Pipe::CLOSE_WRITE); - return new ProcessHandleImpl(pid); - } - else - { - return launchByForkExecImpl(command, args, initialDirectory, inPipe, outPipe, errPipe, env); - } -#else return launchByForkExecImpl(command, args, initialDirectory, inPipe, outPipe, errPipe, env); -#endif } From bc1f5f8464d0075f907bac64a499105c48b89683 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 14 Feb 2023 22:11:17 +0000 Subject: [PATCH 222/717] Remove DEC true unix --- base/poco/Foundation/src/Thread_POSIX.cpp | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/base/poco/Foundation/src/Thread_POSIX.cpp b/base/poco/Foundation/src/Thread_POSIX.cpp index ac3eefdeb70..7b1efe7a191 100644 --- a/base/poco/Foundation/src/Thread_POSIX.cpp +++ b/base/poco/Foundation/src/Thread_POSIX.cpp @@ -141,8 +141,6 @@ int ThreadImpl::getMinOSPriorityImpl(int policy) { #if defined(POCO_THREAD_PRIORITY_MIN) return POCO_THREAD_PRIORITY_MIN; -#elif defined(__digital__) - return PRI_OTHER_MIN; #else return sched_get_priority_min(policy); #endif @@ -153,8 +151,6 @@ int ThreadImpl::getMaxOSPriorityImpl(int policy) { #if defined(POCO_THREAD_PRIORITY_MAX) return POCO_THREAD_PRIORITY_MAX; -#elif defined(__digital__) - return PRI_OTHER_MAX; #else return sched_get_priority_max(policy); #endif @@ -268,13 +264,7 @@ ThreadImpl::TIDImpl ThreadImpl::currentTidImpl() void ThreadImpl::sleepImpl(long milliseconds) { -#if defined(__digital__) - // This is specific to DECThreads - struct timespec interval; - interval.tv_sec = milliseconds / 1000; - interval.tv_nsec = (milliseconds % 1000)*1000000; - pthread_delay_np(&interval); -#elif POCO_OS == POCO_OS_LINUX || POCO_OS == POCO_OS_ANDROID || POCO_OS == POCO_OS_MAC_OS_X || POCO_OS == POCO_OS_QNX || POCO_OS == POCO_OS_VXWORKS +#if POCO_OS == POCO_OS_LINUX || POCO_OS == POCO_OS_ANDROID || POCO_OS == POCO_OS_MAC_OS_X || POCO_OS == POCO_OS_QNX || POCO_OS == POCO_OS_VXWORKS Poco::Timespan remainingTime(1000*Poco::Timespan::TimeDiff(milliseconds)); int rc; do From ac14941b84d3b64ce15e5cb9ffdfedfbcf98894d Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 14 Feb 2023 22:12:22 +0000 Subject: [PATCH 223/717] Remove VMS --- .../include/Poco/FPEnvironment_DEC.h | 16 ------ base/poco/Foundation/include/Poco/Platform.h | 3 -- .../poco/Foundation/src/FPEnvironment_DEC.cpp | 51 ------------------- base/poco/XML/src/xmlrole.h | 5 -- 4 files changed, 75 deletions(-) diff --git a/base/poco/Foundation/include/Poco/FPEnvironment_DEC.h b/base/poco/Foundation/include/Poco/FPEnvironment_DEC.h index caaa0e09602..8f764750b5e 100644 --- a/base/poco/Foundation/include/Poco/FPEnvironment_DEC.h +++ b/base/poco/Foundation/include/Poco/FPEnvironment_DEC.h @@ -19,11 +19,7 @@ #include "Poco/Foundation.h" -#if defined(__VMS) -# include -#else # include -#endif namespace Poco @@ -42,19 +38,11 @@ protected: }; enum FlagImpl { -#if defined(__VMS) - FP_DIVIDE_BY_ZERO_IMPL = IEEE$M_STATUS_DZE, - FP_INEXACT_IMPL = IEEE$M_STATUS_INE, - FP_OVERFLOW_IMPL = IEEE$M_STATUS_OVF, - FP_UNDERFLOW_IMPL = IEEE$M_STATUS_UNF, - FP_INVALID_IMPL = IEEE$M_STATUS_INV -#else FP_DIVIDE_BY_ZERO_IMPL = IEEE_STATUS_DZE, FP_INEXACT_IMPL = IEEE_STATUS_INE, FP_OVERFLOW_IMPL = IEEE_STATUS_OVF, FP_UNDERFLOW_IMPL = IEEE_STATUS_UNF, FP_INVALID_IMPL = IEEE_STATUS_INV -#endif }; FPEnvironmentImpl(); FPEnvironmentImpl(const FPEnvironmentImpl & env); @@ -76,11 +64,7 @@ protected: static long double copySignImpl(long double target, long double source); private: -#if defined(__VMS) - struct _ieee _env; -#else unsigned long _env; -#endif }; diff --git a/base/poco/Foundation/include/Poco/Platform.h b/base/poco/Foundation/include/Poco/Platform.h index a0d88d096c9..0b95f12d960 100644 --- a/base/poco/Foundation/include/Poco/Platform.h +++ b/base/poco/Foundation/include/Poco/Platform.h @@ -95,9 +95,6 @@ #elif defined(_WIN32) || defined(_WIN64) # define POCO_OS_FAMILY_WINDOWS 1 # define POCO_OS POCO_OS_WINDOWS_NT -#elif defined(__VMS) -# define POCO_OS_FAMILY_VMS 1 -# define POCO_OS POCO_OS_VMS #endif diff --git a/base/poco/Foundation/src/FPEnvironment_DEC.cpp b/base/poco/Foundation/src/FPEnvironment_DEC.cpp index 037e28df76c..b5995f83bf5 100644 --- a/base/poco/Foundation/src/FPEnvironment_DEC.cpp +++ b/base/poco/Foundation/src/FPEnvironment_DEC.cpp @@ -22,9 +22,6 @@ #include #include #include -#if defined(__VMS) -#include -#endif #include "Poco/FPEnvironment_DEC.h" @@ -33,16 +30,7 @@ namespace Poco { FPEnvironmentImpl::FPEnvironmentImpl() { -#if defined(__VMS) - #pragma pointer_size save - #pragma pointer_size 32 - struct _ieee env; - sys$ieee_set_fp_control(0, 0, &env); - #pragma pointer_size restore - _env = env; -#else _env = ieee_get_fp_control(); -#endif } @@ -54,17 +42,7 @@ FPEnvironmentImpl::FPEnvironmentImpl(const FPEnvironmentImpl& env) FPEnvironmentImpl::~FPEnvironmentImpl() { -#if defined(__VMS) - #pragma pointer_size save - #pragma pointer_size 32 - struct _ieee mask; - mask.ieee$q_flags = 0xFFFFFFFFFFFFFFFF; - struct _ieee env = _env; - sys$ieee_set_fp_control(&mask, &env, 0); - #pragma pointer_size restore -#else ieee_set_fp_control(_env); -#endif } @@ -134,48 +112,19 @@ long double FPEnvironmentImpl::copySignImpl(long double target, long double sour void FPEnvironmentImpl::keepCurrentImpl() { -#if defined(__VMS) - #pragma pointer_size save - #pragma pointer_size 32 - struct _ieee env; - sys$ieee_set_fp_control(0, 0, &env); - #pragma pointer_size restore - _env = env; -#else ieee_set_fp_control(_env); -#endif } void FPEnvironmentImpl::clearFlagsImpl() { -#if defined(__VMS) - #pragma pointer_size save - #pragma pointer_size 32 - struct _ieee mask; - mask.ieee$q_flags = 0xFFFFFFFFFFFFFFFF; - struct _ieee clr; - clr.ieee$q_flags = 0; - sys$ieee_set_fp_control(&mask, &clr, 0); - #pragma pointer_size restore -#else ieee_set_fp_control(0); -#endif } bool FPEnvironmentImpl::isFlagImpl(FlagImpl flag) { -#if defined(__VMS) - #pragma pointer_size save - #pragma pointer_size 32 - struct _ieee flags; - sys$ieee_set_fp_control(0, 0, &flags); - return (flags.ieee$q_flags & flag) != 0; - #pragma pointer_size restore -#else return (ieee_get_fp_control() & flag) != 0; -#endif } diff --git a/base/poco/XML/src/xmlrole.h b/base/poco/XML/src/xmlrole.h index f33f859b2c4..11f0fca2232 100644 --- a/base/poco/XML/src/xmlrole.h +++ b/base/poco/XML/src/xmlrole.h @@ -36,11 +36,6 @@ #ifndef XmlRole_INCLUDED #define XmlRole_INCLUDED 1 -#ifdef __VMS -/* 0 1 2 3 0 1 2 3 - 1234567890123456789012345678901 1234567890123456789012345678901 */ -# define XmlPrologStateInitExternalEntity XmlPrologStateInitExternalEnt -#endif #include "xmltok.h" From 3f99daf86d061fa2dbf13efb7650dc08dd63e6f9 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 14 Feb 2023 22:13:11 +0000 Subject: [PATCH 224/717] Remove Windows --- base/poco/Crypto/include/Poco/Crypto/Crypto.h | 9 - base/poco/Crypto/src/OpenSSLInitializer.cpp | 16 - base/poco/Crypto/src/X509Certificate.cpp | 4 - .../Data/ODBC/include/Poco/Data/ODBC/Binder.h | 3 - .../include/Poco/Data/ODBC/ConnectionHandle.h | 3 - .../ODBC/include/Poco/Data/ODBC/Diagnostics.h | 3 - .../Poco/Data/ODBC/EnvironmentHandle.h | 3 - .../Data/ODBC/include/Poco/Data/ODBC/Error.h | 3 - .../ODBC/include/Poco/Data/ODBC/Extractor.h | 3 - .../Data/ODBC/include/Poco/Data/ODBC/Handle.h | 3 - .../Data/ODBC/include/Poco/Data/ODBC/ODBC.h | 10 - .../include/Poco/Data/ODBC/ODBCMetaColumn.h | 3 - .../Poco/Data/ODBC/ODBCStatementImpl.h | 3 - .../ODBC/include/Poco/Data/ODBC/Parameter.h | 3 - .../ODBC/include/Poco/Data/ODBC/Preparator.h | 3 - .../ODBC/include/Poco/Data/ODBC/SessionImpl.h | 3 - .../ODBC/include/Poco/Data/ODBC/TypeInfo.h | 3 - .../ODBC/include/Poco/Data/ODBC/Unicode.h | 8 +- base/poco/Data/ODBC/src/Binder.cpp | 12 - base/poco/Data/ODBC/src/ODBCStatementImpl.cpp | 3 - base/poco/Data/include/Poco/Data/Data.h | 7 - .../Foundation/include/Poco/ClassLibrary.h | 4 - base/poco/Foundation/include/Poco/Error.h | 8 - base/poco/Foundation/include/Poco/Event.h | 4 - .../Foundation/include/Poco/FPEnvironment.h | 2 - base/poco/Foundation/include/Poco/File.h | 10 +- .../poco/Foundation/include/Poco/FileStream.h | 4 - .../poco/Foundation/include/Poco/Foundation.h | 15 +- base/poco/Foundation/include/Poco/LogFile.h | 6 - base/poco/Foundation/include/Poco/Mutex.h | 8 - .../poco/Foundation/include/Poco/NamedEvent.h | 6 +- .../poco/Foundation/include/Poco/NamedMutex.h | 6 +- base/poco/Foundation/include/Poco/Path.h | 4 - base/poco/Foundation/include/Poco/PipeImpl.h | 8 +- base/poco/Foundation/include/Poco/Platform.h | 9 +- base/poco/Foundation/include/Poco/Process.h | 10 +- base/poco/Foundation/include/Poco/RWLock.h | 8 +- base/poco/Foundation/include/Poco/Semaphore.h | 4 - .../Foundation/include/Poco/SharedLibrary.h | 4 - base/poco/Foundation/include/Poco/Thread.h | 8 - base/poco/Foundation/include/Poco/Timestamp.h | 4 - base/poco/Foundation/include/Poco/Types.h | 8 - base/poco/Foundation/include/Poco/UTFString.h | 7 +- base/poco/Foundation/include/Poco/UnWindows.h | 5 - base/poco/Foundation/src/Clock.cpp | 33 +- base/poco/Foundation/src/Debugger.cpp | 42 +- .../poco/Foundation/src/DirectoryIterator.cpp | 6 +- base/poco/Foundation/src/Environment.cpp | 10 - base/poco/Foundation/src/Error.cpp | 29 -- base/poco/Foundation/src/Event.cpp | 4 - base/poco/Foundation/src/EventLogChannel.cpp | 16 - base/poco/Foundation/src/FPEnvironment.cpp | 2 - base/poco/Foundation/src/File.cpp | 10 +- base/poco/Foundation/src/FileStream.cpp | 4 - base/poco/Foundation/src/LogFile.cpp | 6 - base/poco/Foundation/src/LoggingFactory.cpp | 12 - base/poco/Foundation/src/Mutex.cpp | 8 - base/poco/Foundation/src/NamedEvent.cpp | 6 +- base/poco/Foundation/src/NamedMutex.cpp | 6 +- base/poco/Foundation/src/Path.cpp | 34 -- base/poco/Foundation/src/PipeImpl.cpp | 8 +- base/poco/Foundation/src/Process.cpp | 10 +- base/poco/Foundation/src/RWLock.cpp | 8 +- base/poco/Foundation/src/RandomStream.cpp | 13 +- base/poco/Foundation/src/Semaphore.cpp | 4 - base/poco/Foundation/src/SharedLibrary.cpp | 4 - base/poco/Foundation/src/SharedMemory.cpp | 2 - base/poco/Foundation/src/Thread.cpp | 8 - base/poco/Foundation/src/Timestamp.cpp | 59 +-- base/poco/Foundation/src/Timezone.cpp | 8 - base/poco/Foundation/src/gzguts.h | 3 - base/poco/Foundation/src/pcre.h | 13 - base/poco/Foundation/src/pcre_internal.h | 12 - base/poco/Foundation/src/utils.h | 19 - base/poco/JSON/include/Poco/JSON/JSON.h | 7 - .../MongoDB/include/Poco/MongoDB/MongoDB.h | 7 - base/poco/Net/include/Poco/Net/IPAddress.h | 4 - base/poco/Net/include/Poco/Net/Net.h | 28 -- .../Net/include/Poco/Net/NetworkInterface.h | 4 - base/poco/Net/include/Poco/Net/SocketDefs.h | 58 +-- base/poco/Net/include/Poco/Net/SocketImpl.h | 4 - base/poco/Net/src/DNS.cpp | 10 - base/poco/Net/src/HTTPServerConnection.cpp | 8 - base/poco/Net/src/IPAddress.cpp | 15 - base/poco/Net/src/IPAddressImpl.cpp | 95 ---- base/poco/Net/src/Net.cpp | 38 -- base/poco/Net/src/NetworkInterface.cpp | 412 +---------------- base/poco/Net/src/PollSet.cpp | 16 +- base/poco/Net/src/SocketImpl.cpp | 52 +-- base/poco/Redis/include/Poco/Redis/Redis.h | 7 - .../poco/Util/include/Poco/Util/Application.h | 18 - .../include/Poco/Util/ServerApplication.h | 57 --- base/poco/Util/include/Poco/Util/Util.h | 7 - base/poco/Util/src/Application.cpp | 22 - base/poco/Util/src/ServerApplication.cpp | 433 +----------------- base/poco/XML/include/Poco/XML/XML.h | 7 - base/poco/XML/src/XMLWriter.cpp | 4 - base/poco/XML/src/internal.h | 11 - base/poco/XML/src/xmlparse.cpp | 59 +-- 99 files changed, 34 insertions(+), 2008 deletions(-) diff --git a/base/poco/Crypto/include/Poco/Crypto/Crypto.h b/base/poco/Crypto/include/Poco/Crypto/Crypto.h index 90df2fed128..42dc8cb137c 100644 --- a/base/poco/Crypto/include/Poco/Crypto/Crypto.h +++ b/base/poco/Crypto/include/Poco/Crypto/Crypto.h @@ -66,15 +66,6 @@ enum RSAPaddingMode // Crypto_API functions as being imported from a DLL, whereas this DLL sees symbols // defined with this macro as being exported. // -#if defined(_WIN32) -# if defined(POCO_DLL) -# if defined(Crypto_EXPORTS) -# define Crypto_API __declspec(dllexport) -# else -# define Crypto_API __declspec(dllimport) -# endif -# endif -#endif #if !defined(Crypto_API) diff --git a/base/poco/Crypto/src/OpenSSLInitializer.cpp b/base/poco/Crypto/src/OpenSSLInitializer.cpp index 9aa44c2d976..31798e8dd7e 100644 --- a/base/poco/Crypto/src/OpenSSLInitializer.cpp +++ b/base/poco/Crypto/src/OpenSSLInitializer.cpp @@ -22,18 +22,6 @@ #if OPENSSL_VERSION_NUMBER >= 0x0907000L #include #endif -#if defined(POCO_OS_FAMILY_WINDOWS) - #define POCO_STR_HELPER(x) #x - #define POCO_STR(x) POCO_STR_HELPER(x) - #if defined POCO_INTERNAL_OPENSSL_MSVC_VER - #define POCO_INTERNAL_OPENSSL_BUILD \ - " (POCO internal build, MSVC version " \ - POCO_STR(POCO_INTERNAL_OPENSSL_MSVC_VER) ")" - #else - #define POCO_INTERNAL_OPENSSL_BUILD "" - #endif - #pragma message (OPENSSL_VERSION_TEXT POCO_INTERNAL_OPENSSL_BUILD) -#endif using Poco::RandomInputStream; @@ -88,7 +76,6 @@ void OpenSSLInitializer::initialize() int nMutexes = CRYPTO_num_locks(); _mutexes = new Poco::FastMutex[nMutexes]; CRYPTO_set_locking_callback(&OpenSSLInitializer::lock); -#ifndef POCO_OS_FAMILY_WINDOWS // Not needed on Windows (see SF #110: random unhandled exceptions when linking with ssl). // https://sourceforge.net/p/poco/bugs/110/ // @@ -97,7 +84,6 @@ void OpenSSLInitializer::initialize() // then a default implementation is used - on Windows and BeOS this uses the system's // default thread identifying APIs" CRYPTO_set_id_callback(&OpenSSLInitializer::id); -#endif CRYPTO_set_dynlock_create_callback(&OpenSSLInitializer::dynlockCreate); CRYPTO_set_dynlock_lock_callback(&OpenSSLInitializer::dynlock); CRYPTO_set_dynlock_destroy_callback(&OpenSSLInitializer::dynlockDestroy); @@ -112,9 +98,7 @@ void OpenSSLInitializer::uninitialize() EVP_cleanup(); ERR_free_strings(); CRYPTO_set_locking_callback(0); -#ifndef POCO_OS_FAMILY_WINDOWS CRYPTO_set_id_callback(0); -#endif delete [] _mutexes; CONF_modules_free(); diff --git a/base/poco/Crypto/src/X509Certificate.cpp b/base/poco/Crypto/src/X509Certificate.cpp index 542066665df..8c2ba698d7b 100644 --- a/base/poco/Crypto/src/X509Certificate.cpp +++ b/base/poco/Crypto/src/X509Certificate.cpp @@ -20,10 +20,6 @@ #include "Poco/Format.h" #include #include -#ifdef _WIN32 -// fix for WIN32 header conflict -#undef X509_NAME -#endif #include #include #include diff --git a/base/poco/Data/ODBC/include/Poco/Data/ODBC/Binder.h b/base/poco/Data/ODBC/include/Poco/Data/ODBC/Binder.h index f0c7aa47d93..3169563879c 100644 --- a/base/poco/Data/ODBC/include/Poco/Data/ODBC/Binder.h +++ b/base/poco/Data/ODBC/include/Poco/Data/ODBC/Binder.h @@ -31,9 +31,6 @@ #include "Poco/Data/ODBC/TypeInfo.h" #include "Poco/Data/ODBC/Utility.h" #include "Poco/Exception.h" -#ifdef POCO_OS_FAMILY_WINDOWS -# include -#endif #include diff --git a/base/poco/Data/ODBC/include/Poco/Data/ODBC/ConnectionHandle.h b/base/poco/Data/ODBC/include/Poco/Data/ODBC/ConnectionHandle.h index e5bfa5efe3e..362361af783 100644 --- a/base/poco/Data/ODBC/include/Poco/Data/ODBC/ConnectionHandle.h +++ b/base/poco/Data/ODBC/include/Poco/Data/ODBC/ConnectionHandle.h @@ -20,9 +20,6 @@ #include "Poco/Data/ODBC/EnvironmentHandle.h" #include "Poco/Data/ODBC/ODBC.h" -#ifdef POCO_OS_FAMILY_WINDOWS -# include -#endif #include diff --git a/base/poco/Data/ODBC/include/Poco/Data/ODBC/Diagnostics.h b/base/poco/Data/ODBC/include/Poco/Data/ODBC/Diagnostics.h index 343514c0626..135a368efad 100644 --- a/base/poco/Data/ODBC/include/Poco/Data/ODBC/Diagnostics.h +++ b/base/poco/Data/ODBC/include/Poco/Data/ODBC/Diagnostics.h @@ -22,9 +22,6 @@ #include #include "Poco/Data/ODBC/ODBC.h" #include "Poco/Data/ODBC/Utility.h" -#ifdef POCO_OS_FAMILY_WINDOWS -# include -#endif #include diff --git a/base/poco/Data/ODBC/include/Poco/Data/ODBC/EnvironmentHandle.h b/base/poco/Data/ODBC/include/Poco/Data/ODBC/EnvironmentHandle.h index 6535f4e34d1..adf567e4158 100644 --- a/base/poco/Data/ODBC/include/Poco/Data/ODBC/EnvironmentHandle.h +++ b/base/poco/Data/ODBC/include/Poco/Data/ODBC/EnvironmentHandle.h @@ -19,9 +19,6 @@ #include "Poco/Data/ODBC/ODBC.h" -#ifdef POCO_OS_FAMILY_WINDOWS -# include -#endif #include diff --git a/base/poco/Data/ODBC/include/Poco/Data/ODBC/Error.h b/base/poco/Data/ODBC/include/Poco/Data/ODBC/Error.h index 3335cbfc2b6..dcded9f217a 100644 --- a/base/poco/Data/ODBC/include/Poco/Data/ODBC/Error.h +++ b/base/poco/Data/ODBC/include/Poco/Data/ODBC/Error.h @@ -23,9 +23,6 @@ #include "Poco/Data/ODBC/ODBC.h" #include "Poco/Data/ODBC/Utility.h" #include "Poco/Format.h" -#ifdef POCO_OS_FAMILY_WINDOWS -# include -#endif #include diff --git a/base/poco/Data/ODBC/include/Poco/Data/ODBC/Extractor.h b/base/poco/Data/ODBC/include/Poco/Data/ODBC/Extractor.h index eb9ae0db019..82e2f895638 100644 --- a/base/poco/Data/ODBC/include/Poco/Data/ODBC/Extractor.h +++ b/base/poco/Data/ODBC/include/Poco/Data/ODBC/Extractor.h @@ -34,9 +34,6 @@ #include "Poco/Exception.h" #include "Poco/Nullable.h" #include "Poco/UTFString.h" -#ifdef POCO_OS_FAMILY_WINDOWS -# include -#endif #include diff --git a/base/poco/Data/ODBC/include/Poco/Data/ODBC/Handle.h b/base/poco/Data/ODBC/include/Poco/Data/ODBC/Handle.h index 41ffd1ce84c..0d1cee48f09 100644 --- a/base/poco/Data/ODBC/include/Poco/Data/ODBC/Handle.h +++ b/base/poco/Data/ODBC/include/Poco/Data/ODBC/Handle.h @@ -23,9 +23,6 @@ #include "Poco/Data/ODBC/ODBC.h" #include "Poco/Data/ODBC/ODBCException.h" #include "Poco/Data/ODBC/Utility.h" -#ifdef POCO_OS_FAMILY_WINDOWS -# include -#endif #include diff --git a/base/poco/Data/ODBC/include/Poco/Data/ODBC/ODBC.h b/base/poco/Data/ODBC/include/Poco/Data/ODBC/ODBC.h index 9cf91df6da5..4ae6d3aeb95 100644 --- a/base/poco/Data/ODBC/include/Poco/Data/ODBC/ODBC.h +++ b/base/poco/Data/ODBC/include/Poco/Data/ODBC/ODBC.h @@ -21,9 +21,6 @@ #include "Poco/Foundation.h" -#ifdef POCO_OS_FAMILY_WINDOWS -# include -#endif // @@ -34,13 +31,6 @@ // ODBC_API functions as being imported from a DLL, whereas this DLL sees symbols // defined with this macro as being exported. // -#if defined(_WIN32) && defined(POCO_DLL) -# if defined(ODBC_EXPORTS) -# define ODBC_API __declspec(dllexport) -# else -# define ODBC_API __declspec(dllimport) -# endif -#endif #if !defined(ODBC_API) diff --git a/base/poco/Data/ODBC/include/Poco/Data/ODBC/ODBCMetaColumn.h b/base/poco/Data/ODBC/include/Poco/Data/ODBC/ODBCMetaColumn.h index f8760532a14..4783427fefa 100644 --- a/base/poco/Data/ODBC/include/Poco/Data/ODBC/ODBCMetaColumn.h +++ b/base/poco/Data/ODBC/include/Poco/Data/ODBC/ODBCMetaColumn.h @@ -23,9 +23,6 @@ #include "Poco/Data/ODBC/Handle.h" #include "Poco/Data/ODBC/ODBC.h" #include "Poco/Data/ODBC/ODBCException.h" -#ifdef POCO_OS_FAMILY_WINDOWS -# include -#endif #include diff --git a/base/poco/Data/ODBC/include/Poco/Data/ODBC/ODBCStatementImpl.h b/base/poco/Data/ODBC/include/Poco/Data/ODBC/ODBCStatementImpl.h index 7ef7aaf7c0b..e50228a6d52 100644 --- a/base/poco/Data/ODBC/include/Poco/Data/ODBC/ODBCStatementImpl.h +++ b/base/poco/Data/ODBC/include/Poco/Data/ODBC/ODBCStatementImpl.h @@ -29,9 +29,6 @@ #include "Poco/Data/StatementImpl.h" #include "Poco/Format.h" #include "Poco/SharedPtr.h" -#ifdef POCO_OS_FAMILY_WINDOWS -# include -#endif #include diff --git a/base/poco/Data/ODBC/include/Poco/Data/ODBC/Parameter.h b/base/poco/Data/ODBC/include/Poco/Data/ODBC/Parameter.h index f32c52b17af..b5baee3b507 100644 --- a/base/poco/Data/ODBC/include/Poco/Data/ODBC/Parameter.h +++ b/base/poco/Data/ODBC/include/Poco/Data/ODBC/Parameter.h @@ -20,9 +20,6 @@ #include "Poco/Data/ODBC/Handle.h" #include "Poco/Data/ODBC/ODBC.h" -#ifdef POCO_OS_FAMILY_WINDOWS -# include -#endif #include diff --git a/base/poco/Data/ODBC/include/Poco/Data/ODBC/Preparator.h b/base/poco/Data/ODBC/include/Poco/Data/ODBC/Preparator.h index af58108cf14..f92e4ea7083 100644 --- a/base/poco/Data/ODBC/include/Poco/Data/ODBC/Preparator.h +++ b/base/poco/Data/ODBC/include/Poco/Data/ODBC/Preparator.h @@ -31,9 +31,6 @@ #include "Poco/DynamicAny.h" #include "Poco/SharedPtr.h" #include "Poco/UTFString.h" -#ifdef POCO_OS_FAMILY_WINDOWS -# include -#endif #include diff --git a/base/poco/Data/ODBC/include/Poco/Data/ODBC/SessionImpl.h b/base/poco/Data/ODBC/include/Poco/Data/ODBC/SessionImpl.h index cd47bc7518b..259f26afff8 100644 --- a/base/poco/Data/ODBC/include/Poco/Data/ODBC/SessionImpl.h +++ b/base/poco/Data/ODBC/include/Poco/Data/ODBC/SessionImpl.h @@ -27,9 +27,6 @@ #include "Poco/Data/ODBC/TypeInfo.h" #include "Poco/Mutex.h" #include "Poco/SharedPtr.h" -#ifdef POCO_OS_FAMILY_WINDOWS -# include -#endif #include diff --git a/base/poco/Data/ODBC/include/Poco/Data/ODBC/TypeInfo.h b/base/poco/Data/ODBC/include/Poco/Data/ODBC/TypeInfo.h index 5200a61c7dc..e26ada65858 100644 --- a/base/poco/Data/ODBC/include/Poco/Data/ODBC/TypeInfo.h +++ b/base/poco/Data/ODBC/include/Poco/Data/ODBC/TypeInfo.h @@ -23,9 +23,6 @@ #include "Poco/Data/ODBC/ODBC.h" #include "Poco/DynamicAny.h" #include "Poco/NamedTuple.h" -#ifdef POCO_OS_FAMILY_WINDOWS -# include -#endif #include diff --git a/base/poco/Data/ODBC/include/Poco/Data/ODBC/Unicode.h b/base/poco/Data/ODBC/include/Poco/Data/ODBC/Unicode.h index 40148ed79c2..170498841d4 100644 --- a/base/poco/Data/ODBC/include/Poco/Data/ODBC/Unicode.h +++ b/base/poco/Data/ODBC/include/Poco/Data/ODBC/Unicode.h @@ -22,9 +22,6 @@ #include "Poco/Buffer.h" #include "Poco/Exception.h" #include "Poco/UnicodeConverter.h" -#ifdef POCO_OS_FAMILY_WINDOWS -# include -#endif #ifndef SQL_NOUNICODEMAP # define SQL_NOUNICODEMAP #endif @@ -33,10 +30,7 @@ #include -#if defined(POCO_OS_FAMILY_WINDOWS) && defined(POCO_WIN32_UTF8) -# define POCO_ODBC_UNICODE -# define POCO_ODBC_UNICODE_WINDOWS -#elif defined(POCO_OS_FAMILY_UNIX) && defined(UNICODE) +#if defined(POCO_OS_FAMILY_UNIX) && defined(UNICODE) # define POCO_ODBC_UNICODE # ifdef POCO_UNIXODBC # define POCO_ODBC_UNICODE_UNIXODBC diff --git a/base/poco/Data/ODBC/src/Binder.cpp b/base/poco/Data/ODBC/src/Binder.cpp index aa2c9e157f1..7e99f71dc28 100644 --- a/base/poco/Data/ODBC/src/Binder.cpp +++ b/base/poco/Data/ODBC/src/Binder.cpp @@ -500,18 +500,6 @@ void Binder::getColumnOrParameterSize(std::size_t pos, SQLINTEGER& size) { size = DEFAULT_PARAM_SIZE; //On Linux, PostgreSQL driver segfaults on SQLGetDescField, so this is disabled for now -#ifdef POCO_OS_FAMILY_WINDOWS - SQLHDESC hIPD = 0; - if (!Utility::isError(SQLGetStmtAttr(_rStmt, SQL_ATTR_IMP_PARAM_DESC, &hIPD, SQL_IS_POINTER, 0))) - { - SQLUINTEGER sz = 0; - if (!Utility::isError(SQLGetDescField(hIPD, (SQLSMALLINT) pos + 1, SQL_DESC_LENGTH, &sz, SQL_IS_UINTEGER, 0)) && - sz > 0) - { - size = sz; - } - } -#endif } if (colSize > 0 && paramSize > 0) diff --git a/base/poco/Data/ODBC/src/ODBCStatementImpl.cpp b/base/poco/Data/ODBC/src/ODBCStatementImpl.cpp index 9dd8c4e7a8f..b27dade266a 100644 --- a/base/poco/Data/ODBC/src/ODBCStatementImpl.cpp +++ b/base/poco/Data/ODBC/src/ODBCStatementImpl.cpp @@ -20,9 +20,6 @@ #include "Poco/Exception.h" -#ifdef POCO_OS_FAMILY_WINDOWS - #pragma warning(disable:4312)// 'type cast' : conversion from 'std::size_t' to 'SQLPOINTER' of greater size -#endif using Poco::DataFormatException; diff --git a/base/poco/Data/include/Poco/Data/Data.h b/base/poco/Data/include/Poco/Data/Data.h index e45def7ea24..3eec5de3251 100644 --- a/base/poco/Data/include/Poco/Data/Data.h +++ b/base/poco/Data/include/Poco/Data/Data.h @@ -31,13 +31,6 @@ // Data_API functions as being imported from a DLL, whereas this DLL sees symbols // defined with this macro as being exported. // -#if defined(_WIN32) && defined(POCO_DLL) -# if defined(Data_EXPORTS) -# define Data_API __declspec(dllexport) -# else -# define Data_API __declspec(dllimport) -# endif -#endif #if !defined(Data_API) diff --git a/base/poco/Foundation/include/Poco/ClassLibrary.h b/base/poco/Foundation/include/Poco/ClassLibrary.h index b254fafd3cb..deb43f26297 100644 --- a/base/poco/Foundation/include/Poco/ClassLibrary.h +++ b/base/poco/Foundation/include/Poco/ClassLibrary.h @@ -23,11 +23,7 @@ #include "Poco/Manifest.h" -#if defined(_WIN32) -# define POCO_LIBRARY_API __declspec(dllexport) -#else # define POCO_LIBRARY_API -#endif // diff --git a/base/poco/Foundation/include/Poco/Error.h b/base/poco/Foundation/include/Poco/Error.h index 534631ded18..9150e470cea 100644 --- a/base/poco/Foundation/include/Poco/Error.h +++ b/base/poco/Foundation/include/Poco/Error.h @@ -30,19 +30,11 @@ class Foundation_API Error /// for error reporting. { public: -#ifdef POCO_OS_FAMILY_WINDOWS - static DWORD last(); - /// Utility function returning the last error. - - static std::string getMessage(DWORD errorCode); - /// Utility function translating numeric error code to string. -#else static int last(); /// Utility function returning the last error. static std::string getMessage(int errorCode); /// Utility function translating numeric error code to string. -#endif }; diff --git a/base/poco/Foundation/include/Poco/Event.h b/base/poco/Foundation/include/Poco/Event.h index 1f5ea71f1f1..9dddb353003 100644 --- a/base/poco/Foundation/include/Poco/Event.h +++ b/base/poco/Foundation/include/Poco/Event.h @@ -22,11 +22,7 @@ #include "Poco/Foundation.h" -#if defined(POCO_OS_FAMILY_WINDOWS) -# include "Poco/Event_WIN32.h" -#else # include "Poco/Event_POSIX.h" -#endif namespace Poco diff --git a/base/poco/Foundation/include/Poco/FPEnvironment.h b/base/poco/Foundation/include/Poco/FPEnvironment.h index 62f96b76730..398c2af5eaa 100644 --- a/base/poco/Foundation/include/Poco/FPEnvironment.h +++ b/base/poco/Foundation/include/Poco/FPEnvironment.h @@ -29,8 +29,6 @@ # include "Poco/FPEnvironment_SUN.h" #elif defined(POCO_OS_FAMILY_UNIX) # include "Poco/FPEnvironment_C99.h" -#elif defined(POCO_OS_FAMILY_WINDOWS) -# include "Poco/FPEnvironment_WIN32.h" #else # include "Poco/FPEnvironment_DUMMY.h" #endif diff --git a/base/poco/Foundation/include/Poco/File.h b/base/poco/Foundation/include/Poco/File.h index cfc2c1f7e47..b7a3816bbed 100644 --- a/base/poco/Foundation/include/Poco/File.h +++ b/base/poco/Foundation/include/Poco/File.h @@ -23,15 +23,7 @@ #include "Poco/Timestamp.h" -#if defined(POCO_OS_FAMILY_WINDOWS) && defined(POCO_WIN32_UTF8) -# if defined(_WIN32_WCE) -# include "File_WINCE.h" -# else -# include "Poco/File_WIN32U.h" -# endif -#elif defined(POCO_OS_FAMILY_WINDOWS) -# include "Poco/File_WIN32.h" -#elif defined(POCO_OS_FAMILY_UNIX) +#if defined(POCO_OS_FAMILY_UNIX) # include "Poco/File_UNIX.h" #endif diff --git a/base/poco/Foundation/include/Poco/FileStream.h b/base/poco/Foundation/include/Poco/FileStream.h index 6b09c971043..819f237af3c 100644 --- a/base/poco/Foundation/include/Poco/FileStream.h +++ b/base/poco/Foundation/include/Poco/FileStream.h @@ -19,11 +19,7 @@ #include "Poco/Foundation.h" -#if defined(POCO_OS_FAMILY_WINDOWS) -# include "Poco/FileStream_WIN32.h" -#else # include "Poco/FileStream_POSIX.h" -#endif #include #include diff --git a/base/poco/Foundation/include/Poco/Foundation.h b/base/poco/Foundation/include/Poco/Foundation.h index 3001a5d4ccd..4259929b909 100644 --- a/base/poco/Foundation/include/Poco/Foundation.h +++ b/base/poco/Foundation/include/Poco/Foundation.h @@ -29,11 +29,6 @@ // // Ensure that POCO_DLL is default unless POCO_STATIC is defined // -#if defined(_WIN32) && defined(_DLL) -# if !defined(POCO_DLL) && !defined(POCO_STATIC) -# define POCO_DLL -# endif -#endif // @@ -71,9 +66,7 @@ // Include platform-specific definitions // #include "Poco/Platform.h" -#if defined(_WIN32) -# include "Poco/Platform_WIN32.h" -#elif defined(POCO_OS_FAMILY_UNIX) +#if defined(POCO_OS_FAMILY_UNIX) # include "Poco/Platform_POSIX.h" #endif @@ -86,15 +79,9 @@ // // Cleanup inconsistencies // -#ifdef POCO_OS_FAMILY_WINDOWS -# if defined(POCO_WIN32_UTF8) && defined(POCO_NO_WSTRING) -# error POCO_WIN32_UTF8 and POCO_NO_WSTRING are mutually exclusive. -# endif -#else # ifdef POCO_WIN32_UTF8 # undef POCO_WIN32_UTF8 # endif -#endif // diff --git a/base/poco/Foundation/include/Poco/LogFile.h b/base/poco/Foundation/include/Poco/LogFile.h index 1614dfccdc0..d831b126fbf 100644 --- a/base/poco/Foundation/include/Poco/LogFile.h +++ b/base/poco/Foundation/include/Poco/LogFile.h @@ -21,13 +21,7 @@ #include "Poco/Foundation.h" -#if defined(POCO_OS_FAMILY_WINDOWS) && defined(POCO_WIN32_UTF8) -# include "Poco/LogFile_WIN32U.h" -#elif defined(POCO_OS_FAMILY_WINDOWS) -# include "Poco/LogFile_WIN32.h" -#else # include "Poco/LogFile_STD.h" -#endif namespace Poco diff --git a/base/poco/Foundation/include/Poco/Mutex.h b/base/poco/Foundation/include/Poco/Mutex.h index d976c8af2ba..6ed3fd44d9e 100644 --- a/base/poco/Foundation/include/Poco/Mutex.h +++ b/base/poco/Foundation/include/Poco/Mutex.h @@ -23,15 +23,7 @@ #include "Poco/ScopedLock.h" -#if defined(POCO_OS_FAMILY_WINDOWS) -# if defined(_WIN32_WCE) -# include "Poco/Mutex_WINCE.h" -# else -# include "Poco/Mutex_WIN32.h" -# endif -#else # include "Poco/Mutex_POSIX.h" -#endif namespace Poco diff --git a/base/poco/Foundation/include/Poco/NamedEvent.h b/base/poco/Foundation/include/Poco/NamedEvent.h index 6affa3cde6f..16a5e387233 100644 --- a/base/poco/Foundation/include/Poco/NamedEvent.h +++ b/base/poco/Foundation/include/Poco/NamedEvent.h @@ -21,11 +21,7 @@ #include "Poco/Foundation.h" -#if defined(POCO_OS_FAMILY_WINDOWS) && defined(POCO_WIN32_UTF8) -# include "Poco/NamedEvent_WIN32U.h" -#elif defined(POCO_OS_FAMILY_WINDOWS) -# include "Poco/NamedEvent_WIN32.h" -#elif POCO_OS == POCO_OS_ANDROID +#if POCO_OS == POCO_OS_ANDROID # include "Poco/NamedEvent_Android.h" #elif defined(POCO_OS_FAMILY_UNIX) # include "Poco/NamedEvent_UNIX.h" diff --git a/base/poco/Foundation/include/Poco/NamedMutex.h b/base/poco/Foundation/include/Poco/NamedMutex.h index d91732fd787..9dce963f2c1 100644 --- a/base/poco/Foundation/include/Poco/NamedMutex.h +++ b/base/poco/Foundation/include/Poco/NamedMutex.h @@ -22,11 +22,7 @@ #include "Poco/ScopedLock.h" -#if defined(POCO_OS_FAMILY_WINDOWS) && defined(POCO_WIN32_UTF8) -# include "Poco/NamedMutex_WIN32U.h" -#elif defined(POCO_OS_FAMILY_WINDOWS) -# include "Poco/NamedMutex_WIN32.h" -#elif POCO_OS == POCO_OS_ANDROID +#if POCO_OS == POCO_OS_ANDROID # include "Poco/NamedMutex_Android.h" #elif defined(POCO_OS_FAMILY_UNIX) # include "Poco/NamedMutex_UNIX.h" diff --git a/base/poco/Foundation/include/Poco/Path.h b/base/poco/Foundation/include/Poco/Path.h index a82cdfb78fa..7c683ec441d 100644 --- a/base/poco/Foundation/include/Poco/Path.h +++ b/base/poco/Foundation/include/Poco/Path.h @@ -470,8 +470,6 @@ inline char Path::separator() { #if defined(POCO_OS_FAMILY_VMS) return '.'; -#elif defined(POCO_OS_FAMILY_WINDOWS) - return '\\'; #else return '/'; #endif @@ -482,8 +480,6 @@ inline char Path::pathSeparator() { #if defined(POCO_OS_FAMILY_VMS) return ','; -#elif defined(POCO_OS_FAMILY_WINDOWS) - return ';'; #else return ':'; #endif diff --git a/base/poco/Foundation/include/Poco/PipeImpl.h b/base/poco/Foundation/include/Poco/PipeImpl.h index 980a75d9690..03e39236be7 100644 --- a/base/poco/Foundation/include/Poco/PipeImpl.h +++ b/base/poco/Foundation/include/Poco/PipeImpl.h @@ -21,13 +21,7 @@ #include "Poco/Foundation.h" -#if defined(POCO_OS_FAMILY_WINDOWS) -# if defined(_WIN32_WCE) -# include "PipeImpl_DUMMY.h" -# else -# include "Poco/PipeImpl_WIN32.h" -# endif -#elif defined(POCO_OS_FAMILY_UNIX) +#if defined(POCO_OS_FAMILY_UNIX) # include "Poco/PipeImpl_POSIX.h" #else # include "Poco/PipeImpl_DUMMY.h" diff --git a/base/poco/Foundation/include/Poco/Platform.h b/base/poco/Foundation/include/Poco/Platform.h index 0b95f12d960..49c9dfa1564 100644 --- a/base/poco/Foundation/include/Poco/Platform.h +++ b/base/poco/Foundation/include/Poco/Platform.h @@ -143,10 +143,7 @@ # define POCO_ARCH_LITTLE_ENDIAN 1 #elif defined(__mips__) || defined(__mips) || defined(__MIPS__) || defined(_M_MRX000) # define POCO_ARCH POCO_ARCH_MIPS -# if defined(POCO_OS_FAMILY_WINDOWS) -// Is this OK? Supports windows only little endian?? -# define POCO_ARCH_LITTLE_ENDIAN 1 -# elif defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) +# if defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) # define POCO_ARCH_BIG_ENDIAN 1 # elif defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) # define POCO_ARCH_LITTLE_ENDIAN 1 @@ -249,11 +246,7 @@ #endif -#if defined(POCO_OS_FAMILY_WINDOWS) -# define POCO_DEFAULT_NEWLINE_CHARS "\r\n" -#else # define POCO_DEFAULT_NEWLINE_CHARS "\n" -#endif #endif // Foundation_Platform_INCLUDED diff --git a/base/poco/Foundation/include/Poco/Process.h b/base/poco/Foundation/include/Poco/Process.h index c930a2240fd..6f45db23062 100644 --- a/base/poco/Foundation/include/Poco/Process.h +++ b/base/poco/Foundation/include/Poco/Process.h @@ -21,15 +21,7 @@ #include "Poco/Foundation.h" -#if defined(POCO_OS_FAMILY_WINDOWS) && defined(POCO_WIN32_UTF8) -# if defined(_WIN32_WCE) -# include "Process_WINCE.h" -# else -# include "Poco/Process_WIN32U.h" -# endif -#elif defined(POCO_OS_FAMILY_WINDOWS) -# include "Poco/Process_WIN32.h" -#elif defined(POCO_OS_FAMILY_UNIX) +#if defined(POCO_OS_FAMILY_UNIX) # include "Poco/Process_UNIX.h" #endif diff --git a/base/poco/Foundation/include/Poco/RWLock.h b/base/poco/Foundation/include/Poco/RWLock.h index 292ff6afcbe..f187a298365 100644 --- a/base/poco/Foundation/include/Poco/RWLock.h +++ b/base/poco/Foundation/include/Poco/RWLock.h @@ -22,13 +22,7 @@ #include "Poco/Foundation.h" -#if defined(POCO_OS_FAMILY_WINDOWS) -# if defined(_WIN32_WCE) -# include "Poco/RWLock_WINCE.h" -# else -# include "Poco/RWLock_WIN32.h" -# endif -#elif POCO_OS == POCO_OS_ANDROID +#if POCO_OS == POCO_OS_ANDROID # include "Poco/RWLock_Android.h" #else # include "Poco/RWLock_POSIX.h" diff --git a/base/poco/Foundation/include/Poco/Semaphore.h b/base/poco/Foundation/include/Poco/Semaphore.h index 1e23ff37848..07a2350a949 100644 --- a/base/poco/Foundation/include/Poco/Semaphore.h +++ b/base/poco/Foundation/include/Poco/Semaphore.h @@ -22,11 +22,7 @@ #include "Poco/Foundation.h" -#if defined(POCO_OS_FAMILY_WINDOWS) -# include "Poco/Semaphore_WIN32.h" -#else # include "Poco/Semaphore_POSIX.h" -#endif namespace Poco diff --git a/base/poco/Foundation/include/Poco/SharedLibrary.h b/base/poco/Foundation/include/Poco/SharedLibrary.h index 387a9793f45..c68ddd7e03b 100644 --- a/base/poco/Foundation/include/Poco/SharedLibrary.h +++ b/base/poco/Foundation/include/Poco/SharedLibrary.h @@ -25,10 +25,6 @@ # include "Poco/SharedLibrary_HPUX.h" #elif defined(POCO_OS_FAMILY_UNIX) # include "Poco/SharedLibrary_UNIX.h" -#elif defined(POCO_OS_FAMILY_WINDOWS) && defined(POCO_WIN32_UTF8) -# include "Poco/SharedLibrary_WIN32U.h" -#elif defined(POCO_OS_FAMILY_WINDOWS) -# include "Poco/SharedLibrary_WIN32.h" #endif diff --git a/base/poco/Foundation/include/Poco/Thread.h b/base/poco/Foundation/include/Poco/Thread.h index 1534b09b788..91ef2ca6746 100644 --- a/base/poco/Foundation/include/Poco/Thread.h +++ b/base/poco/Foundation/include/Poco/Thread.h @@ -23,15 +23,7 @@ #include "Poco/Mutex.h" -#if defined(POCO_OS_FAMILY_WINDOWS) -# if defined(_WIN32_WCE) -# include "Poco/Thread_WINCE.h" -# else -# include "Poco/Thread_WIN32.h" -# endif -#else # include "Poco/Thread_POSIX.h" -#endif namespace Poco diff --git a/base/poco/Foundation/include/Poco/Timestamp.h b/base/poco/Foundation/include/Poco/Timestamp.h index 2f94b36b2de..d13d99d3f0b 100644 --- a/base/poco/Foundation/include/Poco/Timestamp.h +++ b/base/poco/Foundation/include/Poco/Timestamp.h @@ -140,10 +140,6 @@ public: /// Since the timestamp has microsecond resolution, /// the returned value is always 1000000. -#if defined(_WIN32) - static Timestamp fromFileTimeNP(UInt32 fileTimeLow, UInt32 fileTimeHigh); - void toFileTimeNP(UInt32 & fileTimeLow, UInt32 & fileTimeHigh) const; -#endif private: TimeVal _ts; diff --git a/base/poco/Foundation/include/Poco/Types.h b/base/poco/Foundation/include/Poco/Types.h index 327aea02286..156b3584d15 100644 --- a/base/poco/Foundation/include/Poco/Types.h +++ b/base/poco/Foundation/include/Poco/Types.h @@ -35,13 +35,6 @@ typedef signed short Int16; typedef unsigned short UInt16; typedef signed int Int32; typedef unsigned int UInt32; -# if defined(_WIN64) -# define POCO_PTR_IS_64_BIT 1 -typedef signed long long IntPtr; -typedef unsigned long long UIntPtr; -typedef signed long long Int64; -typedef unsigned long long UInt64; -# else typedef signed long IntPtr; typedef unsigned long UIntPtr; # if defined(__LP64__) @@ -53,7 +46,6 @@ typedef unsigned long UInt64; typedef signed long long Int64; typedef unsigned long long UInt64; # endif -# endif # define POCO_HAVE_INT64 1 #endif diff --git a/base/poco/Foundation/include/Poco/UTFString.h b/base/poco/Foundation/include/Poco/UTFString.h index f03d80fa90f..cd34d212393 100644 --- a/base/poco/Foundation/include/Poco/UTFString.h +++ b/base/poco/Foundation/include/Poco/UTFString.h @@ -225,12 +225,7 @@ typedef std::basic_string UTF16String; typedef UInt32 UTF32Char; typedef std::basic_string UTF32String; #else // POCO_NO_WSTRING -# if defined(POCO_OS_FAMILY_WINDOWS) -typedef wchar_t UTF16Char; -typedef std::wstring UTF16String; -typedef UInt32 UTF32Char; -typedef std::basic_string UTF32String; -# elif defined(__SIZEOF_WCHAR_T__) //gcc +# if defined(__SIZEOF_WCHAR_T__) //gcc # if (__SIZEOF_WCHAR_T__ == 2) typedef wchar_t UTF16Char; typedef std::wstring UTF16String; diff --git a/base/poco/Foundation/include/Poco/UnWindows.h b/base/poco/Foundation/include/Poco/UnWindows.h index 5678752bb5d..7609b9016e4 100644 --- a/base/poco/Foundation/include/Poco/UnWindows.h +++ b/base/poco/Foundation/include/Poco/UnWindows.h @@ -38,11 +38,6 @@ // Reduce bloat -#if defined(_WIN32) -# if !defined(WIN32_LEAN_AND_MEAN) && !defined(POCO_BLOATED_WIN32) -# define WIN32_LEAN_AND_MEAN -# endif -#endif // Microsoft Visual C++ includes copies of the Windows header files diff --git a/base/poco/Foundation/src/Clock.cpp b/base/poco/Foundation/src/Clock.cpp index 169f2a09805..75ab43042cf 100644 --- a/base/poco/Foundation/src/Clock.cpp +++ b/base/poco/Foundation/src/Clock.cpp @@ -21,8 +21,6 @@ #elif defined(POCO_OS_FAMILY_UNIX) #include #include -#elif defined(POCO_OS_FAMILY_WINDOWS) -#include "Poco/UnWindows.h" #endif #include #undef min @@ -91,18 +89,7 @@ void Clock::swap(Clock& timestamp) void Clock::update() { -#if defined(POCO_OS_FAMILY_WINDOWS) - - LARGE_INTEGER perfCounter; - LARGE_INTEGER perfFreq; - if (QueryPerformanceCounter(&perfCounter) && QueryPerformanceFrequency(&perfFreq)) - { - _clock = resolution()*(perfCounter.QuadPart/perfFreq.QuadPart); - _clock += (perfCounter.QuadPart % perfFreq.QuadPart)*resolution()/perfFreq.QuadPart; - } - else throw Poco::SystemException("cannot get system clock"); - -#elif defined(__MACH__) +#if defined(__MACH__) clock_serv_t cs; mach_timespec_t ts; @@ -131,17 +118,7 @@ void Clock::update() Clock::ClockDiff Clock::accuracy() { -#if defined(POCO_OS_FAMILY_WINDOWS) - - LARGE_INTEGER perfFreq; - if (QueryPerformanceFrequency(&perfFreq) && perfFreq.QuadPart > 0) - { - ClockVal acc = resolution()/perfFreq.QuadPart; - return acc > 0 ? acc : 1; - } - else throw Poco::SystemException("cannot get system clock accuracy"); - -#elif defined(__MACH__) +#if defined(__MACH__) clock_serv_t cs; int nanosecs; @@ -173,11 +150,7 @@ Clock::ClockDiff Clock::accuracy() bool Clock::monotonic() { -#if defined(POCO_OS_FAMILY_WINDOWS) - - return true; - -#elif defined(__MACH__) +#if defined(__MACH__) return true; diff --git a/base/poco/Foundation/src/Debugger.cpp b/base/poco/Foundation/src/Debugger.cpp index 16cc146d638..6e388398405 100644 --- a/base/poco/Foundation/src/Debugger.cpp +++ b/base/poco/Foundation/src/Debugger.cpp @@ -16,9 +16,7 @@ #include #include #include -#if defined(POCO_OS_FAMILY_WINDOWS) - #include "Poco/UnWindows.h" -#elif defined(POCO_OS_FAMILY_UNIX) && !defined(POCO_VXWORKS) +#if defined(POCO_OS_FAMILY_UNIX) && !defined(POCO_VXWORKS) #include #include #endif @@ -38,22 +36,7 @@ namespace Poco { bool Debugger::isAvailable() { #if defined(_DEBUG) - #if defined(POCO_OS_FAMILY_WINDOWS) - #if defined(_WIN32_WCE) - #if (_WIN32_WCE >= 0x600) - BOOL isDebuggerPresent; - if (CheckRemoteDebuggerPresent(GetCurrentProcess(), &isDebuggerPresent)) - { - return isDebuggerPresent ? true : false; - } - return false; - #else - return false; - #endif - #else - return IsDebuggerPresent() ? true : false; - #endif - #elif defined(POCO_OS_FAMILY_UNIX) + #if defined(POCO_OS_FAMILY_UNIX) return std::getenv("POCO_ENABLE_DEBUGGER") ? true : false; #endif #else @@ -68,20 +51,6 @@ void Debugger::message(const std::string& msg) std::fputs("\n%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n", stderr); std::fputs(msg.c_str(), stderr); std::fputs("\n%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n", stderr); - #if defined(POCO_OS_FAMILY_WINDOWS) - if (isAvailable()) - { - #if defined(POCO_WIN32_UTF8) && !defined(POCO_NO_WSTRING) - std::wstring umsg; - UnicodeConverter::toUTF16(msg, umsg); - umsg += '\n'; - OutputDebugStringW(umsg.c_str()); - #else - OutputDebugStringA(msg.c_str()); - OutputDebugStringA("\n"); - #endif - } - #endif #endif } @@ -99,12 +68,7 @@ void Debugger::message(const std::string& msg, const char* file, int line) void Debugger::enter() { #if defined(_DEBUG) - #if defined(POCO_OS_FAMILY_WINDOWS) - if (isAvailable()) - { - DebugBreak(); - } - #elif defined(POCO_OS_FAMILY_UNIX) + #if defined(POCO_OS_FAMILY_UNIX) if (isAvailable()) { kill(getpid(), SIGINT); diff --git a/base/poco/Foundation/src/DirectoryIterator.cpp b/base/poco/Foundation/src/DirectoryIterator.cpp index 28d29e0d2d7..94faaacf536 100644 --- a/base/poco/Foundation/src/DirectoryIterator.cpp +++ b/base/poco/Foundation/src/DirectoryIterator.cpp @@ -15,11 +15,7 @@ #include "Poco/DirectoryIterator.h" -#if defined(POCO_OS_FAMILY_WINDOWS) && defined(POCO_WIN32_UTF8) -#include "DirectoryIterator_WIN32U.cpp" -#elif defined(POCO_OS_FAMILY_WINDOWS) -#include "DirectoryIterator_WIN32.cpp" -#elif defined(POCO_OS_FAMILY_UNIX) +#if defined(POCO_OS_FAMILY_UNIX) #include "DirectoryIterator_UNIX.cpp" #endif diff --git a/base/poco/Foundation/src/Environment.cpp b/base/poco/Foundation/src/Environment.cpp index 50a3469820d..00891ee2b92 100644 --- a/base/poco/Foundation/src/Environment.cpp +++ b/base/poco/Foundation/src/Environment.cpp @@ -20,12 +20,6 @@ #if defined(POCO_OS_FAMILY_UNIX) #include "Environment_UNIX.cpp" -#elif defined(POCO_OS_FAMILY_WINDOWS) -#if defined(_WIN32_WCE) -#include "Environment_WINCE.cpp" -#else -#include "Environment_WIN32.cpp" -#endif #endif @@ -147,11 +141,7 @@ bool Environment::isUnix() bool Environment::isWindows() { -#if defined(POCO_OS_FAMILY_WINDOWS) - return true; -#else return false; -#endif } diff --git a/base/poco/Foundation/src/Error.cpp b/base/poco/Foundation/src/Error.cpp index fbd3673f6bc..750c9c7f344 100644 --- a/base/poco/Foundation/src/Error.cpp +++ b/base/poco/Foundation/src/Error.cpp @@ -23,34 +23,6 @@ namespace Poco { -#ifdef POCO_OS_FAMILY_WINDOWS - - - DWORD Error::last() - { - return GetLastError(); - } - - - std::string Error::getMessage(DWORD errorCode) - { - std::string errMsg; - DWORD dwFlg = FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS; - #if defined(POCO_WIN32_UTF8) && !defined(POCO_NO_WSTRING) - LPWSTR lpMsgBuf = 0; - if (FormatMessageW(dwFlg, 0, errorCode, 0, (LPWSTR) & lpMsgBuf, 0, NULL)) - UnicodeConverter::toUTF8(lpMsgBuf, errMsg); - #else - LPTSTR lpMsgBuf = 0; - if (FormatMessageA(dwFlg, 0, errorCode, 0, (LPTSTR) & lpMsgBuf, 0, NULL)) - errMsg = lpMsgBuf; - #endif - LocalFree(lpMsgBuf); - return errMsg; - } - - -#else int Error::last() @@ -111,7 +83,6 @@ namespace Poco { } -#endif } // namespace Poco diff --git a/base/poco/Foundation/src/Event.cpp b/base/poco/Foundation/src/Event.cpp index e042f53be97..6bdeabd6ced 100644 --- a/base/poco/Foundation/src/Event.cpp +++ b/base/poco/Foundation/src/Event.cpp @@ -15,11 +15,7 @@ #include "Poco/Event.h" -#if defined(POCO_OS_FAMILY_WINDOWS) -#include "Event_WIN32.cpp" -#else #include "Event_POSIX.cpp" -#endif namespace Poco { diff --git a/base/poco/Foundation/src/EventLogChannel.cpp b/base/poco/Foundation/src/EventLogChannel.cpp index ed4e25a6dc5..8d7d7cb8274 100644 --- a/base/poco/Foundation/src/EventLogChannel.cpp +++ b/base/poco/Foundation/src/EventLogChannel.cpp @@ -224,17 +224,9 @@ void EventLogChannel::setUpRegistry() const std::wstring path; #if defined(POCO_DLL) #if defined(_DEBUG) - #if defined(_WIN64) - path = findLibrary(L"PocoFoundation64d.dll"); - #else path = findLibrary(L"PocoFoundationd.dll"); - #endif #else - #if defined(_WIN64) - path = findLibrary(L"PocoFoundation64.dll"); - #else path = findLibrary(L"PocoFoundation.dll"); - #endif #endif #endif @@ -244,17 +236,9 @@ void EventLogChannel::setUpRegistry() const std::string path; #if defined(POCO_DLL) #if defined(_DEBUG) - #if defined(_WIN64) - path = findLibrary("PocoFoundation64d.dll"); - #else path = findLibrary("PocoFoundationd.dll"); - #endif #else - #if defined(_WIN64) - path = findLibrary("PocoFoundation64.dll"); - #else path = findLibrary("PocoFoundation.dll"); - #endif #endif #endif diff --git a/base/poco/Foundation/src/FPEnvironment.cpp b/base/poco/Foundation/src/FPEnvironment.cpp index 70c8e68af5a..5eee4cf9646 100644 --- a/base/poco/Foundation/src/FPEnvironment.cpp +++ b/base/poco/Foundation/src/FPEnvironment.cpp @@ -25,8 +25,6 @@ #include "FPEnvironment_SUN.cpp" #elif defined(POCO_OS_FAMILY_UNIX) #include "FPEnvironment_C99.cpp" -#elif defined(POCO_OS_FAMILY_WINDOWS) -#include "FPEnvironment_WIN32.cpp" #else #include "FPEnvironment_DUMMY.cpp" #endif diff --git a/base/poco/Foundation/src/File.cpp b/base/poco/Foundation/src/File.cpp index fe719338d4d..220242eecfb 100644 --- a/base/poco/Foundation/src/File.cpp +++ b/base/poco/Foundation/src/File.cpp @@ -17,15 +17,7 @@ #include "Poco/DirectoryIterator.h" -#if defined(POCO_OS_FAMILY_WINDOWS) && defined(POCO_WIN32_UTF8) -#if defined(_WIN32_WCE) -#include "File_WINCE.cpp" -#else -#include "File_WIN32U.cpp" -#endif -#elif defined(POCO_OS_FAMILY_WINDOWS) -#include "File_WIN32.cpp" -#elif defined(POCO_OS_FAMILY_UNIX) +#if defined(POCO_OS_FAMILY_UNIX) #include "File_UNIX.cpp" #endif #include "Poco/Thread.h" diff --git a/base/poco/Foundation/src/FileStream.cpp b/base/poco/Foundation/src/FileStream.cpp index 4f94a249087..db9e4e9fd9e 100644 --- a/base/poco/Foundation/src/FileStream.cpp +++ b/base/poco/Foundation/src/FileStream.cpp @@ -14,11 +14,7 @@ #include "Poco/FileStream.h" #include "Poco/Exception.h" -#if defined(POCO_OS_FAMILY_WINDOWS) -#include "FileStream_WIN32.cpp" -#else #include "FileStream_POSIX.cpp" -#endif namespace Poco { diff --git a/base/poco/Foundation/src/LogFile.cpp b/base/poco/Foundation/src/LogFile.cpp index 2aaaf8c5698..3dd69216990 100644 --- a/base/poco/Foundation/src/LogFile.cpp +++ b/base/poco/Foundation/src/LogFile.cpp @@ -15,13 +15,7 @@ #include "Poco/LogFile.h" -#if defined(POCO_OS_FAMILY_WINDOWS) && defined(POCO_WIN32_UTF8) -#include "LogFile_WIN32U.cpp" -#elif defined(POCO_OS_FAMILY_WINDOWS) -#include "LogFile_WIN32.cpp" -#else #include "LogFile_STD.cpp" -#endif namespace Poco { diff --git a/base/poco/Foundation/src/LoggingFactory.cpp b/base/poco/Foundation/src/LoggingFactory.cpp index f3e067a6111..5f8076abb5e 100644 --- a/base/poco/Foundation/src/LoggingFactory.cpp +++ b/base/poco/Foundation/src/LoggingFactory.cpp @@ -24,10 +24,6 @@ #if defined(POCO_OS_FAMILY_UNIX) && !defined(POCO_NO_SYSLOGCHANNEL) #include "Poco/SyslogChannel.h" #endif -#if defined(POCO_OS_FAMILY_WINDOWS) && !defined(_WIN32_WCE) -#include "Poco/EventLogChannel.h" -#include "Poco/WindowsConsoleChannel.h" -#endif #include "Poco/PatternFormatter.h" @@ -84,13 +80,8 @@ LoggingFactory& LoggingFactory::defaultFactory() void LoggingFactory::registerBuiltins() { _channelFactory.registerClass("AsyncChannel", new Instantiator); -#if defined(POCO_OS_FAMILY_WINDOWS) && !defined(_WIN32_WCE) - _channelFactory.registerClass("ConsoleChannel", new Instantiator); - _channelFactory.registerClass("ColorConsoleChannel", new Instantiator); -#else _channelFactory.registerClass("ConsoleChannel", new Instantiator); _channelFactory.registerClass("ColorConsoleChannel", new Instantiator); -#endif #ifndef POCO_NO_FILECHANNEL _channelFactory.registerClass("FileChannel", new Instantiator); #endif @@ -105,9 +96,6 @@ void LoggingFactory::registerBuiltins() #ifndef POCO_NO_SYSLOGCHANNEL _channelFactory.registerClass("SyslogChannel", new Instantiator); #endif -#endif -#if defined(POCO_OS_FAMILY_WINDOWS) && !defined(_WIN32_WCE) - _channelFactory.registerClass("EventLogChannel", new Instantiator); #endif _formatterFactory.registerClass("PatternFormatter", new Instantiator); diff --git a/base/poco/Foundation/src/Mutex.cpp b/base/poco/Foundation/src/Mutex.cpp index 55619846d35..1b5cf4b7fd0 100644 --- a/base/poco/Foundation/src/Mutex.cpp +++ b/base/poco/Foundation/src/Mutex.cpp @@ -15,15 +15,7 @@ #include "Poco/Mutex.h" -#if defined(POCO_OS_FAMILY_WINDOWS) -#if defined(_WIN32_WCE) -#include "Mutex_WINCE.cpp" -#else -#include "Mutex_WIN32.cpp" -#endif -#else #include "Mutex_POSIX.cpp" -#endif namespace Poco { diff --git a/base/poco/Foundation/src/NamedEvent.cpp b/base/poco/Foundation/src/NamedEvent.cpp index 6615ca8f048..50360e5db45 100644 --- a/base/poco/Foundation/src/NamedEvent.cpp +++ b/base/poco/Foundation/src/NamedEvent.cpp @@ -15,11 +15,7 @@ #include "Poco/NamedEvent.h" -#if defined(POCO_OS_FAMILY_WINDOWS) && defined(POCO_WIN32_UTF8) -#include "NamedEvent_WIN32U.cpp" -#elif defined(POCO_OS_FAMILY_WINDOWS) -#include "NamedEvent_WIN32.cpp" -#elif POCO_OS == POCO_OS_ANDROID +#if POCO_OS == POCO_OS_ANDROID #include "NamedEvent_Android.cpp" #elif defined(POCO_OS_FAMILY_UNIX) #include "NamedEvent_UNIX.cpp" diff --git a/base/poco/Foundation/src/NamedMutex.cpp b/base/poco/Foundation/src/NamedMutex.cpp index 77fd672c640..98b2399ea6b 100644 --- a/base/poco/Foundation/src/NamedMutex.cpp +++ b/base/poco/Foundation/src/NamedMutex.cpp @@ -15,11 +15,7 @@ #include "Poco/NamedMutex.h" -#if defined(POCO_OS_FAMILY_WINDOWS) && defined(POCO_WIN32_UTF8) -#include "NamedMutex_WIN32U.cpp" -#elif defined(POCO_OS_FAMILY_WINDOWS) -#include "NamedMutex_WIN32.cpp" -#elif POCO_OS == POCO_OS_ANDROID +#if POCO_OS == POCO_OS_ANDROID #include "NamedMutex_Android.cpp" #elif defined(POCO_OS_FAMILY_UNIX) #include "NamedMutex_UNIX.cpp" diff --git a/base/poco/Foundation/src/Path.cpp b/base/poco/Foundation/src/Path.cpp index 6b26ca04667..8be42a4bdeb 100644 --- a/base/poco/Foundation/src/Path.cpp +++ b/base/poco/Foundation/src/Path.cpp @@ -16,23 +16,11 @@ #include "Poco/File.h" #include "Poco/Exception.h" #include "Poco/StringTokenizer.h" -#if defined(_WIN32) && defined(POCO_WIN32_UTF8) -#include "Poco/UnicodeConverter.h" -#include "Poco/Buffer.h" -#endif #include #if defined(POCO_OS_FAMILY_UNIX) #include "Path_UNIX.cpp" -#elif defined(POCO_OS_FAMILY_WINDOWS) && defined(POCO_WIN32_UTF8) -#if defined(_WIN32_WCE) -#include "Path_WINCE.cpp" -#else -#include "Path_WIN32U.cpp" -#endif -#elif defined(POCO_OS_FAMILY_WINDOWS) -#include "Path_WIN32.cpp" #endif @@ -176,11 +164,7 @@ Path& Path::assign(const Path& path) Path& Path::assign(const std::string& path) { -#if defined(POCO_OS_FAMILY_WINDOWS) - parseWindows(path); -#else parseUnix(path); -#endif return *this; } @@ -219,11 +203,7 @@ Path& Path::assign(const char* path) std::string Path::toString() const { -#if defined(POCO_OS_FAMILY_WINDOWS) - return buildWindows(); -#else return buildUnix(); -#endif } @@ -1043,20 +1023,6 @@ std::string Path::buildVMS() const std::string Path::transcode(const std::string& path) { -#if defined(_WIN32) && defined(POCO_WIN32_UTF8) - std::wstring uniPath; - UnicodeConverter::toUTF16(path, uniPath); - DWORD len = WideCharToMultiByte(CP_ACP, WC_NO_BEST_FIT_CHARS, uniPath.c_str(), static_cast(uniPath.length()), NULL, 0, NULL, NULL); - if (len > 0) - { - Buffer buffer(len); - DWORD rc = WideCharToMultiByte(CP_ACP, WC_NO_BEST_FIT_CHARS, uniPath.c_str(), static_cast(uniPath.length()), buffer.begin(), static_cast(buffer.size()), NULL, NULL); - if (rc) - { - return std::string(buffer.begin(), buffer.size()); - } - } -#endif return path; } diff --git a/base/poco/Foundation/src/PipeImpl.cpp b/base/poco/Foundation/src/PipeImpl.cpp index bafc713a10e..d17f1f41a1f 100644 --- a/base/poco/Foundation/src/PipeImpl.cpp +++ b/base/poco/Foundation/src/PipeImpl.cpp @@ -15,13 +15,7 @@ #include "Poco/PipeImpl.h" -#if defined(POCO_OS_FAMILY_WINDOWS) -#if defined(_WIN32_WCE) -/// #include "PipeImpl_DUMMY.cpp" -#else -#include "PipeImpl_WIN32.cpp" -#endif -#elif defined(POCO_OS_FAMILY_UNIX) +#if defined(POCO_OS_FAMILY_UNIX) #include "PipeImpl_POSIX.cpp" #else #include "PipeImpl_DUMMY.cpp" diff --git a/base/poco/Foundation/src/Process.cpp b/base/poco/Foundation/src/Process.cpp index b864f030d5d..7099939926d 100644 --- a/base/poco/Foundation/src/Process.cpp +++ b/base/poco/Foundation/src/Process.cpp @@ -47,15 +47,7 @@ namespace } -#if defined(POCO_OS_FAMILY_WINDOWS) && defined(POCO_WIN32_UTF8) -#if defined(_WIN32_WCE) -#include "Process_WINCE.cpp" -#else -#include "Process_WIN32U.cpp" -#endif -#elif defined(POCO_OS_FAMILY_WINDOWS) -#include "Process_WIN32.cpp" -#elif defined(POCO_OS_FAMILY_UNIX) +#if defined(POCO_OS_FAMILY_UNIX) #include "Process_UNIX.cpp" #endif diff --git a/base/poco/Foundation/src/RWLock.cpp b/base/poco/Foundation/src/RWLock.cpp index da679bf1b51..ffd0baf0e94 100644 --- a/base/poco/Foundation/src/RWLock.cpp +++ b/base/poco/Foundation/src/RWLock.cpp @@ -15,13 +15,7 @@ #include "Poco/RWLock.h" -#if defined(POCO_OS_FAMILY_WINDOWS) -#if defined(_WIN32_WCE) -#include "RWLock_WINCE.cpp" -#else -#include "RWLock_WIN32.cpp" -#endif -#elif POCO_OS == POCO_OS_ANDROID +#if POCO_OS == POCO_OS_ANDROID #include "RWLock_Android.cpp" #else #include "RWLock_POSIX.cpp" diff --git a/base/poco/Foundation/src/RandomStream.cpp b/base/poco/Foundation/src/RandomStream.cpp index 7a490eb603d..568398961be 100644 --- a/base/poco/Foundation/src/RandomStream.cpp +++ b/base/poco/Foundation/src/RandomStream.cpp @@ -15,10 +15,7 @@ #include "Poco/RandomStream.h" #include "Poco/Random.h" #include "Poco/SHA1Engine.h" -#if defined(POCO_OS_FAMILY_WINDOWS) -#include "Poco/UnWindows.h" -#include -#elif defined(POCO_OS_FAMILY_UNIX) +#if defined(POCO_OS_FAMILY_UNIX) #include #include #endif @@ -42,13 +39,6 @@ int RandomBuf::readFromDevice(char* buffer, std::streamsize length) { int n = 0; -#if defined(POCO_OS_FAMILY_WINDOWS) - HCRYPTPROV hProvider = 0; - CryptAcquireContext(&hProvider, 0, 0, PROV_RSA_FULL, CRYPT_VERIFYCONTEXT); - CryptGenRandom(hProvider, (DWORD) length, (BYTE*) buffer); - CryptReleaseContext(hProvider, 0); - n = static_cast(length); -#else #if defined(POCO_OS_FAMILY_UNIX) int fd = open("/dev/urandom", O_RDONLY, 0); if (fd >= 0) @@ -92,7 +82,6 @@ int RandomBuf::readFromDevice(char* buffer, std::streamsize length) } } } -#endif return n; } diff --git a/base/poco/Foundation/src/Semaphore.cpp b/base/poco/Foundation/src/Semaphore.cpp index 85782cb821c..3c80b960921 100644 --- a/base/poco/Foundation/src/Semaphore.cpp +++ b/base/poco/Foundation/src/Semaphore.cpp @@ -15,11 +15,7 @@ #include "Poco/Semaphore.h" -#if defined(POCO_OS_FAMILY_WINDOWS) -#include "Semaphore_WIN32.cpp" -#else #include "Semaphore_POSIX.cpp" -#endif namespace Poco { diff --git a/base/poco/Foundation/src/SharedLibrary.cpp b/base/poco/Foundation/src/SharedLibrary.cpp index 478bbbaf831..038bc72d001 100644 --- a/base/poco/Foundation/src/SharedLibrary.cpp +++ b/base/poco/Foundation/src/SharedLibrary.cpp @@ -20,10 +20,6 @@ #include "SharedLibrary_HPUX.cpp" #elif defined(POCO_OS_FAMILY_UNIX) #include "SharedLibrary_UNIX.cpp" -#elif defined(POCO_OS_FAMILY_WINDOWS) && defined(POCO_WIN32_UTF8) -#include "SharedLibrary_WIN32U.cpp" -#elif defined(POCO_OS_FAMILY_WINDOWS) -#include "SharedLibrary_WIN32.cpp" #endif diff --git a/base/poco/Foundation/src/SharedMemory.cpp b/base/poco/Foundation/src/SharedMemory.cpp index 154555c4fa5..6ce97618f37 100644 --- a/base/poco/Foundation/src/SharedMemory.cpp +++ b/base/poco/Foundation/src/SharedMemory.cpp @@ -16,8 +16,6 @@ #include "Poco/Exception.h" #if defined(POCO_NO_SHAREDMEMORY) /// #include "SharedMemory_DUMMY.cpp" -#elif defined(POCO_OS_FAMILY_WINDOWS) -#include "SharedMemory_WIN32.cpp" #elif defined(POCO_OS_FAMILY_UNIX) #include "SharedMemory_POSIX.cpp" #else diff --git a/base/poco/Foundation/src/Thread.cpp b/base/poco/Foundation/src/Thread.cpp index c3523bf7994..e4d16495ba3 100644 --- a/base/poco/Foundation/src/Thread.cpp +++ b/base/poco/Foundation/src/Thread.cpp @@ -20,15 +20,7 @@ #include -#if defined(POCO_OS_FAMILY_WINDOWS) -#if defined(_WIN32_WCE) -#include "Thread_WINCE.cpp" -#else -#include "Thread_WIN32.cpp" -#endif -#else #include "Thread_POSIX.cpp" -#endif namespace Poco { diff --git a/base/poco/Foundation/src/Timestamp.cpp b/base/poco/Foundation/src/Timestamp.cpp index a95964f65ba..ae99fec834b 100644 --- a/base/poco/Foundation/src/Timestamp.cpp +++ b/base/poco/Foundation/src/Timestamp.cpp @@ -24,11 +24,6 @@ #include #include #include -#elif defined(POCO_OS_FAMILY_WINDOWS) -#include "Poco/UnWindows.h" -#if defined(_WIN32_WCE) -#include -#endif #endif @@ -210,26 +205,7 @@ Timestamp Timestamp::fromUtcTime(UtcTimeVal val) void Timestamp::update() { -#if defined(POCO_OS_FAMILY_WINDOWS) - - FILETIME ft; -#if defined(_WIN32_WCE) && defined(POCO_WINCE_TIMESTAMP_HACK) - GetSystemTimeAsFileTimeWithMillisecondResolution(&ft); -#else - GetSystemTimeAsFileTime(&ft); -#endif - - ULARGE_INTEGER epoch; // UNIX epoch (1970-01-01 00:00:00) expressed in Windows NT FILETIME - epoch.LowPart = 0xD53E8000; - epoch.HighPart = 0x019DB1DE; - - ULARGE_INTEGER ts; - ts.LowPart = ft.dwLowDateTime; - ts.HighPart = ft.dwHighDateTime; - ts.QuadPart -= epoch.QuadPart; - _ts = ts.QuadPart/10; - -#elif defined(POCO_HAVE_CLOCK_GETTIME) +#if defined(POCO_HAVE_CLOCK_GETTIME) struct timespec ts; if (clock_gettime(CLOCK_REALTIME, &ts)) @@ -271,39 +247,6 @@ Timestamp& Timestamp::operator -= (const Timespan& span) } -#if defined(_WIN32) - - -Timestamp Timestamp::fromFileTimeNP(UInt32 fileTimeLow, UInt32 fileTimeHigh) -{ - ULARGE_INTEGER epoch; // UNIX epoch (1970-01-01 00:00:00) expressed in Windows NT FILETIME - epoch.LowPart = 0xD53E8000; - epoch.HighPart = 0x019DB1DE; - - ULARGE_INTEGER ts; - ts.LowPart = fileTimeLow; - ts.HighPart = fileTimeHigh; - ts.QuadPart -= epoch.QuadPart; - - return Timestamp(ts.QuadPart/10); -} - - -void Timestamp::toFileTimeNP(UInt32& fileTimeLow, UInt32& fileTimeHigh) const -{ - ULARGE_INTEGER epoch; // UNIX epoch (1970-01-01 00:00:00) expressed in Windows NT FILETIME - epoch.LowPart = 0xD53E8000; - epoch.HighPart = 0x019DB1DE; - - ULARGE_INTEGER ts; - ts.QuadPart = _ts*10; - ts.QuadPart += epoch.QuadPart; - fileTimeLow = ts.LowPart; - fileTimeHigh = ts.HighPart; -} - - -#endif } // namespace Poco diff --git a/base/poco/Foundation/src/Timezone.cpp b/base/poco/Foundation/src/Timezone.cpp index 2e062dcc6b5..267d3c26a3d 100644 --- a/base/poco/Foundation/src/Timezone.cpp +++ b/base/poco/Foundation/src/Timezone.cpp @@ -16,15 +16,7 @@ #include -#if defined(POCO_OS_FAMILY_WINDOWS) -#if defined(_WIN32_WCE) -#include "Timezone_WINCE.cpp" -#else -#include "Timezone_WIN32.cpp" -#endif -#else #include "Timezone_UNIX.cpp" -#endif namespace Poco { diff --git a/base/poco/Foundation/src/gzguts.h b/base/poco/Foundation/src/gzguts.h index 834e7c8a7e1..cccc0b5957e 100644 --- a/base/poco/Foundation/src/gzguts.h +++ b/base/poco/Foundation/src/gzguts.h @@ -31,9 +31,6 @@ #endif #include -#ifdef _WIN32 -# include -#endif #ifndef _WIN32_WCE # if defined(__TURBOC__) || defined(_MSC_VER) || defined(_WIN32) diff --git a/base/poco/Foundation/src/pcre.h b/base/poco/Foundation/src/pcre.h index 763465e25df..82dd7850391 100644 --- a/base/poco/Foundation/src/pcre.h +++ b/base/poco/Foundation/src/pcre.h @@ -51,19 +51,6 @@ imported have to be identified as such. When building PCRE, the appropriate export setting is defined in pcre_internal.h, which includes this file. So we don't change existing definitions of PCRE_EXP_DECL and PCRECPP_EXP_DECL. */ -#if defined(_WIN32) && !defined(PCRE_STATIC) -# ifndef PCRE_EXP_DECL -# define PCRE_EXP_DECL extern __declspec(dllimport) -# endif -# ifdef __cplusplus -# ifndef PCRECPP_EXP_DECL -# define PCRECPP_EXP_DECL extern __declspec(dllimport) -# endif -# ifndef PCRECPP_EXP_DEFN -# define PCRECPP_EXP_DEFN __declspec(dllimport) -# endif -# endif -#endif /* By default, we use the standard "extern" declarations. */ diff --git a/base/poco/Foundation/src/pcre_internal.h b/base/poco/Foundation/src/pcre_internal.h index 6310e0010b3..360ddd87673 100644 --- a/base/poco/Foundation/src/pcre_internal.h +++ b/base/poco/Foundation/src/pcre_internal.h @@ -146,17 +146,6 @@ exported symbols. That's why, in the non-Windows case, we set PCRE_EXP_DEFN and PCRE_EXP_DATA_DEFN only if they are not already set. */ # ifndef PCRE_EXP_DECL -# ifdef _WIN32 -# ifndef PCRE_STATIC -# define PCRE_EXP_DECL extern __declspec(dllexport) -# define PCRE_EXP_DEFN __declspec(dllexport) -# define PCRE_EXP_DATA_DEFN __declspec(dllexport) -# else -# define PCRE_EXP_DECL extern -# define PCRE_EXP_DEFN -# define PCRE_EXP_DATA_DEFN -# endif -# else # ifdef __cplusplus # define PCRE_EXP_DECL extern "C" # else @@ -168,7 +157,6 @@ PCRE_EXP_DATA_DEFN only if they are not already set. */ # ifndef PCRE_EXP_DATA_DEFN # define PCRE_EXP_DATA_DEFN # endif -# endif # endif /* When compiling with the MSVC compiler, it is sometimes necessary to include diff --git a/base/poco/Foundation/src/utils.h b/base/poco/Foundation/src/utils.h index 4f33d7c3ba3..bdfdaed9833 100644 --- a/base/poco/Foundation/src/utils.h +++ b/base/poco/Foundation/src/utils.h @@ -59,12 +59,7 @@ || defined(__nios2__) # define DOUBLE_CONVERSION_CORRECT_DOUBLE_OPERATIONS 1 #elif defined(_M_IX86) || defined(__i386__) || defined(__i386) -# if defined(_WIN32) -// Windows uses a 64bit wide floating point stack. -# define DOUBLE_CONVERSION_CORRECT_DOUBLE_OPERATIONS 1 -# else # undef DOUBLE_CONVERSION_CORRECT_DOUBLE_OPERATIONS -# endif // _WIN32 #else # error Target architecture was not detected as supported by Double-Conversion. #endif @@ -75,23 +70,9 @@ # define DOUBLE_CONVERSION_UNUSED #endif -#if defined(_WIN32) && !defined(__MINGW32__) - -typedef signed char int8_t; -typedef unsigned char uint8_t; -typedef short int16_t; // NOLINT -typedef unsigned short uint16_t; // NOLINT -typedef int int32_t; -typedef unsigned int uint32_t; -typedef __int64 int64_t; -typedef unsigned __int64 uint64_t; -// intptr_t and friends are defined in crtdefs.h through stdio.h. - -#else # include -#endif // The following macro works on both 32 and 64-bit platforms. // Usage: instead of writing 0x1234567890123456 diff --git a/base/poco/JSON/include/Poco/JSON/JSON.h b/base/poco/JSON/include/Poco/JSON/JSON.h index 2e1717a040d..984aafd8649 100644 --- a/base/poco/JSON/include/Poco/JSON/JSON.h +++ b/base/poco/JSON/include/Poco/JSON/JSON.h @@ -31,13 +31,6 @@ // JSON_API functions as being imported from a DLL, whereas this DLL sees symbols // defined with this macro as being exported. // -#if defined(_WIN32) && defined(POCO_DLL) -# if defined(JSON_EXPORTS) -# define JSON_API __declspec(dllexport) -# else -# define JSON_API __declspec(dllimport) -# endif -#endif #if !defined(JSON_API) diff --git a/base/poco/MongoDB/include/Poco/MongoDB/MongoDB.h b/base/poco/MongoDB/include/Poco/MongoDB/MongoDB.h index d336c1a8478..253f1f8ab27 100644 --- a/base/poco/MongoDB/include/Poco/MongoDB/MongoDB.h +++ b/base/poco/MongoDB/include/Poco/MongoDB/MongoDB.h @@ -33,13 +33,6 @@ // -#if defined(_WIN32) && defined(POCO_DLL) -# if defined(MongoDB_EXPORTS) -# define MongoDB_API __declspec(dllexport) -# else -# define MongoDB_API __declspec(dllimport) -# endif -#endif #if !defined(MongoDB_API) diff --git a/base/poco/Net/include/Poco/Net/IPAddress.h b/base/poco/Net/include/Poco/Net/IPAddress.h index c059f39f457..8f8b6fc1834 100644 --- a/base/poco/Net/include/Poco/Net/IPAddress.h +++ b/base/poco/Net/include/Poco/Net/IPAddress.h @@ -109,10 +109,6 @@ namespace Net IPAddress(unsigned prefix, Family family); /// Creates an IPAddress mask with the given length of prefix. -#if defined(_WIN32) - IPAddress(const SOCKET_ADDRESS & socket_address); - /// Creates an IPAddress from Windows SOCKET_ADDRESS structure. -#endif IPAddress(const struct sockaddr & sockaddr); /// Same for struct sock_addr on POSIX. diff --git a/base/poco/Net/include/Poco/Net/Net.h b/base/poco/Net/include/Poco/Net/Net.h index cc89471ae06..1a5bb3ac1df 100644 --- a/base/poco/Net/include/Poco/Net/Net.h +++ b/base/poco/Net/include/Poco/Net/Net.h @@ -31,13 +31,6 @@ // Net_API functions as being imported from a DLL, whereas this DLL sees symbols // defined with this macro as being exported. // -#if defined(_WIN32) && defined(POCO_DLL) -# if defined(Net_EXPORTS) -# define Net_API __declspec(dllexport) -# else -# define Net_API __declspec(dllimport) -# endif -#endif #if !defined(Net_API) @@ -86,27 +79,6 @@ namespace Net // Automate network initialization (only relevant on Windows). // -#if defined(POCO_OS_FAMILY_WINDOWS) && !defined(POCO_NO_AUTOMATIC_LIB_INIT) && !defined(__GNUC__) - -extern "C" const struct Net_API NetworkInitializer pocoNetworkInitializer; - -# if defined(Net_EXPORTS) -# if defined(_WIN64) || defined(_WIN32_WCE) -# define POCO_NET_FORCE_SYMBOL(s) __pragma(comment(linker, "/export:" # s)) -# elif defined(_WIN32) -# define POCO_NET_FORCE_SYMBOL(s) __pragma(comment(linker, "/export:_" # s)) -# endif -# else // !Net_EXPORTS -# if defined(_WIN64) || defined(_WIN32_WCE) -# define POCO_NET_FORCE_SYMBOL(s) __pragma(comment(linker, "/include:" # s)) -# elif defined(_WIN32) -# define POCO_NET_FORCE_SYMBOL(s) __pragma(comment(linker, "/include:_" # s)) -# endif -# endif // Net_EXPORTS - -POCO_NET_FORCE_SYMBOL(pocoNetworkInitializer) - -#endif // POCO_OS_FAMILY_WINDOWS // diff --git a/base/poco/Net/include/Poco/Net/NetworkInterface.h b/base/poco/Net/include/Poco/Net/NetworkInterface.h index 58685786c88..11a7eca116e 100644 --- a/base/poco/Net/include/Poco/Net/NetworkInterface.h +++ b/base/poco/Net/include/Poco/Net/NetworkInterface.h @@ -100,11 +100,7 @@ namespace Net }; static const unsigned NO_INDEX = ~0; -# if defined(POCO_OS_FAMILY_WINDOWS) - static const char MAC_SEPARATOR = '-'; -# else static const char MAC_SEPARATOR = ':'; -# endif NetworkInterface(unsigned index = NO_INDEX); /// Creates a NetworkInterface representing the diff --git a/base/poco/Net/include/Poco/Net/SocketDefs.h b/base/poco/Net/include/Poco/Net/SocketDefs.h index 0dc7e0e6012..5571633e91c 100644 --- a/base/poco/Net/include/Poco/Net/SocketDefs.h +++ b/base/poco/Net/include/Poco/Net/SocketDefs.h @@ -21,59 +21,7 @@ #define POCO_ENOERR 0 -#if defined(POCO_OS_FAMILY_WINDOWS) -# include "Poco/UnWindows.h" -# include -# include -# define POCO_INVALID_SOCKET INVALID_SOCKET -# define poco_socket_t SOCKET -# define poco_socklen_t int -# define poco_ioctl_request_t int -# define poco_closesocket(s) closesocket(s) -# define POCO_EINTR WSAEINTR -# define POCO_EACCES WSAEACCES -# define POCO_EFAULT WSAEFAULT -# define POCO_EINVAL WSAEINVAL -# define POCO_EMFILE WSAEMFILE -# define POCO_EAGAIN WSAEWOULDBLOCK -# define POCO_EWOULDBLOCK WSAEWOULDBLOCK -# define POCO_EINPROGRESS WSAEINPROGRESS -# define POCO_EALREADY WSAEALREADY -# define POCO_ENOTSOCK WSAENOTSOCK -# define POCO_EDESTADDRREQ WSAEDESTADDRREQ -# define POCO_EMSGSIZE WSAEMSGSIZE -# define POCO_EPROTOTYPE WSAEPROTOTYPE -# define POCO_ENOPROTOOPT WSAENOPROTOOPT -# define POCO_EPROTONOSUPPORT WSAEPROTONOSUPPORT -# define POCO_ESOCKTNOSUPPORT WSAESOCKTNOSUPPORT -# define POCO_ENOTSUP WSAEOPNOTSUPP -# define POCO_EPFNOSUPPORT WSAEPFNOSUPPORT -# define POCO_EAFNOSUPPORT WSAEAFNOSUPPORT -# define POCO_EADDRINUSE WSAEADDRINUSE -# define POCO_EADDRNOTAVAIL WSAEADDRNOTAVAIL -# define POCO_ENETDOWN WSAENETDOWN -# define POCO_ENETUNREACH WSAENETUNREACH -# define POCO_ENETRESET WSAENETRESET -# define POCO_ECONNABORTED WSAECONNABORTED -# define POCO_ECONNRESET WSAECONNRESET -# define POCO_ENOBUFS WSAENOBUFS -# define POCO_EISCONN WSAEISCONN -# define POCO_ENOTCONN WSAENOTCONN -# define POCO_ESHUTDOWN WSAESHUTDOWN -# define POCO_ETIMEDOUT WSAETIMEDOUT -# define POCO_ECONNREFUSED WSAECONNREFUSED -# define POCO_EHOSTDOWN WSAEHOSTDOWN -# define POCO_EHOSTUNREACH WSAEHOSTUNREACH -# define POCO_ESYSNOTREADY WSASYSNOTREADY -# define POCO_ENOTINIT WSANOTINITIALISED -# define POCO_HOST_NOT_FOUND WSAHOST_NOT_FOUND -# define POCO_TRY_AGAIN WSATRY_AGAIN -# define POCO_NO_RECOVERY WSANO_RECOVERY -# define POCO_NO_DATA WSANO_DATA -# ifndef ADDRESS_FAMILY -# define ADDRESS_FAMILY USHORT -# endif -#elif defined(POCO_OS_FAMILY_UNIX) +#if defined(POCO_OS_FAMILY_UNIX) # include # include # include @@ -274,11 +222,7 @@ extern "C" { #if !defined(s6_addr16) -# if defined(POCO_OS_FAMILY_WINDOWS) -# define s6_addr16 u.Word -# else # define s6_addr16 __u6_addr.__u6_addr16 -# endif #endif diff --git a/base/poco/Net/include/Poco/Net/SocketImpl.h b/base/poco/Net/include/Poco/Net/SocketImpl.h index 3f7337af63b..e08d49be7a2 100644 --- a/base/poco/Net/include/Poco/Net/SocketImpl.h +++ b/base/poco/Net/include/Poco/Net/SocketImpl.h @@ -485,11 +485,7 @@ namespace Net inline int SocketImpl::lastError() { -#if defined(_WIN32) - return WSAGetLastError(); -#else return errno; -#endif } diff --git a/base/poco/Net/src/DNS.cpp b/base/poco/Net/src/DNS.cpp index a74292e40fd..fdcc7782406 100644 --- a/base/poco/Net/src/DNS.cpp +++ b/base/poco/Net/src/DNS.cpp @@ -318,11 +318,7 @@ std::string DNS::decodeIDNLabel(const std::string& encodedIDN) int DNS::lastError() { -#if defined(_WIN32) - return GetLastError(); -#else return h_errno; -#endif } @@ -357,11 +353,9 @@ void DNS::aierror(int code, const std::string& arg) throw DNSException("Temporary DNS error while resolving", arg); case EAI_FAIL: throw DNSException("Non recoverable DNS error while resolving", arg); -#if !defined(_WIN32) // EAI_NODATA and EAI_NONAME have the same value #if defined(EAI_NODATA) // deprecated in favor of EAI_NONAME on FreeBSD case EAI_NODATA: throw NoAddressFoundException(arg); -#endif #endif case EAI_NONAME: throw HostNotFoundException(arg); @@ -369,10 +363,6 @@ void DNS::aierror(int code, const std::string& arg) case EAI_SYSTEM: error(lastError(), arg); break; -#endif -#if defined(_WIN32) - case WSANO_DATA: // may happen on XP - throw HostNotFoundException(arg); #endif default: throw DNSException("EAI", gai_strerror(code)); diff --git a/base/poco/Net/src/HTTPServerConnection.cpp b/base/poco/Net/src/HTTPServerConnection.cpp index a6c27ce8c9e..c57984b0162 100644 --- a/base/poco/Net/src/HTTPServerConnection.cpp +++ b/base/poco/Net/src/HTTPServerConnection.cpp @@ -148,11 +148,7 @@ void HTTPServerConnection::onServerStopped(const bool& abortCurrent) // Note: On Windows, select() will not return if one of its socket is being // shut down. Therefore we have to call close(), which works better. // On other platforms, we do the more graceful thing. -#if defined(_WIN32) - socket().close(); -#else socket().shutdown(); -#endif } catch (...) { @@ -164,11 +160,7 @@ void HTTPServerConnection::onServerStopped(const bool& abortCurrent) try { -#if defined(_WIN32) - socket().close(); -#else socket().shutdown(); -#endif } catch (...) { diff --git a/base/poco/Net/src/IPAddress.cpp b/base/poco/Net/src/IPAddress.cpp index 90d9400304a..5a7281e4be4 100644 --- a/base/poco/Net/src/IPAddress.cpp +++ b/base/poco/Net/src/IPAddress.cpp @@ -182,21 +182,6 @@ IPAddress::IPAddress(unsigned prefix, Family family) } -#if defined(_WIN32) -IPAddress::IPAddress(const SOCKET_ADDRESS& socket_address) - : _pImpl(0) -{ - ADDRESS_FAMILY family = socket_address.lpSockaddr->sa_family; - if (family == AF_INET) - newIPv4(&reinterpret_cast(socket_address.lpSockaddr)->sin_addr); -#if defined(POCO_HAVE_IPv6) - else if (family == AF_INET6) - newIPv6(&reinterpret_cast(socket_address.lpSockaddr)->sin6_addr, - reinterpret_cast(socket_address.lpSockaddr)->sin6_scope_id); -#endif - else throw Poco::InvalidArgumentException("Invalid or unsupported address family passed to IPAddress()"); -} -#endif IPAddress::IPAddress(const struct sockaddr& sockaddr) diff --git a/base/poco/Net/src/IPAddressImpl.cpp b/base/poco/Net/src/IPAddressImpl.cpp index f867acf8906..ec789bdd64d 100644 --- a/base/poco/Net/src/IPAddressImpl.cpp +++ b/base/poco/Net/src/IPAddressImpl.cpp @@ -256,14 +256,6 @@ bool IPv4AddressImpl::isGlobalMC() const IPv4AddressImpl IPv4AddressImpl::parse(const std::string& addr) { if (addr.empty()) return IPv4AddressImpl(); -#if defined(_WIN32) - struct in_addr ia; - ia.s_addr = inet_addr(addr.c_str()); - if (ia.s_addr == INADDR_NONE && addr != "255.255.255.255") - return IPv4AddressImpl(); - else - return IPv4AddressImpl(&ia); -#else #if __GNUC__ < 3 || defined(POCO_VXWORKS) struct in_addr ia; ia.s_addr = inet_addr(const_cast(addr.c_str())); @@ -278,7 +270,6 @@ IPv4AddressImpl IPv4AddressImpl::parse(const std::string& addr) else return IPv4AddressImpl(); #endif -#endif } @@ -388,20 +379,6 @@ IPv6AddressImpl::IPv6AddressImpl(unsigned prefix): _scope(0) { unsigned i = 0; -#ifdef POCO_OS_FAMILY_WINDOWS - for (; prefix >= 16; ++i, prefix -= 16) - { - _addr.s6_addr16[i] = 0xffff; - } - if (prefix > 0) - { - _addr.s6_addr16[i++] = ByteOrder::toNetwork(static_cast(~(0xffff >> prefix))); - } - while (i < 8) - { - _addr.s6_addr16[i++] = 0; - } -#else for (; prefix >= 32; ++i, prefix -= 32) { _addr.s6_addr32[i] = 0xffffffff; @@ -414,7 +391,6 @@ IPv6AddressImpl::IPv6AddressImpl(unsigned prefix): { _addr.s6_addr32[i++] = 0; } -#endif } @@ -467,9 +443,6 @@ std::string IPv6AddressImpl::toString() const if (_scope > 0) { result.append("%"); -#if defined(_WIN32) - NumberFormatter::append(result, _scope); -#else char buffer[IFNAMSIZ]; if (if_indextoname(_scope, buffer)) { @@ -479,7 +452,6 @@ std::string IPv6AddressImpl::toString() const { NumberFormatter::append(result, _scope); } -#endif } return toLower(result); } @@ -522,14 +494,6 @@ unsigned IPv6AddressImpl::prefixLength() const bitPos -= 32; } return 0; -#elif defined(POCO_OS_FAMILY_WINDOWS) - for (int i = 7; i >= 0; --i) - { - unsigned short addr = ByteOrder::fromNetwork(_addr.s6_addr16[i]); - if ((bits = maskBits(addr, 16))) return (bitPos - (16 - bits)); - bitPos -= 16; - } - return 0; #else #warning prefixLength() not implemented throw NotImplementedException("prefixLength() not implemented"); @@ -645,20 +609,6 @@ bool IPv6AddressImpl::isGlobalMC() const IPv6AddressImpl IPv6AddressImpl::parse(const std::string& addr) { if (addr.empty()) return IPv6AddressImpl(); -#if defined(_WIN32) - struct addrinfo* pAI; - struct addrinfo hints; - std::memset(&hints, 0, sizeof(hints)); - hints.ai_flags = AI_NUMERICHOST; - int rc = getaddrinfo(addr.c_str(), NULL, &hints, &pAI); - if (rc == 0) - { - IPv6AddressImpl result = IPv6AddressImpl(&reinterpret_cast(pAI->ai_addr)->sin6_addr, static_cast(reinterpret_cast(pAI->ai_addr)->sin6_scope_id)); - freeaddrinfo(pAI); - return result; - } - else return IPv6AddressImpl(); -#else struct in6_addr ia; std::string::size_type pos = addr.find('%'); if (std::string::npos != pos) @@ -681,7 +631,6 @@ IPv6AddressImpl IPv6AddressImpl::parse(const std::string& addr) else return IPv6AddressImpl(); } -#endif } @@ -703,21 +652,10 @@ IPv6AddressImpl IPv6AddressImpl::operator & (const IPv6AddressImpl& addr) const throw Poco::InvalidArgumentException("Scope ID of passed IPv6 address does not match with the source one."); IPv6AddressImpl result(*this); -#ifdef POCO_OS_FAMILY_WINDOWS - result._addr.s6_addr16[0] &= addr._addr.s6_addr16[0]; - result._addr.s6_addr16[1] &= addr._addr.s6_addr16[1]; - result._addr.s6_addr16[2] &= addr._addr.s6_addr16[2]; - result._addr.s6_addr16[3] &= addr._addr.s6_addr16[3]; - result._addr.s6_addr16[4] &= addr._addr.s6_addr16[4]; - result._addr.s6_addr16[5] &= addr._addr.s6_addr16[5]; - result._addr.s6_addr16[6] &= addr._addr.s6_addr16[6]; - result._addr.s6_addr16[7] &= addr._addr.s6_addr16[7]; -#else result._addr.s6_addr32[0] &= addr._addr.s6_addr32[0]; result._addr.s6_addr32[1] &= addr._addr.s6_addr32[1]; result._addr.s6_addr32[2] &= addr._addr.s6_addr32[2]; result._addr.s6_addr32[3] &= addr._addr.s6_addr32[3]; -#endif return result; } @@ -728,21 +666,10 @@ IPv6AddressImpl IPv6AddressImpl::operator | (const IPv6AddressImpl& addr) const throw Poco::InvalidArgumentException("Scope ID of passed IPv6 address does not match with the source one."); IPv6AddressImpl result(*this); -#ifdef POCO_OS_FAMILY_WINDOWS - result._addr.s6_addr16[0] |= addr._addr.s6_addr16[0]; - result._addr.s6_addr16[1] |= addr._addr.s6_addr16[1]; - result._addr.s6_addr16[2] |= addr._addr.s6_addr16[2]; - result._addr.s6_addr16[3] |= addr._addr.s6_addr16[3]; - result._addr.s6_addr16[4] |= addr._addr.s6_addr16[4]; - result._addr.s6_addr16[5] |= addr._addr.s6_addr16[5]; - result._addr.s6_addr16[6] |= addr._addr.s6_addr16[6]; - result._addr.s6_addr16[7] |= addr._addr.s6_addr16[7]; -#else result._addr.s6_addr32[0] |= addr._addr.s6_addr32[0]; result._addr.s6_addr32[1] |= addr._addr.s6_addr32[1]; result._addr.s6_addr32[2] |= addr._addr.s6_addr32[2]; result._addr.s6_addr32[3] |= addr._addr.s6_addr32[3]; -#endif return result; } @@ -754,21 +681,10 @@ IPv6AddressImpl IPv6AddressImpl::operator ^ (const IPv6AddressImpl& addr) const IPv6AddressImpl result(*this); -#ifdef POCO_OS_FAMILY_WINDOWS - result._addr.s6_addr16[0] ^= addr._addr.s6_addr16[0]; - result._addr.s6_addr16[1] ^= addr._addr.s6_addr16[1]; - result._addr.s6_addr16[2] ^= addr._addr.s6_addr16[2]; - result._addr.s6_addr16[3] ^= addr._addr.s6_addr16[3]; - result._addr.s6_addr16[4] ^= addr._addr.s6_addr16[4]; - result._addr.s6_addr16[5] ^= addr._addr.s6_addr16[5]; - result._addr.s6_addr16[6] ^= addr._addr.s6_addr16[6]; - result._addr.s6_addr16[7] ^= addr._addr.s6_addr16[7]; -#else result._addr.s6_addr32[0] ^= addr._addr.s6_addr32[0]; result._addr.s6_addr32[1] ^= addr._addr.s6_addr32[1]; result._addr.s6_addr32[2] ^= addr._addr.s6_addr32[2]; result._addr.s6_addr32[3] ^= addr._addr.s6_addr32[3]; -#endif return result; } @@ -776,21 +692,10 @@ IPv6AddressImpl IPv6AddressImpl::operator ^ (const IPv6AddressImpl& addr) const IPv6AddressImpl IPv6AddressImpl::operator ~ () const { IPv6AddressImpl result(*this); -#ifdef POCO_OS_FAMILY_WINDOWS - result._addr.s6_addr16[0] ^= 0xffff; - result._addr.s6_addr16[1] ^= 0xffff; - result._addr.s6_addr16[2] ^= 0xffff; - result._addr.s6_addr16[3] ^= 0xffff; - result._addr.s6_addr16[4] ^= 0xffff; - result._addr.s6_addr16[5] ^= 0xffff; - result._addr.s6_addr16[6] ^= 0xffff; - result._addr.s6_addr16[7] ^= 0xffff; -#else result._addr.s6_addr32[0] ^= 0xffffffff; result._addr.s6_addr32[1] ^= 0xffffffff; result._addr.s6_addr32[2] ^= 0xffffffff; result._addr.s6_addr32[3] ^= 0xffffffff; -#endif return result; } diff --git a/base/poco/Net/src/Net.cpp b/base/poco/Net/src/Net.cpp index e81b4b54590..2ddf8196563 100644 --- a/base/poco/Net/src/Net.cpp +++ b/base/poco/Net/src/Net.cpp @@ -25,52 +25,14 @@ namespace Net { void Net_API initializeNetwork() { -#if defined(POCO_OS_FAMILY_WINDOWS) - WORD version = MAKEWORD(2, 2); - WSADATA data; - if (WSAStartup(version, &data) != 0) - throw NetException("Failed to initialize network subsystem"); -#endif } void Net_API uninitializeNetwork() { -#if defined(POCO_OS_FAMILY_WINDOWS) - WSACleanup(); -#endif } } } // namespace Poco::Net -#if defined(POCO_OS_FAMILY_WINDOWS) && !defined(POCO_NO_AUTOMATIC_LIB_INIT) - - struct NetworkInitializer - /// Network initializer for windows statically - /// linked library. - { - NetworkInitializer() - /// Calls Poco::Net::initializeNetwork(); - { - Poco::Net::initializeNetwork(); - } - - ~NetworkInitializer() - /// Calls Poco::Net::uninitializeNetwork(); - { - try - { - Poco::Net::uninitializeNetwork(); - } - catch (...) - { - poco_unexpected(); - } - } - }; - - const NetworkInitializer pocoNetworkInitializer; - -#endif diff --git a/base/poco/Net/src/NetworkInterface.cpp b/base/poco/Net/src/NetworkInterface.cpp index e744de22277..0540c32b436 100644 --- a/base/poco/Net/src/NetworkInterface.cpp +++ b/base/poco/Net/src/NetworkInterface.cpp @@ -25,15 +25,6 @@ #include "Poco/StringTokenizer.h" #include "Poco/RefCountedObject.h" #include "Poco/Format.h" -#if defined(POCO_OS_FAMILY_WINDOWS) - #if defined(POCO_WIN32_UTF8) - #include "Poco/UnicodeConverter.h" - #endif - #include "Poco/Error.h" - #include - #include - #include -#endif #include #include #include @@ -123,12 +114,7 @@ public: bool running() const; bool up() const; -#if defined(POCO_OS_FAMILY_WINDOWS) - void setFlags(DWORD flags, DWORD iftype); - void setRunning(bool running); -#else void setFlags(short flags); -#endif void setUp(bool up); void setMTU(unsigned mtu); @@ -443,40 +429,6 @@ inline bool NetworkInterfaceImpl::up() const } -#if defined(POCO_OS_FAMILY_WINDOWS) - - -void NetworkInterfaceImpl::setFlags(DWORD flags, DWORD iftype) -{ - _running = _up = false; - switch (iftype) { - case IF_TYPE_ETHERNET_CSMACD: - case IF_TYPE_ISO88025_TOKENRING: - case IF_TYPE_IEEE80211: - _multicast = _broadcast = true; - break; - case IF_TYPE_SOFTWARE_LOOPBACK: - _loopback = true; - break; - case IF_TYPE_PPP: - case IF_TYPE_ATM: - case IF_TYPE_TUNNEL: - case IF_TYPE_IEEE1394: - _pointToPoint = true; - break; - } - if (!(flags & IP_ADAPTER_NO_MULTICAST)) - _multicast = true; -} - - -void NetworkInterfaceImpl::setRunning(bool running) -{ - _running = running; -} - - -#else void NetworkInterfaceImpl::setFlags(short flags) @@ -492,7 +444,6 @@ void NetworkInterfaceImpl::setFlags(short flags) } -#endif inline void NetworkInterfaceImpl::setUp(bool up) @@ -927,368 +878,7 @@ NetworkInterface::List NetworkInterface::list(bool ipOnly, bool upOnly) // -#if defined(POCO_OS_FAMILY_WINDOWS) -// -// Windows -// - - -#include "Poco/Buffer.h" -#include - - -namespace Poco { -namespace Net { - - -namespace { - - -IPAddress getBroadcastAddress(PIP_ADAPTER_PREFIX pPrefix, const IPAddress& addr, ULONG* pprefix = 0) - /// This function relies on (1) subnet prefix being at the position - /// immediately preceding and (2) broadcast address being at the position - /// immediately succeeding the IPv4 unicast address. - /// - /// Since there is no explicit guarantee on order, to ensure correctness, - /// the above constraints are checked prior to returning the result. - /// Additionally, on pre-Vista versions on Windows, the main structure does - /// not contain prefix length; for those platforms, this function - /// returns prefix through pprefix argument. -{ - PIP_ADAPTER_PREFIX pPrev = 0; - for (int i = 0; pPrefix; pPrefix = pPrefix->Next, ++i) - { - ADDRESS_FAMILY family = pPrefix->Address.lpSockaddr->sa_family; - if ((family == AF_INET) && (addr == IPAddress(pPrefix->Address))) - break; - pPrev = pPrefix; - } - - if (pPrefix && pPrefix->Next && pPrev) - { - IPAddress ipPrefix(pPrev->PrefixLength, IPAddress::IPv4); - IPAddress mask(pPrefix->Next->Address); - if ((ipPrefix & mask) == (ipPrefix & addr)) - { - if (pprefix) *pprefix = pPrefix->PrefixLength; - return IPAddress(pPrefix->Next->Address); - } - } - - return IPAddress(IPAddress::IPv4); -} - - -NetworkInterface::Type fromNative(DWORD type) -{ - switch (type) - { - case IF_TYPE_ETHERNET_CSMACD: return NetworkInterface::NI_TYPE_ETHERNET_CSMACD; - case IF_TYPE_ISO88025_TOKENRING: return NetworkInterface::NI_TYPE_ISO88025_TOKENRING; - case IF_TYPE_FRAMERELAY: return NetworkInterface::NI_TYPE_FRAMERELAY; - case IF_TYPE_PPP: return NetworkInterface::NI_TYPE_PPP; - case IF_TYPE_SOFTWARE_LOOPBACK: return NetworkInterface::NI_TYPE_SOFTWARE_LOOPBACK; - case IF_TYPE_ATM: return NetworkInterface::NI_TYPE_ATM; - case IF_TYPE_IEEE80211: return NetworkInterface::NI_TYPE_IEEE80211; - case IF_TYPE_TUNNEL: return NetworkInterface::NI_TYPE_TUNNEL; - case IF_TYPE_IEEE1394: return NetworkInterface::NI_TYPE_IEEE1394; - default: return NetworkInterface::NI_TYPE_OTHER; - } -} - - -IPAddress subnetMaskForInterface(const std::string& name, bool isLoopback) -{ - if (isLoopback) - { - return IPAddress::parse("255.0.0.0"); - } - else - { -#if !defined(_WIN32_WCE) - std::string subKey("SYSTEM\\CurrentControlSet\\services\\Tcpip\\Parameters\\Interfaces\\"); - subKey += name; - std::string netmask; - HKEY hKey; -#if defined(POCO_WIN32_UTF8) && !defined(POCO_NO_WSTRING) - std::wstring usubKey; - Poco::UnicodeConverter::toUTF16(subKey, usubKey); - if (RegOpenKeyExW(HKEY_LOCAL_MACHINE, usubKey.c_str(), 0, KEY_READ, &hKey) != ERROR_SUCCESS) - return IPAddress(); - wchar_t unetmask[16]; - DWORD size = sizeof(unetmask); - if (RegQueryValueExW(hKey, L"DhcpSubnetMask", NULL, NULL, (LPBYTE)&unetmask, &size) != ERROR_SUCCESS) - { - if (RegQueryValueExW(hKey, L"SubnetMask", NULL, NULL, (LPBYTE)&unetmask, &size) != ERROR_SUCCESS) - { - RegCloseKey(hKey); - return IPAddress(); - } - } - Poco::UnicodeConverter::toUTF8(unetmask, netmask); -#else - if (RegOpenKeyExA(HKEY_LOCAL_MACHINE, subKey.c_str(), 0, KEY_READ, &hKey) != ERROR_SUCCESS) - return IPAddress(); - char unetmask[16]; - DWORD size = sizeof(unetmask); - if (RegQueryValueExA(hKey, "DhcpSubnetMask", NULL, NULL, (LPBYTE)&unetmask, &size) != ERROR_SUCCESS) - { - if (RegQueryValueExA(hKey, "SubnetMask", NULL, NULL, (LPBYTE)&unetmask, &size) != ERROR_SUCCESS) - { - RegCloseKey(hKey); - return IPAddress(); - } - } - netmask = unetmask; -#endif - RegCloseKey(hKey); - return IPAddress::parse(netmask); -#else - return IPAddress(); -#endif // !defined(_WIN32_WCE) - } -} - - -} /// namespace - - -NetworkInterface::Map NetworkInterface::map(bool ipOnly, bool upOnly) -{ - OSVERSIONINFO osvi; - ZeroMemory(&osvi, sizeof(OSVERSIONINFO)); - osvi.dwOSVersionInfoSize = sizeof(OSVERSIONINFO); - GetVersionEx(&osvi); - - FastMutex::ScopedLock lock(_mutex); - Map result; - ULONG outBufLen = 16384; - Poco::Buffer memory(outBufLen); - ULONG flags = (GAA_FLAG_SKIP_ANYCAST | GAA_FLAG_SKIP_MULTICAST | GAA_FLAG_SKIP_DNS_SERVER | GAA_FLAG_INCLUDE_PREFIX); -#ifdef GAA_FLAG_INCLUDE_ALL_INTERFACES - flags |= GAA_FLAG_INCLUDE_ALL_INTERFACES; -#endif -#if defined(POCO_HAVE_IPv6) - const unsigned family = AF_UNSPEC; //IPv4 and IPv6 -#else - const unsigned family = AF_INET; //IPv4 only -#endif - DWORD dwRetVal = 0; - ULONG iterations = 0; - PIP_ADAPTER_ADDRESSES pAddress = 0; - do - { - pAddress = reinterpret_cast(memory.begin()); // leave in the loop, begin may change after resize - poco_assert (memory.capacity() >= outBufLen); - if (ERROR_BUFFER_OVERFLOW == (dwRetVal = GetAdaptersAddresses(family, flags, 0, pAddress, &outBufLen))) - memory.resize(outBufLen, false); // adjust size and try again - else if (ERROR_NO_DATA == dwRetVal) // no network interfaces found - return result; - else if (NO_ERROR != dwRetVal) // error occurred - throw SystemException(format("An error occurred while trying to obtain list of network interfaces: [%s]", Error::getMessage(dwRetVal))); - else - break; - } - while ((ERROR_BUFFER_OVERFLOW == dwRetVal) && (++iterations <= 2)); - - poco_assert (NO_ERROR == dwRetVal); - for (; pAddress; pAddress = pAddress->Next) - { - IPAddress address; - IPAddress subnetMask; - IPAddress broadcastAddress; - unsigned ifIndex = 0; - -#if defined(POCO_HAVE_IPv6) - #if defined(_WIN32_WCE) - ifIndex = pAddress->Ipv6IfIndex; - #elif (_WIN32_WINNT >= 0x0501) && (NTDDI_VERSION >= 0x05010100) // Win XP SP1 - #if defined (IP_ADAPTER_IPV6_ENABLED) // Vista - if(osvi.dwMajorVersion>=6)//vista - { - if ((pAddress->Flags & IP_ADAPTER_IPV6_ENABLED) && - (osvi.dwMajorVersion >= 5) && - (osvi.dwMinorVersion >= 1) && - (osvi.dwBuildNumber >=1)) - { - ifIndex = pAddress->Ipv6IfIndex; - } - } - else - { - if ((osvi.dwMajorVersion >= 5) && - (osvi.dwMinorVersion >= 1) && - (osvi.dwBuildNumber >= 1)) - { - ifIndex = pAddress->Ipv6IfIndex; - } - } - #else // !defined(IP_ADAPTER_IPV6_ENABLED) - if ((osvi.dwMajorVersion >= 5) && - (osvi.dwMinorVersion >= 1) && - (osvi.dwBuildNumber >= 1)) - { - ifIndex = pAddress->Ipv6IfIndex; - } - #endif // defined(IP_ADAPTER_IPV6_ENABLED) - #endif // (_WIN32_WINNT >= 0x0501) && (NTDDI_VERSION >= 0x05010100) -#endif // POCO_HAVE_IPv6 - -#if defined (IP_ADAPTER_IPV4_ENABLED) - if(osvi.dwMajorVersion>=6) - {//vista - if (pAddress->Flags & IP_ADAPTER_IPV4_ENABLED) - { - ifIndex = pAddress->IfIndex; - } - } - else - { - ifIndex = pAddress->IfIndex; - } -#else // !IP_ADAPTER_IPV4_ENABLED - ifIndex = pAddress->IfIndex; -#endif - if (ifIndex == 0) continue; - - std::string name; - std::string displayName; - std::string adapterName(pAddress->AdapterName); -#ifdef POCO_WIN32_UTF8 - Poco::UnicodeConverter::toUTF8(pAddress->FriendlyName, name); - Poco::UnicodeConverter::toUTF8(pAddress->Description, displayName); -#else - char nameBuffer[1024]; - int rc = WideCharToMultiByte(CP_ACP, 0, pAddress->FriendlyName, -1, nameBuffer, sizeof(nameBuffer), NULL, NULL); - if (rc) name = nameBuffer; - char displayNameBuffer[1024]; - rc = WideCharToMultiByte(CP_ACP, 0, pAddress->Description, -1, displayNameBuffer, sizeof(displayNameBuffer), NULL, NULL); - if (rc) displayName = displayNameBuffer; -#endif - - bool isUp = (pAddress->OperStatus == IfOperStatusUp); - bool isIP = (0 != pAddress->FirstUnicastAddress); - if (((ipOnly && isIP) || !ipOnly) && ((upOnly && isUp) || !upOnly)) - { - NetworkInterface ni(name, displayName, adapterName, ifIndex); - // Create interface even if it has an empty list of addresses; also, set - // physical attributes which are protocol independent (name, media type, - // MAC address, MTU, operational status, etc). - Map::iterator ifIt = result.find(ifIndex); - if (ifIt == result.end()) - ifIt = result.insert(Map::value_type(ifIndex, ni)).first; - - ifIt->second.impl().setFlags(pAddress->Flags, pAddress->IfType); - ifIt->second.impl().setMTU(pAddress->Mtu); - ifIt->second.impl().setUp(pAddress->OperStatus == IfOperStatusUp); -#if (_WIN32_WINNT >= 0x0600) // Vista and newer only - if ((osvi.dwMajorVersion >= 6) && - (osvi.dwMinorVersion >= 0) && - (osvi.dwBuildNumber >= 0)) - { - ifIt->second.impl().setRunning(pAddress->ReceiveLinkSpeed > 0 || pAddress->TransmitLinkSpeed > 0); - } -#endif - ifIt->second.impl().setType(fromNative(pAddress->IfType)); - if (pAddress->PhysicalAddressLength) - ifIt->second.impl().setMACAddress(pAddress->PhysicalAddress, pAddress->PhysicalAddressLength); - - for (PIP_ADAPTER_UNICAST_ADDRESS pUniAddr = pAddress->FirstUnicastAddress; - pUniAddr; - pUniAddr = pUniAddr->Next) - { - address = IPAddress(pUniAddr->Address); - ADDRESS_FAMILY family = pUniAddr->Address.lpSockaddr->sa_family; - switch (family) - { - case AF_INET: - { - // Windows lists broadcast address on localhost - bool hasBroadcast = (pAddress->IfType == IF_TYPE_ETHERNET_CSMACD) || (pAddress->IfType == IF_TYPE_SOFTWARE_LOOPBACK) || (pAddress->IfType == IF_TYPE_IEEE80211); - if (hasBroadcast) - { - // On Windows, a valid broadcast address will be all 1's (== address | ~subnetMask); additionally, on pre-Vista versions of - // OS, master address structure does not contain member for prefix length; we go an extra mile here in order to make sure - // we reflect the actual values held by system and protect against misconfiguration (e.g. bad DHCP config entry) - ULONG prefixLength = 0; -#if defined(_WIN32_WCE) - #if _WIN32_WCE >= 0x0800 - prefixLength = pUniAddr->OnLinkPrefixLength; - broadcastAddress = getBroadcastAddress(pAddress->FirstPrefix, address); - #else - broadcastAddress = getBroadcastAddress(pAddress->FirstPrefix, address, &prefixLength); - #endif - // if previous call did not do it, make last-ditch attempt for prefix and broadcast - if (prefixLength == 0 && pAddress->FirstPrefix) - prefixLength = pAddress->FirstPrefix->PrefixLength; - poco_assert (prefixLength <= 32); - if (broadcastAddress.isWildcard()) - { - IPAddress mask(static_cast(prefixLength), IPAddress::IPv4); - IPAddress host(mask & address); - broadcastAddress = host | ~mask; - } -#elif (_WIN32_WINNT >= 0x0501) && (NTDDI_VERSION >= 0x05010100) // Win XP SP1 - #if (_WIN32_WINNT >= 0x0600) // Vista and newer - if (osvi.dwMajorVersion >= 6) - { - prefixLength = pUniAddr->OnLinkPrefixLength; - broadcastAddress = getBroadcastAddress(pAddress->FirstPrefix, address); - } - else - { - broadcastAddress = getBroadcastAddress(pAddress->FirstPrefix, address, &prefixLength); - } - #else - broadcastAddress = getBroadcastAddress(pAddress->FirstPrefix, address, &prefixLength); - #endif - poco_assert (prefixLength <= 32); - if (broadcastAddress.isWildcard()) - { - IPAddress mask(static_cast(prefixLength), IPAddress::IPv4); - IPAddress host(mask & address); - broadcastAddress = host | ~mask; - } -#endif // (_WIN32_WINNT >= 0x0501) && (NTDDI_VERSION >= 0x05010100) - if (prefixLength) - { - subnetMask = IPAddress(static_cast(prefixLength), IPAddress::IPv4); - } - else // if all of the above fails, look up the subnet mask in the registry - { - address = IPAddress(&reinterpret_cast(pUniAddr->Address.lpSockaddr)->sin_addr, sizeof(in_addr)); - subnetMask = subnetMaskForInterface(name, address.isLoopback()); - if (!address.isLoopback()) - { - broadcastAddress = address; - broadcastAddress.mask(subnetMask, IPAddress::broadcast()); - } - } - ifIt->second.addAddress(address, subnetMask, broadcastAddress); - } - else - { - ifIt->second.addAddress(address); - } - } - break; -#if defined(POCO_HAVE_IPv6) - case AF_INET6: - ifIt->second.addAddress(address); - break; -#endif - } // switch family - } // for addresses - } // if ipOnly/upOnly - } // for adapters - return result; -} - - -} } // namespace Poco::Net - - -#elif defined(POCO_OS_FAMILY_BSD) || (POCO_OS == POCO_OS_QNX) || (POCO_OS == POCO_OS_SOLARIS) +#if defined(POCO_OS_FAMILY_BSD) || (POCO_OS == POCO_OS_QNX) || (POCO_OS == POCO_OS_SOLARIS) // // BSD variants, QNX(?) and Solaris // diff --git a/base/poco/Net/src/PollSet.cpp b/base/poco/Net/src/PollSet.cpp index ff580fc39cb..36acc912990 100644 --- a/base/poco/Net/src/PollSet.cpp +++ b/base/poco/Net/src/PollSet.cpp @@ -21,11 +21,7 @@ #include #endif -#if defined(_WIN32) && _WIN32_WINNT >= 0x0600 -#ifndef POCO_HAVE_FD_POLL -#define POCO_HAVE_FD_POLL 1 -#endif -#elif defined(POCO_OS_FAMILY_BSD) +#if defined(POCO_OS_FAMILY_BSD) #ifndef POCO_HAVE_FD_POLL #define POCO_HAVE_FD_POLL 1 #endif @@ -35,10 +31,8 @@ #if defined(POCO_HAVE_FD_EPOLL) #include #elif defined(POCO_HAVE_FD_POLL) -#ifndef _WIN32 #include #endif -#endif namespace Poco { @@ -296,11 +290,7 @@ public: do { Poco::Timestamp start; -#ifdef _WIN32 - rc = WSAPoll(&_pollfds[0], _pollfds.size(), static_cast(timeout.totalMilliseconds())); -#else rc = ::poll(&_pollfds[0], _pollfds.size(), timeout.totalMilliseconds()); -#endif if (rc < 0 && SocketImpl::lastError() == POCO_EINTR) { Poco::Timestamp end; @@ -330,10 +320,6 @@ public: result[its->second] |= PollSet::POLL_WRITE; if (it->revents & POLLERR) result[its->second] |= PollSet::POLL_ERROR; -#ifdef _WIN32 - if (it->revents & POLLHUP) - result[its->second] |= PollSet::POLL_READ; -#endif } it->revents = 0; } diff --git a/base/poco/Net/src/SocketImpl.cpp b/base/poco/Net/src/SocketImpl.cpp index 6fdcbc1af7d..2aba413b322 100644 --- a/base/poco/Net/src/SocketImpl.cpp +++ b/base/poco/Net/src/SocketImpl.cpp @@ -20,11 +20,7 @@ #include // FD_SET needs memset on some platforms, so we can't use -#if defined(_WIN32) && _WIN32_WINNT >= 0x0600 -#ifndef POCO_HAVE_FD_POLL -#define POCO_HAVE_FD_POLL 1 -#endif -#elif defined(POCO_OS_FAMILY_UNIX) +#if defined(POCO_OS_FAMILY_UNIX) #ifndef POCO_HAVE_FD_POLL #define POCO_HAVE_FD_POLL 1 #endif @@ -32,10 +28,8 @@ #if defined(POCO_HAVE_FD_POLL) -#ifndef _WIN32 #include #endif -#endif #if defined(sun) || defined(__sun) || defined(__sun__) @@ -44,9 +38,6 @@ #endif -#ifdef POCO_OS_FAMILY_WINDOWS -#include -#endif using Poco::IOException; @@ -64,13 +55,6 @@ bool checkIsBrokenTimeout() { #if defined(POCO_BROKEN_TIMEOUTS) return true; -#elif defined(POCO_OS_FAMILY_WINDOWS) - // on Windows 7 and lower, socket timeouts have a minimum of 500ms, use poll for timeouts on this case - // https://social.msdn.microsoft.com/Forums/en-US/76620f6d-22b1-4872-aaf0-833204f3f867/minimum-timeout-value-for-sorcvtimeo - OSVERSIONINFO vi; - vi.dwOSVersionInfoSize = sizeof(vi); - if (GetVersionEx(&vi) == 0) return true; //throw SystemException("Cannot get OS version information"); - return vi.dwMajorVersion < 6 || (vi.dwMajorVersion == 6 && vi.dwMinorVersion < 2); #endif return false; } @@ -445,11 +429,7 @@ bool SocketImpl::pollImpl(Poco::Timespan& remainingTime, int mode) do { Poco::Timestamp start; -#ifdef _WIN32 - rc = WSAPoll(&pollBuf, 1, static_cast(remainingTime.totalMilliseconds())); -#else rc = ::poll(&pollBuf, 1, remainingTime.totalMilliseconds()); -#endif /// Decrease timeout in case of retriable error. /// /// But do this only if the timeout is positive, @@ -550,12 +530,7 @@ int SocketImpl::getReceiveBufferSize() void SocketImpl::setSendTimeout(const Poco::Timespan& timeout) { -#if defined(_WIN32) - int value = (int) timeout.totalMilliseconds(); - setOption(SOL_SOCKET, SO_SNDTIMEO, value); -#else setOption(SOL_SOCKET, SO_SNDTIMEO, timeout); -#endif _sndTimeout = timeout; } @@ -563,11 +538,7 @@ void SocketImpl::setSendTimeout(const Poco::Timespan& timeout) Poco::Timespan SocketImpl::getSendTimeout() { Timespan result; -#if defined(_WIN32) && !defined(POCO_BROKEN_TIMEOUTS) - int value; - getOption(SOL_SOCKET, SO_SNDTIMEO, value); - result = Timespan::TimeDiff(value)*1000; -#elif !defined(POCO_BROKEN_TIMEOUTS) +#if !defined(POCO_BROKEN_TIMEOUTS) getOption(SOL_SOCKET, SO_SNDTIMEO, result); #endif if (_isBrokenTimeout) @@ -578,12 +549,7 @@ Poco::Timespan SocketImpl::getSendTimeout() void SocketImpl::setReceiveTimeout(const Poco::Timespan& timeout) { -#if defined(_WIN32) - int value = (int) timeout.totalMilliseconds(); - setOption(SOL_SOCKET, SO_RCVTIMEO, value); -#else setOption(SOL_SOCKET, SO_RCVTIMEO, timeout); -#endif _recvTimeout = timeout; } @@ -591,11 +557,7 @@ void SocketImpl::setReceiveTimeout(const Poco::Timespan& timeout) Poco::Timespan SocketImpl::getReceiveTimeout() { Timespan result; -#if defined(_WIN32) && !defined(POCO_BROKEN_TIMEOUTS) - int value; - getOption(SOL_SOCKET, SO_RCVTIMEO, value); - result = Timespan::TimeDiff(value)*1000; -#elif !defined(POCO_BROKEN_TIMEOUTS) +#if !defined(POCO_BROKEN_TIMEOUTS) getOption(SOL_SOCKET, SO_RCVTIMEO, result); #endif if (_isBrokenTimeout) @@ -905,22 +867,14 @@ void SocketImpl::initSocket(int af, int type, int proto) void SocketImpl::ioctl(poco_ioctl_request_t request, int& arg) { -#if defined(_WIN32) - int rc = ioctlsocket(_sockfd, request, reinterpret_cast(&arg)); -#else int rc = ::ioctl(_sockfd, request, &arg); -#endif if (rc != 0) error(); } void SocketImpl::ioctl(poco_ioctl_request_t request, void* arg) { -#if defined(_WIN32) - int rc = ioctlsocket(_sockfd, request, reinterpret_cast(arg)); -#else int rc = ::ioctl(_sockfd, request, arg); -#endif if (rc != 0) error(); } diff --git a/base/poco/Redis/include/Poco/Redis/Redis.h b/base/poco/Redis/include/Poco/Redis/Redis.h index ef9400fbfd7..0cd55263c3f 100644 --- a/base/poco/Redis/include/Poco/Redis/Redis.h +++ b/base/poco/Redis/include/Poco/Redis/Redis.h @@ -31,13 +31,6 @@ // Redis_API functions as being imported from a DLL, whereas this DLL sees symbols // defined with this macro as being exported. // -#if defined(_WIN32) && defined(POCO_DLL) -# if defined(Redis_EXPORTS) -# define Redis_API __declspec(dllexport) -# else -# define Redis_API __declspec(dllimport) -# endif -#endif #if !defined(Redis_API) diff --git a/base/poco/Util/include/Poco/Util/Application.h b/base/poco/Util/include/Poco/Util/Application.h index 0991fff7cb5..7743ad1126c 100644 --- a/base/poco/Util/include/Poco/Util/Application.h +++ b/base/poco/Util/include/Poco/Util/Application.h @@ -482,23 +482,6 @@ namespace Util // // Macro to implement main() // -#if defined(_WIN32) && defined(POCO_WIN32_UTF8) && !defined(POCO_NO_WSTRING) -# define POCO_APP_MAIN(App) \ - int wmain(int argc, wchar_t ** argv) \ - { \ - Poco::AutoPtr pApp = new App; \ - try \ - { \ - pApp->init(argc, argv); \ - } \ - catch (Poco::Exception & exc) \ - { \ - pApp->logger().log(exc); \ - return Poco::Util::Application::EXIT_CONFIG; \ - } \ - return pApp->run(); \ - } -#else # define POCO_APP_MAIN(App) \ int main(int argc, char ** argv) \ { \ @@ -514,7 +497,6 @@ namespace Util } \ return pApp->run(); \ } -#endif #endif // Util_Application_INCLUDED diff --git a/base/poco/Util/include/Poco/Util/ServerApplication.h b/base/poco/Util/include/Poco/Util/ServerApplication.h index 15294733078..b4db7ecbe9d 100644 --- a/base/poco/Util/include/Poco/Util/ServerApplication.h +++ b/base/poco/Util/include/Poco/Util/ServerApplication.h @@ -21,9 +21,6 @@ #include "Poco/Event.h" #include "Poco/Util/Application.h" #include "Poco/Util/Util.h" -#if defined(POCO_OS_FAMILY_WINDOWS) -# include "Poco/NamedEvent.h" -#endif namespace Poco @@ -178,43 +175,6 @@ namespace Util # if POCO_OS == POCO_OS_ANDROID static Poco::Event _terminate; # endif -#elif defined(POCO_OS_FAMILY_WINDOWS) -# if !defined(_WIN32_WCE) - enum Action - { - SRV_RUN, - SRV_REGISTER, - SRV_UNREGISTER - }; - static BOOL __stdcall ConsoleCtrlHandler(DWORD ctrlType); - static void __stdcall ServiceControlHandler(DWORD control); -# if defined(POCO_WIN32_UTF8) && !defined(POCO_NO_WSTRING) - static void __stdcall ServiceMain(DWORD argc, LPWSTR * argv); -# else - static void __stdcall ServiceMain(DWORD argc, LPTSTR * argv); -# endif - - bool hasConsole(); - bool isService(); - void beService(); - void registerService(); - void unregisterService(); - void handleRegisterService(const std::string & name, const std::string & value); - void handleUnregisterService(const std::string & name, const std::string & value); - void handleDisplayName(const std::string & name, const std::string & value); - void handleDescription(const std::string & name, const std::string & value); - void handleStartup(const std::string & name, const std::string & value); - - Action _action; - std::string _displayName; - std::string _description; - std::string _startup; - - static Poco::Event _terminated; - static SERVICE_STATUS _serviceStatus; - static SERVICE_STATUS_HANDLE _serviceStatusHandle; -# endif // _WIN32_WCE - static Poco::NamedEvent _terminate; #endif }; @@ -226,22 +186,6 @@ namespace Util // // Macro to implement main() // -#if defined(_WIN32) && defined(POCO_WIN32_UTF8) -# define POCO_SERVER_MAIN(App) \ - int wmain(int argc, wchar_t ** argv) \ - { \ - try \ - { \ - App app; \ - return app.run(argc, argv); \ - } \ - catch (Poco::Exception & exc) \ - { \ - std::cerr << exc.displayText() << std::endl; \ - return Poco::Util::Application::EXIT_SOFTWARE; \ - } \ - } -#else # define POCO_SERVER_MAIN(App) \ int main(int argc, char ** argv) \ { \ @@ -256,7 +200,6 @@ namespace Util return Poco::Util::Application::EXIT_SOFTWARE; \ } \ } -#endif #endif // Util_ServerApplication_INCLUDED diff --git a/base/poco/Util/include/Poco/Util/Util.h b/base/poco/Util/include/Poco/Util/Util.h index ea59e9192f1..d168a5e8f25 100644 --- a/base/poco/Util/include/Poco/Util/Util.h +++ b/base/poco/Util/include/Poco/Util/Util.h @@ -31,13 +31,6 @@ // Util_API functions as being imported from a DLL, whereas this DLL sees symbols // defined with this macro as being exported. // -#if defined(_WIN32) && defined(POCO_DLL) -# if defined(Util_EXPORTS) -# define Util_API __declspec(dllexport) -# else -# define Util_API __declspec(dllimport) -# endif -#endif #if !defined(Util_API) diff --git a/base/poco/Util/src/Application.cpp b/base/poco/Util/src/Application.cpp index c7a64eb7dba..e8b3a4feaa3 100644 --- a/base/poco/Util/src/Application.cpp +++ b/base/poco/Util/src/Application.cpp @@ -35,9 +35,6 @@ #include "Poco/String.h" #include "Poco/ConsoleChannel.h" #include "Poco/AutoPtr.h" -#if defined(POCO_OS_FAMILY_WINDOWS) -#include "Poco/UnWindows.h" -#endif #if defined(POCO_OS_FAMILY_UNIX) && !defined(POCO_VXWORKS) #include "Poco/SignalHandler.h" #endif @@ -436,25 +433,6 @@ void Application::getApplicationPath(Poco::Path& appPath) const appPath = Path(_workingDirAtLaunch, _command); appPath.makeAbsolute(); } -#elif defined(POCO_OS_FAMILY_WINDOWS) - #if defined(POCO_WIN32_UTF8) && !defined(POCO_NO_WSTRING) - wchar_t path[1024]; - int n = GetModuleFileNameW(0, path, sizeof(path)/sizeof(wchar_t)); - if (n > 0) - { - std::string p; - Poco::UnicodeConverter::toUTF8(path, p); - appPath = p; - } - else throw SystemException("Cannot get application file name."); - #else - char path[1024]; - int n = GetModuleFileNameA(0, path, sizeof(path)); - if (n > 0) - appPath = path; - else - throw SystemException("Cannot get application file name."); - #endif #else appPath = _command; #endif diff --git a/base/poco/Util/src/ServerApplication.cpp b/base/poco/Util/src/ServerApplication.cpp index 9da5d457d2f..b6f8205e38e 100644 --- a/base/poco/Util/src/ServerApplication.cpp +++ b/base/poco/Util/src/ServerApplication.cpp @@ -31,13 +31,6 @@ #include #include #include -#elif defined(POCO_OS_FAMILY_WINDOWS) -#if !defined(_WIN32_WCE) -#include "Poco/Util/WinService.h" -#include "Poco/Util/WinRegistryKey.h" -#endif -#include "Poco/UnWindows.h" -#include #endif #if defined(POCO_WIN32_UTF8) && !defined(POCO_NO_WSTRING) #include "Poco/UnicodeConverter.h" @@ -53,14 +46,6 @@ namespace Poco { namespace Util { -#if defined(POCO_OS_FAMILY_WINDOWS) -Poco::NamedEvent ServerApplication::_terminate(Poco::ProcessImpl::terminationEventName(Poco::Process::id())); -#if !defined(_WIN32_WCE) -Poco::Event ServerApplication::_terminated; -SERVICE_STATUS ServerApplication::_serviceStatus; -SERVICE_STATUS_HANDLE ServerApplication::_serviceStatusHandle = 0; -#endif -#endif #if defined(POCO_VXWORKS) || POCO_OS == POCO_OS_ANDROID Poco::Event ServerApplication::_terminate; #endif @@ -68,12 +53,6 @@ Poco::Event ServerApplication::_terminate; ServerApplication::ServerApplication() { -#if defined(POCO_OS_FAMILY_WINDOWS) -#if !defined(_WIN32_WCE) - _action = SRV_RUN; - std::memset(&_serviceStatus, 0, sizeof(_serviceStatus)); -#endif -#endif } @@ -97,9 +76,7 @@ int ServerApplication::run() void ServerApplication::terminate() { -#if defined(POCO_OS_FAMILY_WINDOWS) - _terminate.set(); -#elif defined(POCO_VXWORKS) || POCO_OS == POCO_OS_ANDROID +#if defined(POCO_VXWORKS) || POCO_OS == POCO_OS_ANDROID _terminate.set(); #else Poco::Process::requestTermination(Process::id()); @@ -107,413 +84,7 @@ void ServerApplication::terminate() } -#if defined(POCO_OS_FAMILY_WINDOWS) -#if !defined(_WIN32_WCE) - - -// -// Windows specific code -// -BOOL ServerApplication::ConsoleCtrlHandler(DWORD ctrlType) -{ - switch (ctrlType) - { - case CTRL_C_EVENT: - case CTRL_CLOSE_EVENT: - case CTRL_BREAK_EVENT: - terminate(); - return _terminated.tryWait(10000) ? TRUE : FALSE; - default: - return FALSE; - } -} - - -void ServerApplication::ServiceControlHandler(DWORD control) -{ - switch (control) - { - case SERVICE_CONTROL_STOP: - case SERVICE_CONTROL_SHUTDOWN: - terminate(); - _serviceStatus.dwCurrentState = SERVICE_STOP_PENDING; - break; - case SERVICE_CONTROL_INTERROGATE: - break; - } - SetServiceStatus(_serviceStatusHandle, &_serviceStatus); -} - - -#if defined(POCO_WIN32_UTF8) && !defined(POCO_NO_WSTRING) -void ServerApplication::ServiceMain(DWORD argc, LPWSTR* argv) -#else -void ServerApplication::ServiceMain(DWORD argc, LPTSTR* argv) -#endif -{ - ServerApplication& app = static_cast(Application::instance()); - - app.config().setBool("application.runAsService", true); - -#if defined(POCO_WIN32_UTF8) && !defined(POCO_NO_WSTRING) - _serviceStatusHandle = RegisterServiceCtrlHandlerW(L"", ServiceControlHandler); -#else - _serviceStatusHandle = RegisterServiceCtrlHandlerA("", ServiceControlHandler); -#endif - if (!_serviceStatusHandle) - throw SystemException("cannot register service control handler"); - - _serviceStatus.dwServiceType = SERVICE_WIN32; - _serviceStatus.dwCurrentState = SERVICE_START_PENDING; - _serviceStatus.dwControlsAccepted = SERVICE_ACCEPT_STOP | SERVICE_ACCEPT_SHUTDOWN; - _serviceStatus.dwWin32ExitCode = 0; - _serviceStatus.dwServiceSpecificExitCode = 0; - _serviceStatus.dwCheckPoint = 0; - _serviceStatus.dwWaitHint = 0; - SetServiceStatus(_serviceStatusHandle, &_serviceStatus); - - try - { -#if defined(POCO_WIN32_UTF8) && !defined(POCO_NO_WSTRING) - std::vector args; - for (DWORD i = 0; i < argc; ++i) - { - std::string arg; - Poco::UnicodeConverter::toUTF8(argv[i], arg); - args.push_back(arg); - } - app.init(args); -#else - app.init(argc, argv); -#endif - _serviceStatus.dwCurrentState = SERVICE_RUNNING; - SetServiceStatus(_serviceStatusHandle, &_serviceStatus); - int rc = app.run(); - _serviceStatus.dwWin32ExitCode = rc ? ERROR_SERVICE_SPECIFIC_ERROR : 0; - _serviceStatus.dwServiceSpecificExitCode = rc; - } - catch (Exception& exc) - { - app.logger().log(exc); - _serviceStatus.dwWin32ExitCode = ERROR_SERVICE_SPECIFIC_ERROR; - _serviceStatus.dwServiceSpecificExitCode = EXIT_CONFIG; - } - catch (...) - { - app.logger().error("fatal error - aborting"); - _serviceStatus.dwWin32ExitCode = ERROR_SERVICE_SPECIFIC_ERROR; - _serviceStatus.dwServiceSpecificExitCode = EXIT_SOFTWARE; - } - _serviceStatus.dwCurrentState = SERVICE_STOPPED; - SetServiceStatus(_serviceStatusHandle, &_serviceStatus); -} - - -void ServerApplication::waitForTerminationRequest() -{ - SetConsoleCtrlHandler(ConsoleCtrlHandler, TRUE); - _terminate.wait(); - _terminated.set(); -} - - -int ServerApplication::run(int argc, char** argv) -{ - if (!hasConsole() && isService()) - { - return 0; - } - else - { - int rc = EXIT_OK; - try - { - init(argc, argv); - switch (_action) - { - case SRV_REGISTER: - registerService(); - rc = EXIT_OK; - break; - case SRV_UNREGISTER: - unregisterService(); - rc = EXIT_OK; - break; - default: - rc = run(); - } - } - catch (Exception& exc) - { - logger().log(exc); - rc = EXIT_SOFTWARE; - } - return rc; - } -} - - -int ServerApplication::run(const std::vector& args) -{ - if (!hasConsole() && isService()) - { - return 0; - } - else - { - int rc = EXIT_OK; - try - { - init(args); - switch (_action) - { - case SRV_REGISTER: - registerService(); - rc = EXIT_OK; - break; - case SRV_UNREGISTER: - unregisterService(); - rc = EXIT_OK; - break; - default: - rc = run(); - } - } - catch (Exception& exc) - { - logger().log(exc); - rc = EXIT_SOFTWARE; - } - return rc; - } -} - - -#if defined(POCO_WIN32_UTF8) && !defined(POCO_NO_WSTRING) -int ServerApplication::run(int argc, wchar_t** argv) -{ - if (!hasConsole() && isService()) - { - return 0; - } - else - { - int rc = EXIT_OK; - try - { - init(argc, argv); - switch (_action) - { - case SRV_REGISTER: - registerService(); - rc = EXIT_OK; - break; - case SRV_UNREGISTER: - unregisterService(); - rc = EXIT_OK; - break; - default: - rc = run(); - } - } - catch (Exception& exc) - { - logger().log(exc); - rc = EXIT_SOFTWARE; - } - return rc; - } -} -#endif - - -bool ServerApplication::isService() -{ -#if defined(POCO_WIN32_UTF8) && !defined(POCO_NO_WSTRING) - SERVICE_TABLE_ENTRYW svcDispatchTable[2]; - svcDispatchTable[0].lpServiceName = L""; - svcDispatchTable[0].lpServiceProc = ServiceMain; - svcDispatchTable[1].lpServiceName = NULL; - svcDispatchTable[1].lpServiceProc = NULL; - return StartServiceCtrlDispatcherW(svcDispatchTable) != 0; -#else - SERVICE_TABLE_ENTRY svcDispatchTable[2]; - svcDispatchTable[0].lpServiceName = ""; - svcDispatchTable[0].lpServiceProc = ServiceMain; - svcDispatchTable[1].lpServiceName = NULL; - svcDispatchTable[1].lpServiceProc = NULL; - return StartServiceCtrlDispatcherA(svcDispatchTable) != 0; -#endif -} - - -bool ServerApplication::hasConsole() -{ - HANDLE hStdOut = GetStdHandle(STD_OUTPUT_HANDLE); - return hStdOut != INVALID_HANDLE_VALUE && hStdOut != NULL; -} - - -void ServerApplication::registerService() -{ - std::string name = config().getString("application.baseName"); - std::string path = config().getString("application.path"); - - WinService service(name); - if (_displayName.empty()) - service.registerService(path); - else - service.registerService(path, _displayName); - if (_startup == "auto") - service.setStartup(WinService::SVC_AUTO_START); - else if (_startup == "manual") - service.setStartup(WinService::SVC_MANUAL_START); - if (!_description.empty()) - service.setDescription(_description); - logger().information("The application has been successfully registered as a service."); -} - - -void ServerApplication::unregisterService() -{ - std::string name = config().getString("application.baseName"); - - WinService service(name); - service.unregisterService(); - logger().information("The service has been successfully unregistered."); -} - - -void ServerApplication::defineOptions(OptionSet& options) -{ - Application::defineOptions(options); - - options.addOption( - Option("registerService", "", "Register the application as a service.") - .required(false) - .repeatable(false) - .callback(OptionCallback(this, &ServerApplication::handleRegisterService))); - - options.addOption( - Option("unregisterService", "", "Unregister the application as a service.") - .required(false) - .repeatable(false) - .callback(OptionCallback(this, &ServerApplication::handleUnregisterService))); - - options.addOption( - Option("displayName", "", "Specify a display name for the service (only with /registerService).") - .required(false) - .repeatable(false) - .argument("name") - .callback(OptionCallback(this, &ServerApplication::handleDisplayName))); - - options.addOption( - Option("description", "", "Specify a description for the service (only with /registerService).") - .required(false) - .repeatable(false) - .argument("text") - .callback(OptionCallback(this, &ServerApplication::handleDescription))); - - options.addOption( - Option("startup", "", "Specify the startup mode for the service (only with /registerService).") - .required(false) - .repeatable(false) - .argument("automatic|manual") - .callback(OptionCallback(this, &ServerApplication::handleStartup))); -} - - -void ServerApplication::handleRegisterService(const std::string& name, const std::string& value) -{ - _action = SRV_REGISTER; -} - - -void ServerApplication::handleUnregisterService(const std::string& name, const std::string& value) -{ - _action = SRV_UNREGISTER; -} - - -void ServerApplication::handleDisplayName(const std::string& name, const std::string& value) -{ - _displayName = value; -} - - -void ServerApplication::handleDescription(const std::string& name, const std::string& value) -{ - _description = value; -} - - -void ServerApplication::handleStartup(const std::string& name, const std::string& value) -{ - if (Poco::icompare(value, 4, std::string("auto")) == 0) - _startup = "auto"; - else if (Poco::icompare(value, std::string("manual")) == 0) - _startup = "manual"; - else - throw InvalidArgumentException("argument to startup option must be 'auto[matic]' or 'manual'"); -} - - -#else // _WIN32_WCE -void ServerApplication::waitForTerminationRequest() -{ - _terminate.wait(); -} - - -int ServerApplication::run(int argc, char** argv) -{ - try - { - init(argc, argv); - } - catch (Exception& exc) - { - logger().log(exc); - return EXIT_CONFIG; - } - return run(); -} - - -int ServerApplication::run(const std::vector& args) -{ - try - { - init(args); - } - catch (Exception& exc) - { - logger().log(exc); - return EXIT_CONFIG; - } - return run(); -} - - -#if defined(POCO_WIN32_UTF8) && !defined(POCO_NO_WSTRING) -int ServerApplication::run(int argc, wchar_t** argv) -{ - try - { - init(argc, argv); - } - catch (Exception& exc) - { - logger().log(exc); - return EXIT_CONFIG; - } - return run(); -} -#endif - - -#endif // _WIN32_WCE -#elif defined(POCO_OS_FAMILY_UNIX) +#if defined(POCO_OS_FAMILY_UNIX) // diff --git a/base/poco/XML/include/Poco/XML/XML.h b/base/poco/XML/include/Poco/XML/XML.h index f527d1a9e41..982c9839555 100644 --- a/base/poco/XML/include/Poco/XML/XML.h +++ b/base/poco/XML/include/Poco/XML/XML.h @@ -31,13 +31,6 @@ // XML_API functions as being imported from a DLL, whereas this DLL sees symbols // defined with this macro as being exported. // -#if defined(_WIN32) && defined(POCO_DLL) -# if defined(XML_EXPORTS) -# define XML_API __declspec(dllexport) -# else -# define XML_API __declspec(dllimport) -# endif -#endif #if !defined(XML_API) diff --git a/base/poco/XML/src/XMLWriter.cpp b/base/poco/XML/src/XMLWriter.cpp index dba21692f0d..41831f48e67 100644 --- a/base/poco/XML/src/XMLWriter.cpp +++ b/base/poco/XML/src/XMLWriter.cpp @@ -151,11 +151,7 @@ void XMLWriter::setNewLine(const std::string& newLineCharacters) { if (newLineCharacters.empty()) { -#if defined(_WIN32) - _newLine = NEWLINE_CRLF; -#else _newLine = NEWLINE_LF; -#endif } else _newLine = newLineCharacters; } diff --git a/base/poco/XML/src/internal.h b/base/poco/XML/src/internal.h index c34e3ae3f83..9b93cec58f2 100644 --- a/base/poco/XML/src/internal.h +++ b/base/poco/XML/src/internal.h @@ -107,16 +107,6 @@ #include // ULONG_MAX -#if defined(_WIN32) && !defined(__USE_MINGW_ANSI_STDIO) -# define EXPAT_FMT_ULL(midpart) "%" midpart "I64u" -# if defined(_WIN64) // Note: modifiers "td" and "zu" do not work for MinGW -# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "I64d" -# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "I64u" -# else -# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "d" -# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "u" -# endif -#else # define EXPAT_FMT_ULL(midpart) "%" midpart "llu" # if !defined(ULONG_MAX) # error Compiler did not define ULONG_MAX for us @@ -127,7 +117,6 @@ # define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "d" # define EXPAT_FMT_SIZE_T(midpart) "%" midpart "u" # endif -#endif #ifndef UNUSED_P # define UNUSED_P(p) (void)p diff --git a/base/poco/XML/src/xmlparse.cpp b/base/poco/XML/src/xmlparse.cpp index 1ac894ff25c..dcd9d0facc6 100644 --- a/base/poco/XML/src/xmlparse.cpp +++ b/base/poco/XML/src/xmlparse.cpp @@ -61,12 +61,6 @@ # define _GNU_SOURCE 1 /* syscall prototype */ #endif -#ifdef _WIN32 -/* force stdlib to define rand_s() */ -# if ! defined(_CRT_RAND_S) -# define _CRT_RAND_S -# endif -#endif #include @@ -83,8 +77,6 @@ #if defined(EXPAT_POCO) # include "Poco/RandomStream.h" # include "Poco/BinaryReader.h" -#elif defined(_WIN32) -# define getpid GetCurrentProcessId #else # include /* gettimeofday() */ # include /* getpid() */ @@ -122,9 +114,6 @@ # include #endif -#if defined(_WIN32) && ! defined(LOAD_LIBRARY_SEARCH_SYSTEM32) -# define LOAD_LIBRARY_SEARCH_SYSTEM32 0x00000800 -#endif #if ! defined(HAVE_GETRANDOM) && ! defined(HAVE_SYSCALL_GETRANDOM) \ && ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) \ @@ -833,52 +822,11 @@ writeRandomBytes_arc4random(void *target, size_t count) { #endif /* defined(HAVE_ARC4RANDOM) && ! defined(HAVE_ARC4RANDOM_BUF) */ -#ifdef _WIN32 - -/* Provide declaration of rand_s() for MinGW-32 (not 64, which has it), - as it didn't declare it in its header prior to version 5.3.0 of its - runtime package (mingwrt, containing stdlib.h). The upstream fix - was introduced at https://osdn.net/projects/mingw/ticket/39658 . */ -# if defined(__MINGW32__) && defined(__MINGW32_VERSION) \ - && __MINGW32_VERSION < 5003000L && ! defined(__MINGW64_VERSION_MAJOR) -__declspec(dllimport) int rand_s(unsigned int *); -# endif - -/* Obtain entropy on Windows using the rand_s() function which - * generates cryptographically secure random numbers. Internally it - * uses RtlGenRandom API which is present in Windows XP and later. - */ -static int -writeRandomBytes_rand_s(void *target, size_t count) { - size_t bytesWrittenTotal = 0; - - while (bytesWrittenTotal < count) { - unsigned int random32 = 0; - size_t i = 0; - - if (rand_s(&random32)) - return 0; /* failure */ - - for (; (i < sizeof(random32)) && (bytesWrittenTotal < count); - i++, bytesWrittenTotal++) { - const uint8_t random8 = (uint8_t)(random32 >> (i * 8)); - ((uint8_t *)target)[bytesWrittenTotal] = random8; - } - } - return 1; /* success */ -} - -#endif /* _WIN32 */ #if ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) static unsigned long gather_time_entropy(void) { -# ifdef _WIN32 - FILETIME ft; - GetSystemTimeAsFileTime(&ft); /* never fails */ - return ft.dwHighDateTime ^ ft.dwLowDateTime; -# else struct timeval tv; int gettimeofday_res; @@ -892,7 +840,6 @@ gather_time_entropy(void) { /* Microseconds time is <20 bits entropy */ return tv.tv_usec; -# endif } #endif /* ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) */ @@ -927,11 +874,7 @@ generate_hash_secret_salt(XML_Parser parser) { return ENTROPY_DEBUG("arc4random", entropy); #else /* Try high quality providers first .. */ -# ifdef _WIN32 - if (writeRandomBytes_rand_s((void *)&entropy, sizeof(entropy))) { - return ENTROPY_DEBUG("rand_s", entropy); - } -# elif defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) +# if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) if (writeRandomBytes_getrandom_nonblock((void *)&entropy, sizeof(entropy))) { return ENTROPY_DEBUG("getrandom", entropy); } From e4298ebbb313fa47ba89850710986bc720956efb Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 14 Feb 2023 22:13:37 +0000 Subject: [PATCH 225/717] Remove WinCE --- base/poco/Foundation/include/Poco/Config.h | 3 - base/poco/Foundation/include/Poco/Platform.h | 3 - base/poco/Foundation/include/Poco/UnWindows.h | 2 - .../src/DirectoryIterator_WIN32U.cpp | 4 - base/poco/Foundation/src/LocalDateTime.cpp | 11 -- base/poco/Foundation/src/Random.cpp | 7 -- .../Foundation/src/SharedLibrary_WIN32U.cpp | 8 -- .../Foundation/src/SharedMemory_WIN32.cpp | 4 - base/poco/Foundation/src/ThreadPool.cpp | 15 --- base/poco/Foundation/src/Timestamp.cpp | 101 ------------------ base/poco/Foundation/src/Timezone_WINCE.cpp | 8 -- base/poco/Foundation/src/gzguts.h | 2 - base/poco/Net/src/TCPServer.cpp | 8 -- .../include/Poco/Util/ServerApplication.h | 2 - 14 files changed, 178 deletions(-) diff --git a/base/poco/Foundation/include/Poco/Config.h b/base/poco/Foundation/include/Poco/Config.h index 10be8cdb7ea..e3fcb9d91cd 100644 --- a/base/poco/Foundation/include/Poco/Config.h +++ b/base/poco/Foundation/include/Poco/Config.h @@ -159,9 +159,6 @@ // Windows CE has no locale support -#if defined(_WIN32_WCE) -# define POCO_NO_LOCALE -#endif // Enable the poco_debug_* and poco_trace_* macros diff --git a/base/poco/Foundation/include/Poco/Platform.h b/base/poco/Foundation/include/Poco/Platform.h index 49c9dfa1564..eb8f80a0d25 100644 --- a/base/poco/Foundation/include/Poco/Platform.h +++ b/base/poco/Foundation/include/Poco/Platform.h @@ -89,9 +89,6 @@ #elif defined(unix) || defined(__unix) || defined(__unix__) # define POCO_OS_FAMILY_UNIX 1 # define POCO_OS POCO_OS_UNKNOWN_UNIX -#elif defined(_WIN32_WCE) -# define POCO_OS_FAMILY_WINDOWS 1 -# define POCO_OS POCO_OS_WINDOWS_CE #elif defined(_WIN32) || defined(_WIN64) # define POCO_OS_FAMILY_WINDOWS 1 # define POCO_OS POCO_OS_WINDOWS_NT diff --git a/base/poco/Foundation/include/Poco/UnWindows.h b/base/poco/Foundation/include/Poco/UnWindows.h index 7609b9016e4..1f3835b8af5 100644 --- a/base/poco/Foundation/include/Poco/UnWindows.h +++ b/base/poco/Foundation/include/Poco/UnWindows.h @@ -51,7 +51,6 @@ // definitions.) For more information, see SdkDdkVer.h. -#if !defined(_WIN32_WCE) # if defined(_WIN32_WINNT) # if (_WIN32_WINNT < 0x0502) # error Unsupported Windows version. @@ -71,7 +70,6 @@ # define _WIN32_WINNT 0x0502 # define NTDDI_VERSION 0x05020000 # endif -#endif // To prevent Platform_WIN32.h to modify version defines, diff --git a/base/poco/Foundation/src/DirectoryIterator_WIN32U.cpp b/base/poco/Foundation/src/DirectoryIterator_WIN32U.cpp index 8f17bc31441..e72136bd478 100644 --- a/base/poco/Foundation/src/DirectoryIterator_WIN32U.cpp +++ b/base/poco/Foundation/src/DirectoryIterator_WIN32U.cpp @@ -13,11 +13,7 @@ #include "Poco/DirectoryIterator_WIN32U.h" -#if defined(_WIN32_WCE) -#include "Poco/File_WINCE.h" -#else #include "Poco/File_WIN32U.h" -#endif #include "Poco/Path.h" #include "Poco/UnicodeConverter.h" #include diff --git a/base/poco/Foundation/src/LocalDateTime.cpp b/base/poco/Foundation/src/LocalDateTime.cpp index 29093290f76..33d67891fc9 100644 --- a/base/poco/Foundation/src/LocalDateTime.cpp +++ b/base/poco/Foundation/src/LocalDateTime.cpp @@ -18,9 +18,6 @@ #include "Poco/Exception.h" #include #include -#if defined(_WIN32_WCE) && _WIN32_WCE < 0x800 -#include "wce_time.h" -#endif namespace Poco { @@ -261,11 +258,7 @@ void LocalDateTime::determineTzd(bool adjust) { std::time_t epochTime = _dateTime.timestamp().epochTime(); #if defined(_WIN32) || defined(POCO_NO_POSIX_TSF) -#if defined(_WIN32_WCE) && _WIN32_WCE < 0x800 - std::tm* broken = wceex_localtime(&epochTime); -#else std::tm* broken = std::localtime(&epochTime); -#endif if (!broken) throw Poco::SystemException("cannot get local time"); _tzd = (Timezone::utcOffset() + ((broken->tm_isdst == 1) ? 3600 : 0)); #else @@ -297,11 +290,7 @@ std::time_t LocalDateTime::dstOffset(int& dstOffset) const broken.tm_min = _dateTime.minute(); broken.tm_sec = _dateTime.second(); broken.tm_isdst = -1; -#if defined(_WIN32_WCE) && _WIN32_WCE < 0x800 - local = wceex_mktime(&broken); -#else local = std::mktime(&broken); -#endif dstOffset = (broken.tm_isdst == 1) ? 3600 : 0; return local; diff --git a/base/poco/Foundation/src/Random.cpp b/base/poco/Foundation/src/Random.cpp index 10bf730374e..93e6bd678b1 100644 --- a/base/poco/Foundation/src/Random.cpp +++ b/base/poco/Foundation/src/Random.cpp @@ -47,9 +47,6 @@ #include "Poco/Random.h" #include "Poco/RandomStream.h" #include -#if defined(_WIN32_WCE) && _WIN32_WCE < 0x800 -#include "wce_time.h" -#endif /* @@ -153,11 +150,7 @@ Random::Random(int stateSize) poco_assert (BREAK_0 <= stateSize && stateSize <= BREAK_4); _pBuffer = new char[stateSize]; -#if defined(_WIN32_WCE) && _WIN32_WCE < 0x800 - initState((UInt32) wceex_time(NULL), _pBuffer, stateSize); -#else initState((UInt32) std::time(NULL), _pBuffer, stateSize); -#endif } diff --git a/base/poco/Foundation/src/SharedLibrary_WIN32U.cpp b/base/poco/Foundation/src/SharedLibrary_WIN32U.cpp index b4f697ea0ac..0a52be7966c 100644 --- a/base/poco/Foundation/src/SharedLibrary_WIN32U.cpp +++ b/base/poco/Foundation/src/SharedLibrary_WIN32U.cpp @@ -41,10 +41,8 @@ void SharedLibraryImpl::loadImpl(const std::string& path, int /*flags*/) if (_handle) throw LibraryAlreadyLoadedException(_path); DWORD flags(0); -#if !defined(_WIN32_WCE) Path p(path); if (p.isAbsolute()) flags |= LOAD_WITH_ALTERED_SEARCH_PATH; -#endif std::wstring upath; UnicodeConverter::toUTF16(path, upath); _handle = LoadLibraryExW(upath.c_str(), 0, flags); @@ -78,13 +76,7 @@ void* SharedLibraryImpl::findSymbolImpl(const std::string& name) if (_handle) { -#if defined(_WIN32_WCE) - std::wstring uname; - UnicodeConverter::toUTF16(name, uname); - return (void*) GetProcAddressW((HMODULE) _handle, uname.c_str()); -#else return (void*) GetProcAddress((HMODULE) _handle, name.c_str()); -#endif } else return 0; } diff --git a/base/poco/Foundation/src/SharedMemory_WIN32.cpp b/base/poco/Foundation/src/SharedMemory_WIN32.cpp index 1e86d7d10ed..eadf37bd018 100644 --- a/base/poco/Foundation/src/SharedMemory_WIN32.cpp +++ b/base/poco/Foundation/src/SharedMemory_WIN32.cpp @@ -48,9 +48,6 @@ SharedMemoryImpl::SharedMemoryImpl(const std::string& name, std::size_t size, Sh if (!_memHandle) { DWORD dwRetVal = GetLastError(); -#if defined (_WIN32_WCE) - throw SystemException(format("Cannot create shared memory object %s [Error %d: %s]", _name, static_cast(dwRetVal), Error::getMessage(dwRetVal))); -#else if (_mode != PAGE_READONLY || dwRetVal != 5) throw SystemException(format("Cannot create shared memory object %s [Error %d: %s]", _name, static_cast(dwRetVal), Error::getMessage(dwRetVal))); @@ -64,7 +61,6 @@ SharedMemoryImpl::SharedMemoryImpl(const std::string& name, std::size_t size, Sh dwRetVal = GetLastError(); throw SystemException(format("Cannot open shared memory object %s [Error %d: %s]", _name, static_cast(dwRetVal), Error::getMessage(dwRetVal))); } -#endif } map(); } diff --git a/base/poco/Foundation/src/ThreadPool.cpp b/base/poco/Foundation/src/ThreadPool.cpp index 39846945042..6335ee82b47 100644 --- a/base/poco/Foundation/src/ThreadPool.cpp +++ b/base/poco/Foundation/src/ThreadPool.cpp @@ -20,9 +20,6 @@ #include "Poco/ErrorHandler.h" #include #include -#if defined(_WIN32_WCE) && _WIN32_WCE < 0x800 -#include "wce_time.h" -#endif namespace Poco { @@ -67,11 +64,7 @@ PooledThread::PooledThread(const std::string& name, int stackSize): { poco_assert_dbg (stackSize >= 0); _thread.setStackSize(stackSize); -#if defined(_WIN32_WCE) && _WIN32_WCE < 0x800 - _idleTime = wceex_time(NULL); -#else _idleTime = std::time(NULL); -#endif } @@ -135,11 +128,7 @@ int PooledThread::idleTime() { FastMutex::ScopedLock lock(_mutex); -#if defined(_WIN32_WCE) && _WIN32_WCE < 0x800 - return (int) (wceex_time(NULL) - _idleTime); -#else return (int) (time(NULL) - _idleTime); -#endif } @@ -212,11 +201,7 @@ void PooledThread::run() } FastMutex::ScopedLock lock(_mutex); _pTarget = 0; -#if defined(_WIN32_WCE) && _WIN32_WCE < 0x800 - _idleTime = wceex_time(NULL); -#else _idleTime = time(NULL); -#endif _idle = true; _targetCompleted.set(); ThreadLocalStorage::clear(); diff --git a/base/poco/Foundation/src/Timestamp.cpp b/base/poco/Foundation/src/Timestamp.cpp index ae99fec834b..be9c32d5607 100644 --- a/base/poco/Foundation/src/Timestamp.cpp +++ b/base/poco/Foundation/src/Timestamp.cpp @@ -36,107 +36,6 @@ #endif -#if defined(_WIN32_WCE) && defined(POCO_WINCE_TIMESTAMP_HACK) - - -// -// See -// for an explanation of the following code. -// -// In short: Windows CE system time in most cases only has a resolution of one second. -// But we want millisecond resolution. -// - - -namespace { - - -class TickOffset -{ -public: - TickOffset() - { - SYSTEMTIME st1, st2; - std::memset(&st1, 0, sizeof(SYSTEMTIME)); - std::memset(&st2, 0, sizeof(SYSTEMTIME)); - GetSystemTime(&st1); - while (true) - { - GetSystemTime(&st2); - - // wait for a rollover - if (st1.wSecond != st2.wSecond) - { - _offset = GetTickCount() % 1000; - break; - } - } - } - - void calibrate(int seconds) - { - SYSTEMTIME st1, st2; - systemTime(&st1); - - WORD s = st1.wSecond; - int sum = 0; - int remaining = seconds; - while (remaining > 0) - { - systemTime(&st2); - WORD s2 = st2.wSecond; - - if (s != s2) - { - remaining--; - // store the offset from zero - sum += (st2.wMilliseconds > 500) ? (st2.wMilliseconds - 1000) : st2.wMilliseconds; - s = st2.wSecond; - } - } - - // adjust the offset by the average deviation from zero (round to the integer farthest from zero) - if (sum < 0) - _offset += (int) std::floor(sum / (float)seconds); - else - _offset += (int) std::ceil(sum / (float)seconds); - } - - void systemTime(SYSTEMTIME* pST) - { - std::memset(pST, 0, sizeof(SYSTEMTIME)); - - WORD tick = GetTickCount() % 1000; - GetSystemTime(pST); - WORD ms = (tick >= _offset) ? (tick - _offset) : (1000 - (_offset - tick)); - pST->wMilliseconds = ms; - } - - void systemTimeAsFileTime(FILETIME* pFT) - { - SYSTEMTIME st; - systemTime(&st); - SystemTimeToFileTime(&st, pFT); - } - -private: - WORD _offset; -}; - - -static TickOffset offset; - - -void GetSystemTimeAsFileTimeWithMillisecondResolution(FILETIME* pFT) -{ - offset.systemTimeAsFileTime(pFT); -} - - -} // namespace - - -#endif // defined(_WIN32_WCE) && defined(POCO_WINCE_TIMESTAMP_HACK) namespace Poco { diff --git a/base/poco/Foundation/src/Timezone_WINCE.cpp b/base/poco/Foundation/src/Timezone_WINCE.cpp index fb794f58b4e..c0cd7c28684 100644 --- a/base/poco/Foundation/src/Timezone_WINCE.cpp +++ b/base/poco/Foundation/src/Timezone_WINCE.cpp @@ -17,11 +17,7 @@ #include "Poco/Exception.h" #include "Poco/UnWindows.h" #include -#if _WIN32_WCE >= 0x800 -#include "time.h" -#else #include "wce_time.h" -#endif namespace Poco { @@ -46,11 +42,7 @@ int Timezone::dst() bool Timezone::isDst(const Timestamp& timestamp) { std::time_t time = timestamp.epochTime(); -#if _WIN32_WCE >= 0x800 - struct std::tm* tms = localtime(&time); -#else struct std::tm* tms = wceex_localtime(&time); -#endif if (!tms) throw SystemException("cannot get local time DST flag"); return tms->tm_isdst > 0; } diff --git a/base/poco/Foundation/src/gzguts.h b/base/poco/Foundation/src/gzguts.h index cccc0b5957e..1b964756065 100644 --- a/base/poco/Foundation/src/gzguts.h +++ b/base/poco/Foundation/src/gzguts.h @@ -32,11 +32,9 @@ #include -#ifndef _WIN32_WCE # if defined(__TURBOC__) || defined(_MSC_VER) || defined(_WIN32) # include # endif -#endif #if defined(_WIN32) || defined(__CYGWIN__) # define WIDECHAR #endif diff --git a/base/poco/Net/src/TCPServer.cpp b/base/poco/Net/src/TCPServer.cpp index 7a7282f785c..9bdae900bd6 100644 --- a/base/poco/Net/src/TCPServer.cpp +++ b/base/poco/Net/src/TCPServer.cpp @@ -225,16 +225,8 @@ void TCPServer::setConnectionFilter(const TCPServerConnectionFilter::Ptr& pConne std::string TCPServer::threadName(const ServerSocket& socket) { -#if _WIN32_WCE == 0x0800 - // Workaround for WEC2013: only the first call to getsockname() - // succeeds. To mitigate the impact of this bug, do not call - // socket.address(), which calls getsockname(), here. - std::string name("TCPServer"); - #pragma message("Using WEC2013 getsockname() workaround in TCPServer::threadName(). Remove when no longer needed.") -#else std::string name("TCPServer: "); name.append(socket.address().toString()); -#endif return name; } diff --git a/base/poco/Util/include/Poco/Util/ServerApplication.h b/base/poco/Util/include/Poco/Util/ServerApplication.h index b4db7ecbe9d..3e90f4505d7 100644 --- a/base/poco/Util/include/Poco/Util/ServerApplication.h +++ b/base/poco/Util/include/Poco/Util/ServerApplication.h @@ -161,9 +161,7 @@ namespace Util protected: int run(); virtual void waitForTerminationRequest(); -#if !defined(_WIN32_WCE) void defineOptions(OptionSet & options); -#endif private: #if defined(POCO_OS_FAMILY_UNIX) From 1fc28c883f8105ddc81ac857ab733606ddf2e895 Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Tue, 14 Feb 2023 22:17:23 +0000 Subject: [PATCH 226/717] Fix tests --- src/Storages/StorageMaterializedView.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp index be090bd9a62..28589cef7a0 100644 --- a/src/Storages/StorageMaterializedView.cpp +++ b/src/Storages/StorageMaterializedView.cpp @@ -347,7 +347,7 @@ void StorageMaterializedView::renameInMemory(const StorageID & new_table_id) }, ASTRenameQuery::Table { - target_table_id.database_name.empty() ? nullptr : std::make_shared(target_table_id.database_name), + new_table_id.database_name.empty() ? nullptr : std::make_shared(new_table_id.database_name), std::make_shared(new_target_table_name) } }; From ff5023049b0b8ea72c1ee911e6cd4c46a01e0242 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 15 Feb 2023 02:17:02 +0300 Subject: [PATCH 227/717] Update test.py --- .../test_replicated_merge_tree_s3_zero_copy/test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py b/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py index 434957de53f..158d559d4b9 100644 --- a/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py +++ b/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py @@ -174,10 +174,10 @@ def test_drop_table(cluster): ) if "1" in replicas and "test_drop_table" not in node.query("show tables"): node2.query("system drop replica '1' from table test_drop_table") - node.query( - "create table test_drop_table (n int) engine=ReplicatedMergeTree('/test/drop_table', '1') " - "order by n partition by n % 99 settings storage_policy='s3'" - ) + node.query( + "create table if not exists test_drop_table (n int) engine=ReplicatedMergeTree('/test/drop_table', '1') " + "order by n partition by n % 99 settings storage_policy='s3'" + ) node.query("system sync replica test_drop_table", settings={"receive_timeout": 60}) node2.query("drop table test_drop_table") assert "1000\t499500\n" == node.query( From f31451822e0eedc290889ffd93538321a1bd379e Mon Sep 17 00:00:00 2001 From: flynn Date: Wed, 15 Feb 2023 03:56:01 +0000 Subject: [PATCH 228/717] fix --- src/Storages/IStorageDataLake.h | 6 +-- ...r.cpp => S3DataLakeMetadataReadHelper.cpp} | 10 ++-- ...elper.h => S3DataLakeMetadataReadHelper.h} | 7 +-- src/Storages/StorageDeltaLake.cpp | 22 ++++---- src/Storages/StorageDeltaLake.h | 8 +-- src/Storages/StorageHudi.cpp | 12 ++--- src/Storages/StorageHudi.h | 8 +-- src/Storages/StorageIceberg.cpp | 50 +++++++++---------- src/Storages/StorageIceberg.h | 10 ++-- src/Storages/StorageS3.h | 22 ++++---- src/TableFunctions/TableFunctionS3.cpp | 6 +-- src/TableFunctions/TableFunctionS3.h | 9 ++-- 12 files changed, 83 insertions(+), 87 deletions(-) rename src/Storages/{S3DataLakeMetaReadHelper.cpp => S3DataLakeMetadataReadHelper.cpp} (89%) rename src/Storages/{S3DataLakeMetaReadHelper.h => S3DataLakeMetadataReadHelper.h} (79%) diff --git a/src/Storages/IStorageDataLake.h b/src/Storages/IStorageDataLake.h index 1b471d48525..687c50b4082 100644 --- a/src/Storages/IStorageDataLake.h +++ b/src/Storages/IStorageDataLake.h @@ -61,7 +61,7 @@ static const std::unordered_set optional_configuration_keys = "secret_access_key", }; -template +template class IStorageDataLake : public IStorage { public: @@ -141,11 +141,11 @@ public: static StorageS3::Configuration getAdjustedS3Configuration(const ContextPtr & context, StorageS3::Configuration & configuration, Poco::Logger * log) { - MetaParser parser{configuration, context}; + MetadataParser parser{configuration, context}; auto keys = parser.getFiles(); String new_uri = std::filesystem::path(configuration.url.uri.toString()) / Name::data_directory_prefix - / MetaParser::generateQueryFromKeys(keys, configuration.format); + / MetadataParser::generateQueryFromKeys(keys, configuration.format); StorageS3::Configuration new_configuration(configuration); new_configuration.url = S3::URI(new_uri); diff --git a/src/Storages/S3DataLakeMetaReadHelper.cpp b/src/Storages/S3DataLakeMetadataReadHelper.cpp similarity index 89% rename from src/Storages/S3DataLakeMetaReadHelper.cpp rename to src/Storages/S3DataLakeMetadataReadHelper.cpp index c9e1d822abd..2890cc5d6e0 100644 --- a/src/Storages/S3DataLakeMetaReadHelper.cpp +++ b/src/Storages/S3DataLakeMetadataReadHelper.cpp @@ -2,9 +2,10 @@ #if USE_AWS_S3 +# include # include # include -# include +# include # include # include # include @@ -19,7 +20,7 @@ namespace ErrorCodes } std::shared_ptr -S3DataLakeMetaReadHelper::createReadBuffer(const String & key, ContextPtr context, const StorageS3::Configuration & base_configuration) +S3DataLakeMetadataReadHelper::createReadBuffer(const String & key, ContextPtr context, const StorageS3::Configuration & base_configuration) { S3Settings::RequestSettings request_settings; request_settings.max_single_read_retries = context->getSettingsRef().s3_max_single_read_retries; @@ -31,7 +32,8 @@ S3DataLakeMetaReadHelper::createReadBuffer(const String & key, ContextPtr contex request_settings, context->getReadSettings()); } -std::vector S3DataLakeMetaReadHelper::listFilesMatchSuffix( + +std::vector S3DataLakeMetadataReadHelper::listFilesMatchSuffix( const StorageS3::Configuration & base_configuration, const String & directory, const String & suffix) { const auto & table_path = base_configuration.url.key; @@ -77,7 +79,7 @@ std::vector S3DataLakeMetaReadHelper::listFilesMatchSuffix( return res; } -std::vector S3DataLakeMetaReadHelper::listFiles(const StorageS3::Configuration & configuration) +std::vector S3DataLakeMetadataReadHelper::listFiles(const StorageS3::Configuration & configuration) { const auto & client = configuration.client; const auto & table_path = configuration.url.key; diff --git a/src/Storages/S3DataLakeMetaReadHelper.h b/src/Storages/S3DataLakeMetadataReadHelper.h similarity index 79% rename from src/Storages/S3DataLakeMetaReadHelper.h rename to src/Storages/S3DataLakeMetadataReadHelper.h index 29a00e47e10..2187ffa3eda 100644 --- a/src/Storages/S3DataLakeMetaReadHelper.h +++ b/src/Storages/S3DataLakeMetadataReadHelper.h @@ -4,19 +4,14 @@ #if USE_AWS_S3 -# include -# include # include -# include - - class ReadBuffer; namespace DB { -struct S3DataLakeMetaReadHelper +struct S3DataLakeMetadataReadHelper { static std::shared_ptr createReadBuffer(const String & key, ContextPtr context, const StorageS3::Configuration & base_configuration); diff --git a/src/Storages/StorageDeltaLake.cpp b/src/Storages/StorageDeltaLake.cpp index 5bca8dc2138..6b64ac76c15 100644 --- a/src/Storages/StorageDeltaLake.cpp +++ b/src/Storages/StorageDeltaLake.cpp @@ -57,14 +57,14 @@ std::vector DeltaLakeMetadata::listCurrentFiles() && } template -DeltaLakeMetaParser::DeltaLakeMetaParser(const Configuration & configuration_, ContextPtr context) +DeltaLakeMetadataParser::DeltaLakeMetadataParser(const Configuration & configuration_, ContextPtr context) : base_configuration(configuration_) { init(context); } template -void DeltaLakeMetaParser::init(ContextPtr context) +void DeltaLakeMetadataParser::init(ContextPtr context) { auto keys = getJsonLogFiles(); @@ -96,7 +96,7 @@ void DeltaLakeMetaParser::init(ContextPtr context } template -std::vector DeltaLakeMetaParser::getJsonLogFiles() const +std::vector DeltaLakeMetadataParser::getJsonLogFiles() const { /// DeltaLake format stores all metadata json files in _delta_log directory @@ -107,7 +107,7 @@ std::vector DeltaLakeMetaParser::getJsonL } template -void DeltaLakeMetaParser::handleJSON(const JSON & json) +void DeltaLakeMetadataParser::handleJSON(const JSON & json) { if (json.has("add")) { @@ -131,20 +131,20 @@ void DeltaLakeMetaParser::handleJSON(const JSON & // generateQueryFromKeys constructs query from parts filenames for // underlying StorageS3 engine template -String DeltaLakeMetaParser::generateQueryFromKeys(const std::vector & keys, const String &) +String DeltaLakeMetadataParser::generateQueryFromKeys(const std::vector & keys, const String &) { std::string new_query = fmt::format("{{{}}}", fmt::join(keys, ",")); return new_query; } -template DeltaLakeMetaParser::DeltaLakeMetaParser( +template DeltaLakeMetadataParser::DeltaLakeMetadataParser( const StorageS3::Configuration & configuration_, ContextPtr context); -template std::vector DeltaLakeMetaParser::getFiles(); -template String DeltaLakeMetaParser::generateQueryFromKeys( +template std::vector DeltaLakeMetadataParser::getFiles(); +template String DeltaLakeMetadataParser::generateQueryFromKeys( const std::vector & keys, const String & format); -template void DeltaLakeMetaParser::init(ContextPtr context); -template std::vector DeltaLakeMetaParser::getJsonLogFiles() const; -template void DeltaLakeMetaParser::handleJSON(const JSON & json); +template void DeltaLakeMetadataParser::init(ContextPtr context); +template std::vector DeltaLakeMetadataParser::getJsonLogFiles() const; +template void DeltaLakeMetadataParser::handleJSON(const JSON & json); void registerStorageDeltaLake(StorageFactory & factory) { diff --git a/src/Storages/StorageDeltaLake.h b/src/Storages/StorageDeltaLake.h index e14725dd165..288f26cf2fa 100644 --- a/src/Storages/StorageDeltaLake.h +++ b/src/Storages/StorageDeltaLake.h @@ -5,7 +5,7 @@ #if USE_AWS_S3 # include -# include +# include # include # include @@ -30,10 +30,10 @@ private: // class to get deltalake log json files and read json from them template -class DeltaLakeMetaParser +class DeltaLakeMetadataParser { public: - DeltaLakeMetaParser(const Configuration & configuration_, ContextPtr context); + DeltaLakeMetadataParser(const Configuration & configuration_, ContextPtr context); std::vector getFiles() { return std::move(metadata).listCurrentFiles(); } @@ -56,7 +56,7 @@ struct StorageDeltaLakeName static constexpr auto data_directory_prefix = ""; }; -using StorageDeltaLake = IStorageDataLake>; +using StorageDeltaLake = IStorageDataLake>; } #endif diff --git a/src/Storages/StorageHudi.cpp b/src/Storages/StorageHudi.cpp index 62b0edb3c9a..ba92ca192dd 100644 --- a/src/Storages/StorageHudi.cpp +++ b/src/Storages/StorageHudi.cpp @@ -27,7 +27,7 @@ namespace ErrorCodes } template -HudiMetaParser::HudiMetaParser(const Configuration & configuration_, ContextPtr context_) +HudiMetadataParser::HudiMetadataParser(const Configuration & configuration_, ContextPtr context_) : configuration(configuration_), context(context_), log(&Poco::Logger::get("StorageHudi")) { } @@ -38,7 +38,7 @@ HudiMetaParser::HudiMetaParser(const Configuratio /// To find needed parts we need to find out latest part file for every partition. /// Part format is usually parquet, but can differ. template -String HudiMetaParser::generateQueryFromKeys(const std::vector & keys, const String & format) +String HudiMetadataParser::generateQueryFromKeys(const std::vector & keys, const String & format) { /// For each partition path take only latest file. struct FileInfo @@ -90,15 +90,15 @@ String HudiMetaParser::generateQueryFromKeys(cons } template -std::vector HudiMetaParser::getFiles() const +std::vector HudiMetadataParser::getFiles() const { return MetaReadHelper::listFiles(configuration); } -template HudiMetaParser::HudiMetaParser( +template HudiMetadataParser::HudiMetadataParser( const StorageS3::Configuration & configuration_, ContextPtr context_); -template std::vector HudiMetaParser::getFiles() const; -template String HudiMetaParser::generateQueryFromKeys( +template std::vector HudiMetadataParser::getFiles() const; +template String HudiMetadataParser::generateQueryFromKeys( const std::vector & keys, const String & format); void registerStorageHudi(StorageFactory & factory) diff --git a/src/Storages/StorageHudi.h b/src/Storages/StorageHudi.h index 88e4956bca1..fc13345e8c4 100644 --- a/src/Storages/StorageHudi.h +++ b/src/Storages/StorageHudi.h @@ -6,7 +6,7 @@ # include # include -# include +# include # include namespace DB @@ -14,10 +14,10 @@ namespace DB template -class HudiMetaParser +class HudiMetadataParser { public: - HudiMetaParser(const Configuration & configuration_, ContextPtr context_); + HudiMetadataParser(const Configuration & configuration_, ContextPtr context_); std::vector getFiles() const; @@ -35,7 +35,7 @@ struct StorageHudiName static constexpr auto data_directory_prefix = ""; }; -using StorageHudi = IStorageDataLake>; +using StorageHudi = IStorageDataLake>; } #endif diff --git a/src/Storages/StorageIceberg.cpp b/src/Storages/StorageIceberg.cpp index b9e23cb9fe6..b97fd85a0b8 100644 --- a/src/Storages/StorageIceberg.cpp +++ b/src/Storages/StorageIceberg.cpp @@ -46,14 +46,14 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; } -template -IcebergMetaParser::IcebergMetaParser(const Configuration & configuration_, ContextPtr context_) +template +IcebergMetadataParser::IcebergMetadataParser(const Configuration & configuration_, ContextPtr context_) : base_configuration(configuration_), context(context_) { } -template -std::vector IcebergMetaParser::getFiles() const +template +std::vector IcebergMetadataParser::getFiles() const { auto metadata = getNewestMetaFile(); auto manifest_list = getManiFestList(metadata); @@ -68,14 +68,14 @@ std::vector IcebergMetaParser::getFiles() return getFilesForRead(manifest_files); } -template -String IcebergMetaParser::getNewestMetaFile() const +template +String IcebergMetadataParser::getNewestMetaFile() const { /// Iceberg stores all the metadata.json in metadata directory, and the /// newest version has the max version name, so we should list all of them /// then find the newest metadata. static constexpr auto meta_file_suffix = ".json"; - auto metadata_files = MetaReadHelper::listFilesMatchSuffix(base_configuration, metadata_directory, meta_file_suffix); + auto metadata_files = MetadataReadHelper::listFilesMatchSuffix(base_configuration, metadata_directory, meta_file_suffix); if (metadata_files.empty()) throw Exception( @@ -85,10 +85,10 @@ String IcebergMetaParser::getNewestMetaFile() con return *it; } -template -String IcebergMetaParser::getManiFestList(const String & metadata_name) const +template +String IcebergMetadataParser::getManiFestList(const String & metadata_name) const { - auto buffer = MetaReadHelper::createReadBuffer(metadata_name, context, base_configuration); + auto buffer = MetadataReadHelper::createReadBuffer(metadata_name, context, base_configuration); String json_str; readJSONObjectPossiblyInvalid(json_str, *buffer); @@ -132,10 +132,10 @@ parseAvro(const std::unique_ptr & file_reader, const D return columns; } -template -std::vector IcebergMetaParser::getManifestFiles(const String & manifest_list) const +template +std::vector IcebergMetadataParser::getManifestFiles(const String & manifest_list) const { - auto buffer = MetaReadHelper::createReadBuffer(manifest_list, context, base_configuration); + auto buffer = MetadataReadHelper::createReadBuffer(manifest_list, context, base_configuration); auto file_reader = std::make_unique(std::make_unique(*buffer)); @@ -169,13 +169,13 @@ std::vector IcebergMetaParser::getManifes col->getFamilyName()); } -template -std::vector IcebergMetaParser::getFilesForRead(const std::vector & manifest_files) const +template +std::vector IcebergMetadataParser::getFilesForRead(const std::vector & manifest_files) const { std::vector keys; for (const auto & manifest_file : manifest_files) { - auto buffer = MetaReadHelper::createReadBuffer(manifest_file, context, base_configuration); + auto buffer = MetadataReadHelper::createReadBuffer(manifest_file, context, base_configuration); auto file_reader = std::make_unique(std::make_unique(*buffer)); @@ -226,24 +226,24 @@ std::vector IcebergMetaParser::getFilesFo // generateQueryFromKeys constructs query from all parquet filenames // for underlying StorageS3 engine -template -String IcebergMetaParser::generateQueryFromKeys(const std::vector & keys, const String &) +template +String IcebergMetadataParser::generateQueryFromKeys(const std::vector & keys, const String &) { std::string new_query = fmt::format("{{{}}}", fmt::join(keys, ",")); return new_query; } -template IcebergMetaParser::IcebergMetaParser( +template IcebergMetadataParser::IcebergMetadataParser( const StorageS3::Configuration & configuration_, ContextPtr context_); -template std::vector IcebergMetaParser::getFiles() const; -template String IcebergMetaParser::generateQueryFromKeys( +template std::vector IcebergMetadataParser::getFiles() const; +template String IcebergMetadataParser::generateQueryFromKeys( const std::vector & keys, const String & format); -template String IcebergMetaParser::getNewestMetaFile() const; -template String IcebergMetaParser::getManiFestList(const String & metadata_name) const; +template String IcebergMetadataParser::getNewestMetaFile() const; +template String IcebergMetadataParser::getManiFestList(const String & metadata_name) const; template std::vector -IcebergMetaParser::getManifestFiles(const String & manifest_list) const; +IcebergMetadataParser::getManifestFiles(const String & manifest_list) const; template std::vector -IcebergMetaParser::getFilesForRead(const std::vector & manifest_files) const; +IcebergMetadataParser::getFilesForRead(const std::vector & manifest_files) const; void registerStorageIceberg(StorageFactory & factory) { diff --git a/src/Storages/StorageIceberg.h b/src/Storages/StorageIceberg.h index a32858d1375..84670511625 100644 --- a/src/Storages/StorageIceberg.h +++ b/src/Storages/StorageIceberg.h @@ -5,7 +5,7 @@ #if USE_AWS_S3 # include -# include +# include namespace DB { @@ -16,11 +16,11 @@ namespace DB // data/ // metadata/ // The metadata has three layers: metadata -> manifest list -> manifest files -template -class IcebergMetaParser +template +class IcebergMetadataParser { public: - IcebergMetaParser(const Configuration & configuration_, ContextPtr context_); + IcebergMetadataParser(const Configuration & configuration_, ContextPtr context_); std::vector getFiles() const; @@ -45,7 +45,7 @@ struct StorageIcebergName static constexpr auto data_directory_prefix = "data"; }; -using StorageIceberg = IStorageDataLake>; +using StorageIceberg = IStorageDataLake>; } #endif diff --git a/src/Storages/StorageS3.h b/src/Storages/StorageS3.h index 8194feaf754..633d9671ba1 100644 --- a/src/Storages/StorageS3.h +++ b/src/Storages/StorageS3.h @@ -35,22 +35,22 @@ class PullingPipelineExecutor; class StorageS3SequentialSource; class NamedCollection; -template +template class IStorageDataLake; -struct S3DataLakeMetaReadHelper; +struct S3DataLakeMetadataReadHelper; struct StorageIcebergName; -template -class IcebergMetaParser; +template +class IcebergMetadataParser; struct StorageDeltaLakeName; -template -class DeltaLakeMetaParser; +template +class DeltaLakeMetadataParser; struct StorageHudiName; -template -class HudiMetaParser; +template +class HudiMetadataParser; class StorageS3Source : public ISource, WithContext { @@ -320,9 +320,9 @@ public: private: friend class StorageS3Cluster; friend class TableFunctionS3Cluster; - friend class IStorageDataLake>; - friend class IStorageDataLake>; - friend class IStorageDataLake>; + friend class IStorageDataLake>; + friend class IStorageDataLake>; + friend class IStorageDataLake>; Configuration s3_configuration; std::vector keys; diff --git a/src/TableFunctions/TableFunctionS3.cpp b/src/TableFunctions/TableFunctionS3.cpp index 38ee28ec9ce..4e4594f8480 100644 --- a/src/TableFunctions/TableFunctionS3.cpp +++ b/src/TableFunctions/TableFunctionS3.cpp @@ -29,9 +29,8 @@ namespace ErrorCodes /// This is needed to avoid copy-pase. Because s3Cluster arguments only differ in additional argument (first) - cluster name -template void TableFunctionS3::parseArgumentsImpl( - const String & error_message, ASTs & args, ContextPtr context, StorageS3::Configuration & s3_configuration) + const String & error_message, ASTs & args, ContextPtr context, StorageS3::Configuration & s3_configuration, bool get_format_from_file) { if (auto named_collection = tryGetNamedCollectionWithOverrides(args)) { @@ -112,9 +111,6 @@ void TableFunctionS3::parseArgumentsImpl( s3_configuration.format = FormatFactory::instance().getFormatFromFileName(s3_configuration.url.uri.getPath(), true); } -template void TableFunctionS3::parseArgumentsImpl( - const String & error_message, ASTs & args, ContextPtr context, StorageS3::Configuration & s3_configuration); - void TableFunctionS3::parseArguments(const ASTPtr & ast_function, ContextPtr context) { /// Parse args diff --git a/src/TableFunctions/TableFunctionS3.h b/src/TableFunctions/TableFunctionS3.h index 5f197a6e058..a0e09a2fd93 100644 --- a/src/TableFunctions/TableFunctionS3.h +++ b/src/TableFunctions/TableFunctionS3.h @@ -35,9 +35,12 @@ public: { return {"_path", "_file"}; } - - template - static void parseArgumentsImpl(const String & error_message, ASTs & args, ContextPtr context, StorageS3::Configuration & configuration); + void parseArgumentsImpl( + const String & error_message, + ASTs & args, + ContextPtr context, + StorageS3::Configuration & configuration, + bool get_format_from_file = true); protected: From e4e3adf1605ced2e00f28b21fd2cdfdf3225967e Mon Sep 17 00:00:00 2001 From: flynn Date: Wed, 15 Feb 2023 05:05:46 +0000 Subject: [PATCH 229/717] fix --- src/Storages/IStorageDataLake.h | 11 ----------- src/Storages/StorageDeltaLake.cpp | 5 ----- src/Storages/StorageHudi.cpp | 1 - src/Storages/StorageIceberg.cpp | 8 -------- src/TableFunctions/ITableFunctionDataLake.h | 2 +- src/TableFunctions/TableFunctionS3.h | 2 +- 6 files changed, 2 insertions(+), 27 deletions(-) diff --git a/src/Storages/IStorageDataLake.h b/src/Storages/IStorageDataLake.h index 687c50b4082..0216b3c607f 100644 --- a/src/Storages/IStorageDataLake.h +++ b/src/Storages/IStorageDataLake.h @@ -6,24 +6,13 @@ # include -# include -# include -# include - -# include -# include -# include -# include # include # include # include -# include # include # include -# include - # include # include # include diff --git a/src/Storages/StorageDeltaLake.cpp b/src/Storages/StorageDeltaLake.cpp index 6b64ac76c15..03a1cf12ae9 100644 --- a/src/Storages/StorageDeltaLake.cpp +++ b/src/Storages/StorageDeltaLake.cpp @@ -7,12 +7,9 @@ #include #include #include -#include #include -#include #include -#include #include @@ -21,8 +18,6 @@ #include #include -#include -#include namespace DB { diff --git a/src/Storages/StorageHudi.cpp b/src/Storages/StorageHudi.cpp index ba92ca192dd..b9744eedaa0 100644 --- a/src/Storages/StorageHudi.cpp +++ b/src/Storages/StorageHudi.cpp @@ -6,7 +6,6 @@ #include #include -#include #include #include #include diff --git a/src/Storages/StorageIceberg.cpp b/src/Storages/StorageIceberg.cpp index b97fd85a0b8..0f2dd44dc11 100644 --- a/src/Storages/StorageIceberg.cpp +++ b/src/Storages/StorageIceberg.cpp @@ -1,7 +1,6 @@ #include "config.h" #if USE_AWS_S3 -# include # include # include @@ -9,12 +8,9 @@ # include # include -# include # include # include -# include -# include # include # include @@ -24,11 +20,7 @@ # include # include -# include - # include -# include -# include # include diff --git a/src/TableFunctions/ITableFunctionDataLake.h b/src/TableFunctions/ITableFunctionDataLake.h index 0ce8cd1f8df..53f94aa30af 100644 --- a/src/TableFunctions/ITableFunctionDataLake.h +++ b/src/TableFunctions/ITableFunctionDataLake.h @@ -87,7 +87,7 @@ protected: auto & args = args_func.at(0)->children; - TableFunctionS3::parseArgumentsImpl(message, args, context, configuration); + TableFunctionS3::parseArgumentsImpl(message, args, context, configuration, false); if (configuration.format == "auto") configuration.format = "Parquet"; diff --git a/src/TableFunctions/TableFunctionS3.h b/src/TableFunctions/TableFunctionS3.h index a0e09a2fd93..29dea5d3631 100644 --- a/src/TableFunctions/TableFunctionS3.h +++ b/src/TableFunctions/TableFunctionS3.h @@ -35,7 +35,7 @@ public: { return {"_path", "_file"}; } - void parseArgumentsImpl( + static void parseArgumentsImpl( const String & error_message, ASTs & args, ContextPtr context, From 1468e9b9d9fe832e88be5065a3b02b2043f2c5a0 Mon Sep 17 00:00:00 2001 From: flynn Date: Wed, 15 Feb 2023 06:37:05 +0000 Subject: [PATCH 230/717] fix --- src/Storages/IStorageDataLake.h | 10 ---------- src/Storages/StorageDeltaLake.cpp | 24 ++++++++++++------------ src/Storages/StorageDeltaLake.h | 2 +- src/Storages/StorageHudi.cpp | 14 +++++++------- src/Storages/StorageHudi.h | 2 +- 5 files changed, 21 insertions(+), 31 deletions(-) diff --git a/src/Storages/IStorageDataLake.h b/src/Storages/IStorageDataLake.h index 0216b3c607f..f5f00a0b8ed 100644 --- a/src/Storages/IStorageDataLake.h +++ b/src/Storages/IStorageDataLake.h @@ -13,23 +13,13 @@ # include # include -# include -# include -# include - # include # include # include -# include # include -namespace Aws::S3 -{ -class S3Client; -} - namespace DB { diff --git a/src/Storages/StorageDeltaLake.cpp b/src/Storages/StorageDeltaLake.cpp index 03a1cf12ae9..02c6a8b9098 100644 --- a/src/Storages/StorageDeltaLake.cpp +++ b/src/Storages/StorageDeltaLake.cpp @@ -51,22 +51,22 @@ std::vector DeltaLakeMetadata::listCurrentFiles() && return keys; } -template -DeltaLakeMetadataParser::DeltaLakeMetadataParser(const Configuration & configuration_, ContextPtr context) +template +DeltaLakeMetadataParser::DeltaLakeMetadataParser(const Configuration & configuration_, ContextPtr context) : base_configuration(configuration_) { init(context); } -template -void DeltaLakeMetadataParser::init(ContextPtr context) +template +void DeltaLakeMetadataParser::init(ContextPtr context) { auto keys = getJsonLogFiles(); // read data from every json log file for (const String & key : keys) { - auto buf = MetaReadHelper::createReadBuffer(key, context, base_configuration); + auto buf = MetadataReadHelper::createReadBuffer(key, context, base_configuration); char c; while (!buf->eof()) @@ -90,19 +90,19 @@ void DeltaLakeMetadataParser::init(ContextPtr con } } -template -std::vector DeltaLakeMetadataParser::getJsonLogFiles() const +template +std::vector DeltaLakeMetadataParser::getJsonLogFiles() const { /// DeltaLake format stores all metadata json files in _delta_log directory static constexpr auto deltalake_metadata_directory = "_delta_log"; static constexpr auto meta_file_suffix = ".json"; - return MetaReadHelper::listFilesMatchSuffix(base_configuration, deltalake_metadata_directory, meta_file_suffix); + return MetadataReadHelper::listFilesMatchSuffix(base_configuration, deltalake_metadata_directory, meta_file_suffix); } -template -void DeltaLakeMetadataParser::handleJSON(const JSON & json) +template +void DeltaLakeMetadataParser::handleJSON(const JSON & json) { if (json.has("add")) { @@ -125,8 +125,8 @@ void DeltaLakeMetadataParser::handleJSON(const JS // keys is vector of parts with latest version // generateQueryFromKeys constructs query from parts filenames for // underlying StorageS3 engine -template -String DeltaLakeMetadataParser::generateQueryFromKeys(const std::vector & keys, const String &) +template +String DeltaLakeMetadataParser::generateQueryFromKeys(const std::vector & keys, const String &) { std::string new_query = fmt::format("{{{}}}", fmt::join(keys, ",")); return new_query; diff --git a/src/Storages/StorageDeltaLake.h b/src/Storages/StorageDeltaLake.h index 288f26cf2fa..bae473e96bc 100644 --- a/src/Storages/StorageDeltaLake.h +++ b/src/Storages/StorageDeltaLake.h @@ -29,7 +29,7 @@ private: }; // class to get deltalake log json files and read json from them -template +template class DeltaLakeMetadataParser { public: diff --git a/src/Storages/StorageHudi.cpp b/src/Storages/StorageHudi.cpp index b9744eedaa0..2f7cd259d59 100644 --- a/src/Storages/StorageHudi.cpp +++ b/src/Storages/StorageHudi.cpp @@ -25,8 +25,8 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -template -HudiMetadataParser::HudiMetadataParser(const Configuration & configuration_, ContextPtr context_) +template +HudiMetadataParser::HudiMetadataParser(const Configuration & configuration_, ContextPtr context_) : configuration(configuration_), context(context_), log(&Poco::Logger::get("StorageHudi")) { } @@ -36,8 +36,8 @@ HudiMetadataParser::HudiMetadataParser(const Conf /// Every partition(directory) in Apache Hudi has different versions of part. /// To find needed parts we need to find out latest part file for every partition. /// Part format is usually parquet, but can differ. -template -String HudiMetadataParser::generateQueryFromKeys(const std::vector & keys, const String & format) +template +String HudiMetadataParser::generateQueryFromKeys(const std::vector & keys, const String & format) { /// For each partition path take only latest file. struct FileInfo @@ -88,10 +88,10 @@ String HudiMetadataParser::generateQueryFromKeys( return "{" + list_of_keys + "}"; } -template -std::vector HudiMetadataParser::getFiles() const +template +std::vector HudiMetadataParser::getFiles() const { - return MetaReadHelper::listFiles(configuration); + return MetadataReadHelper::listFiles(configuration); } template HudiMetadataParser::HudiMetadataParser( diff --git a/src/Storages/StorageHudi.h b/src/Storages/StorageHudi.h index fc13345e8c4..fd7e97919fb 100644 --- a/src/Storages/StorageHudi.h +++ b/src/Storages/StorageHudi.h @@ -13,7 +13,7 @@ namespace DB { -template +template class HudiMetadataParser { public: From 50c0693afdaccb037c76fba743bf33fb2d78e1ac Mon Sep 17 00:00:00 2001 From: flynn Date: Wed, 15 Feb 2023 06:49:53 +0000 Subject: [PATCH 231/717] fix --- src/Storages/StorageIceberg.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Storages/StorageIceberg.cpp b/src/Storages/StorageIceberg.cpp index 0f2dd44dc11..e6444949c91 100644 --- a/src/Storages/StorageIceberg.cpp +++ b/src/Storages/StorageIceberg.cpp @@ -157,7 +157,7 @@ std::vector IcebergMetadataParser::ge } throw Exception( ErrorCodes::ILLEGAL_COLUMN, - "The parsed column from Avro file for manifest_path should have data type String, but get {}", + "The parsed column from Avro file of `manifest_path` field should be String type, got {}", col->getFamilyName()); } @@ -200,7 +200,7 @@ std::vector IcebergMetadataParser::ge { throw Exception( ErrorCodes::ILLEGAL_COLUMN, - "The parsed column from Avro file for file_path should have data type String, got {}", + "The parsed column from Avro file of `file_path` field should be String type, got {}", col_str->getFamilyName()); } } @@ -208,7 +208,7 @@ std::vector IcebergMetadataParser::ge { throw Exception( ErrorCodes::ILLEGAL_COLUMN, - "The parsed column from Avro file for data_file field should have data type Tuple, got {}", + "The parsed column from Avro file of `data_file` field should be Tuple type, got {}", col->getFamilyName()); } } From 18cf72147ec4c6e56fc40f479c30968874ab18e7 Mon Sep 17 00:00:00 2001 From: flynn Date: Wed, 15 Feb 2023 06:54:16 +0000 Subject: [PATCH 232/717] remove more redundant header files --- src/Storages/StorageDeltaLake.cpp | 12 +++++------- src/Storages/StorageHudi.cpp | 6 ++---- src/Storages/StorageIceberg.cpp | 14 ++++++-------- src/Storages/StorageS3.h | 1 - src/TableFunctions/ITableFunctionDataLake.h | 7 ------- 5 files changed, 13 insertions(+), 27 deletions(-) diff --git a/src/Storages/StorageDeltaLake.cpp b/src/Storages/StorageDeltaLake.cpp index 02c6a8b9098..6b09a29f628 100644 --- a/src/Storages/StorageDeltaLake.cpp +++ b/src/Storages/StorageDeltaLake.cpp @@ -4,17 +4,10 @@ #include #include -#include -#include -#include -#include - #include #include -#include - #include #include @@ -134,11 +127,16 @@ String DeltaLakeMetadataParser::generateQuery template DeltaLakeMetadataParser::DeltaLakeMetadataParser( const StorageS3::Configuration & configuration_, ContextPtr context); + template std::vector DeltaLakeMetadataParser::getFiles(); + template String DeltaLakeMetadataParser::generateQueryFromKeys( const std::vector & keys, const String & format); + template void DeltaLakeMetadataParser::init(ContextPtr context); + template std::vector DeltaLakeMetadataParser::getJsonLogFiles() const; + template void DeltaLakeMetadataParser::handleJSON(const JSON & json); void registerStorageDeltaLake(StorageFactory & factory) diff --git a/src/Storages/StorageHudi.cpp b/src/Storages/StorageHudi.cpp index 2f7cd259d59..48a92f5acd6 100644 --- a/src/Storages/StorageHudi.cpp +++ b/src/Storages/StorageHudi.cpp @@ -6,11 +6,7 @@ #include #include -#include -#include #include -#include -#include #include @@ -96,7 +92,9 @@ std::vector HudiMetadataParser:: template HudiMetadataParser::HudiMetadataParser( const StorageS3::Configuration & configuration_, ContextPtr context_); + template std::vector HudiMetadataParser::getFiles() const; + template String HudiMetadataParser::generateQueryFromKeys( const std::vector & keys, const String & format); diff --git a/src/Storages/StorageIceberg.cpp b/src/Storages/StorageIceberg.cpp index e6444949c91..b25d2448a43 100644 --- a/src/Storages/StorageIceberg.cpp +++ b/src/Storages/StorageIceberg.cpp @@ -8,18 +8,10 @@ # include # include -# include -# include - # include -# include # include -# include -# include -# include - # include # include @@ -227,13 +219,19 @@ String IcebergMetadataParser::generateQueryFr template IcebergMetadataParser::IcebergMetadataParser( const StorageS3::Configuration & configuration_, ContextPtr context_); + template std::vector IcebergMetadataParser::getFiles() const; + template String IcebergMetadataParser::generateQueryFromKeys( const std::vector & keys, const String & format); + template String IcebergMetadataParser::getNewestMetaFile() const; + template String IcebergMetadataParser::getManiFestList(const String & metadata_name) const; + template std::vector IcebergMetadataParser::getManifestFiles(const String & manifest_list) const; + template std::vector IcebergMetadataParser::getFilesForRead(const std::vector & manifest_files) const; diff --git a/src/Storages/StorageS3.h b/src/Storages/StorageS3.h index 633d9671ba1..3bacfdca1b2 100644 --- a/src/Storages/StorageS3.h +++ b/src/Storages/StorageS3.h @@ -17,7 +17,6 @@ #include #include #include -#include #include #include #include diff --git a/src/TableFunctions/ITableFunctionDataLake.h b/src/TableFunctions/ITableFunctionDataLake.h index 53f94aa30af..7948d72e0c5 100644 --- a/src/TableFunctions/ITableFunctionDataLake.h +++ b/src/TableFunctions/ITableFunctionDataLake.h @@ -5,17 +5,10 @@ #if USE_AWS_S3 # include -# include # include # include -# include # include -# include # include -# include -# include -# include -# include # include namespace DB From 91298b3617627d3d7b504fb733b723bfb35e74a7 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 15 Feb 2023 09:28:32 +0000 Subject: [PATCH 233/717] Remove POCO_WIN32_UTF8 --- .../Foundation/include/Poco/EventLogChannel.h | 4 - .../poco/Foundation/include/Poco/Foundation.h | 3 - .../Foundation/include/Poco/Platform_WIN32.h | 2 - base/poco/Foundation/src/Debugger.cpp | 3 - base/poco/Foundation/src/DirectoryWatcher.cpp | 6 -- base/poco/Foundation/src/EventLogChannel.cpp | 75 ------------------- base/poco/Foundation/src/FileStream_WIN32.cpp | 9 --- .../Foundation/src/SharedMemory_WIN32.cpp | 19 ----- base/poco/Foundation/src/Timezone_WIN32.cpp | 12 --- .../Foundation/src/WindowsConsoleChannel.cpp | 33 -------- .../poco/Util/include/Poco/Util/Application.h | 12 --- .../include/Poco/Util/ServerApplication.h | 8 -- base/poco/Util/include/Poco/Util/WinService.h | 4 - base/poco/Util/src/Application.cpp | 16 ---- base/poco/Util/src/ServerApplication.cpp | 3 - 15 files changed, 209 deletions(-) diff --git a/base/poco/Foundation/include/Poco/EventLogChannel.h b/base/poco/Foundation/include/Poco/EventLogChannel.h index e0b2089887c..e6a79088204 100644 --- a/base/poco/Foundation/include/Poco/EventLogChannel.h +++ b/base/poco/Foundation/include/Poco/EventLogChannel.h @@ -86,11 +86,7 @@ protected: static int getType(const Message & msg); static int getCategory(const Message & msg); void setUpRegistry() const; -#if defined(POCO_WIN32_UTF8) - static std::wstring findLibrary(const wchar_t * name); -#else static std::string findLibrary(const char * name); -#endif private: std::string _name; diff --git a/base/poco/Foundation/include/Poco/Foundation.h b/base/poco/Foundation/include/Poco/Foundation.h index 4259929b909..747cff7818c 100644 --- a/base/poco/Foundation/include/Poco/Foundation.h +++ b/base/poco/Foundation/include/Poco/Foundation.h @@ -79,9 +79,6 @@ // // Cleanup inconsistencies // -# ifdef POCO_WIN32_UTF8 -# undef POCO_WIN32_UTF8 -# endif // diff --git a/base/poco/Foundation/include/Poco/Platform_WIN32.h b/base/poco/Foundation/include/Poco/Platform_WIN32.h index 0d5ec7ec65d..8f77b171dac 100644 --- a/base/poco/Foundation/include/Poco/Platform_WIN32.h +++ b/base/poco/Foundation/include/Poco/Platform_WIN32.h @@ -149,9 +149,7 @@ #endif -#if !defined(POCO_WIN32_UTF8) # pragma message("Compiling POCO on Windows without #define POCO_WIN32_UTF8 is deprecated.") -#endif // Turn off some annoying warnings diff --git a/base/poco/Foundation/src/Debugger.cpp b/base/poco/Foundation/src/Debugger.cpp index 6e388398405..3a9f7033fcb 100644 --- a/base/poco/Foundation/src/Debugger.cpp +++ b/base/poco/Foundation/src/Debugger.cpp @@ -20,9 +20,6 @@ #include #include #endif -#if defined(POCO_WIN32_UTF8) && !defined(POCO_NO_WSTRING) -#include "Poco/UnicodeConverter.h" -#endif // NOTE: In this module, we use the C library functions (fputs) for, diff --git a/base/poco/Foundation/src/DirectoryWatcher.cpp b/base/poco/Foundation/src/DirectoryWatcher.cpp index 68aef99d29f..b559da65e09 100644 --- a/base/poco/Foundation/src/DirectoryWatcher.cpp +++ b/base/poco/Foundation/src/DirectoryWatcher.cpp @@ -179,13 +179,7 @@ public: filter |= FILE_NOTIFY_CHANGE_SIZE | FILE_NOTIFY_CHANGE_LAST_WRITE; std::string path(owner().directory().path()); -#if defined(POCO_WIN32_UTF8) - std::wstring upath; - FileImpl::convertPath(path.c_str(), upath); - HANDLE hChange = FindFirstChangeNotificationW(upath.c_str(), FALSE, filter); -#else HANDLE hChange = FindFirstChangeNotificationA(path.c_str(), FALSE, filter); -#endif if (hChange == INVALID_HANDLE_VALUE) { diff --git a/base/poco/Foundation/src/EventLogChannel.cpp b/base/poco/Foundation/src/EventLogChannel.cpp index 8d7d7cb8274..33a19cef967 100644 --- a/base/poco/Foundation/src/EventLogChannel.cpp +++ b/base/poco/Foundation/src/EventLogChannel.cpp @@ -16,9 +16,6 @@ #include "Poco/Message.h" #include "Poco/String.h" #include "pocomsg.h" -#if defined(POCO_WIN32_UTF8) -#include "Poco/UnicodeConverter.h" -#endif namespace Poco { @@ -35,18 +32,6 @@ EventLogChannel::EventLogChannel(): _h(0) { const DWORD maxPathLen = MAX_PATH + 1; -#if defined(POCO_WIN32_UTF8) - wchar_t name[maxPathLen]; - int n = GetModuleFileNameW(NULL, name, maxPathLen); - if (n > 0) - { - wchar_t* end = name + n - 1; - while (end > name && *end != '\\') --end; - if (*end == '\\') ++end; - std::wstring uname(end); - UnicodeConverter::toUTF8(uname, _name); - } -#else char name[maxPathLen]; int n = GetModuleFileNameA(NULL, name, maxPathLen); if (n > 0) @@ -56,7 +41,6 @@ EventLogChannel::EventLogChannel(): if (*end == '\\') ++end; _name = end; } -#endif } @@ -93,15 +77,7 @@ EventLogChannel::~EventLogChannel() void EventLogChannel::open() { setUpRegistry(); -#if defined(POCO_WIN32_UTF8) - std::wstring uhost; - UnicodeConverter::toUTF16(_host, uhost); - std::wstring uname; - UnicodeConverter::toUTF16(_name, uname); - _h = RegisterEventSourceW(uhost.empty() ? NULL : uhost.c_str(), uname.c_str()); -#else _h = RegisterEventSource(_host.empty() ? NULL : _host.c_str(), _name.c_str()); -#endif if (!_h) throw SystemException("cannot register event source"); } @@ -116,15 +92,8 @@ void EventLogChannel::close() void EventLogChannel::log(const Message& msg) { if (!_h) open(); -#if defined(POCO_WIN32_UTF8) - std::wstring utext; - UnicodeConverter::toUTF16(msg.getText(), utext); - const wchar_t* pMsg = utext.c_str(); - ReportEventW(_h, getType(msg), getCategory(msg), POCO_MSG_LOG, NULL, 1, 0, &pMsg, NULL); -#else const char* pMsg = msg.getText().c_str(); ReportEvent(_h, getType(msg), getCategory(msg), POCO_MSG_LOG, NULL, 1, 0, &pMsg, NULL); -#endif } @@ -209,30 +178,11 @@ void EventLogChannel::setUpRegistry() const key.append(_name); HKEY hKey; DWORD disp; -#if defined(POCO_WIN32_UTF8) - std::wstring ukey; - UnicodeConverter::toUTF16(key, ukey); - DWORD rc = RegCreateKeyExW(HKEY_LOCAL_MACHINE, ukey.c_str(), 0, NULL, REG_OPTION_NON_VOLATILE, KEY_ALL_ACCESS, NULL, &hKey, &disp); -#else DWORD rc = RegCreateKeyEx(HKEY_LOCAL_MACHINE, key.c_str(), 0, NULL, REG_OPTION_NON_VOLATILE, KEY_ALL_ACCESS, NULL, &hKey, &disp); -#endif if (rc != ERROR_SUCCESS) return; if (disp == REG_CREATED_NEW_KEY) { -#if defined(POCO_WIN32_UTF8) - std::wstring path; - #if defined(POCO_DLL) - #if defined(_DEBUG) - path = findLibrary(L"PocoFoundationd.dll"); - #else - path = findLibrary(L"PocoFoundation.dll"); - #endif - #endif - - if (path.empty()) - path = findLibrary(L"PocoMsg.dll"); -#else std::string path; #if defined(POCO_DLL) #if defined(_DEBUG) @@ -244,45 +194,21 @@ void EventLogChannel::setUpRegistry() const if (path.empty()) path = findLibrary("PocoMsg.dll"); -#endif if (!path.empty()) { DWORD count = 8; DWORD types = 7; -#if defined(POCO_WIN32_UTF8) - RegSetValueExW(hKey, L"CategoryMessageFile", 0, REG_SZ, (const BYTE*) path.c_str(), static_cast(sizeof(wchar_t)*(path.size() + 1))); - RegSetValueExW(hKey, L"EventMessageFile", 0, REG_SZ, (const BYTE*) path.c_str(), static_cast(sizeof(wchar_t)*(path.size() + 1))); - RegSetValueExW(hKey, L"CategoryCount", 0, REG_DWORD, (const BYTE*) &count, static_cast(sizeof(count))); - RegSetValueExW(hKey, L"TypesSupported", 0, REG_DWORD, (const BYTE*) &types, static_cast(sizeof(types))); -#else RegSetValueEx(hKey, "CategoryMessageFile", 0, REG_SZ, (const BYTE*) path.c_str(), static_cast(path.size() + 1)); RegSetValueEx(hKey, "EventMessageFile", 0, REG_SZ, (const BYTE*) path.c_str(), static_cast(path.size() + 1)); RegSetValueEx(hKey, "CategoryCount", 0, REG_DWORD, (const BYTE*) &count, static_cast(sizeof(count))); RegSetValueEx(hKey, "TypesSupported", 0, REG_DWORD, (const BYTE*) &types, static_cast(sizeof(types))); -#endif } } RegCloseKey(hKey); } -#if defined(POCO_WIN32_UTF8) -std::wstring EventLogChannel::findLibrary(const wchar_t* name) -{ - std::wstring path; - HMODULE dll = LoadLibraryW(name); - if (dll) - { - const DWORD maxPathLen = MAX_PATH + 1; - wchar_t name[maxPathLen]; - int n = GetModuleFileNameW(dll, name, maxPathLen); - if (n > 0) path = name; - FreeLibrary(dll); - } - return path; -} -#else std::string EventLogChannel::findLibrary(const char* name) { std::string path; @@ -297,7 +223,6 @@ std::string EventLogChannel::findLibrary(const char* name) } return path; } -#endif } // namespace Poco diff --git a/base/poco/Foundation/src/FileStream_WIN32.cpp b/base/poco/Foundation/src/FileStream_WIN32.cpp index 28f53c9d403..0fc52f0ce01 100644 --- a/base/poco/Foundation/src/FileStream_WIN32.cpp +++ b/base/poco/Foundation/src/FileStream_WIN32.cpp @@ -15,9 +15,6 @@ #include "Poco/FileStream.h" #include "Poco/File.h" #include "Poco/Exception.h" -#if defined (POCO_WIN32_UTF8) -#include "Poco/UnicodeConverter.h" -#endif namespace Poco { @@ -64,13 +61,7 @@ void FileStreamBuf::open(const std::string& path, std::ios::openmode mode) DWORD flags = FILE_ATTRIBUTE_NORMAL; -#if defined (POCO_WIN32_UTF8) - std::wstring utf16Path; - FileImpl::convertPath(path, utf16Path); - _handle = CreateFileW(utf16Path.c_str(), access, shareMode, NULL, creationDisp, flags, NULL); -#else _handle = CreateFileA(path.c_str(), access, shareMode, NULL, creationDisp, flags, NULL); -#endif if (_handle == INVALID_HANDLE_VALUE) File::handleLastError(_path); diff --git a/base/poco/Foundation/src/SharedMemory_WIN32.cpp b/base/poco/Foundation/src/SharedMemory_WIN32.cpp index eadf37bd018..a05a32eca4d 100644 --- a/base/poco/Foundation/src/SharedMemory_WIN32.cpp +++ b/base/poco/Foundation/src/SharedMemory_WIN32.cpp @@ -17,9 +17,6 @@ #include "Poco/Exception.h" #include "Poco/File.h" #include "Poco/Format.h" -#if defined (POCO_WIN32_UTF8) -#include "Poco/UnicodeConverter.h" -#endif #include "Poco/UnWindows.h" @@ -37,13 +34,7 @@ SharedMemoryImpl::SharedMemoryImpl(const std::string& name, std::size_t size, Sh if (mode == SharedMemory::AM_WRITE) _mode = PAGE_READWRITE; -#if defined (POCO_WIN32_UTF8) - std::wstring utf16name; - UnicodeConverter::toUTF16(_name, utf16name); - _memHandle = CreateFileMappingW(INVALID_HANDLE_VALUE, NULL, _mode, 0, _size, utf16name.c_str()); -#else _memHandle = CreateFileMappingA(INVALID_HANDLE_VALUE, NULL, _mode, 0, _size, _name.c_str()); -#endif if (!_memHandle) { @@ -51,11 +42,7 @@ SharedMemoryImpl::SharedMemoryImpl(const std::string& name, std::size_t size, Sh if (_mode != PAGE_READONLY || dwRetVal != 5) throw SystemException(format("Cannot create shared memory object %s [Error %d: %s]", _name, static_cast(dwRetVal), Error::getMessage(dwRetVal))); -#if defined (POCO_WIN32_UTF8) - _memHandle = OpenFileMappingW(PAGE_READONLY, FALSE, utf16name.c_str()); -#else _memHandle = OpenFileMappingA(PAGE_READONLY, FALSE, _name.c_str()); -#endif if (!_memHandle) { dwRetVal = GetLastError(); @@ -88,13 +75,7 @@ SharedMemoryImpl::SharedMemoryImpl(const Poco::File& file, SharedMemory::AccessM fileMode |= GENERIC_WRITE; } -#if defined (POCO_WIN32_UTF8) - std::wstring utf16name; - UnicodeConverter::toUTF16(_name, utf16name); - _fileHandle = CreateFileW(utf16name.c_str(), fileMode, shareMode, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); -#else _fileHandle = CreateFileA(_name.c_str(), fileMode, shareMode, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); -#endif if (_fileHandle == INVALID_HANDLE_VALUE) throw OpenFileException("Cannot open memory mapped file", _name); diff --git a/base/poco/Foundation/src/Timezone_WIN32.cpp b/base/poco/Foundation/src/Timezone_WIN32.cpp index 45be02e0a0b..a64f6cfc53c 100644 --- a/base/poco/Foundation/src/Timezone_WIN32.cpp +++ b/base/poco/Foundation/src/Timezone_WIN32.cpp @@ -53,13 +53,9 @@ std::string Timezone::name() TIME_ZONE_INFORMATION tzInfo; DWORD dstFlag = GetTimeZoneInformation(&tzInfo); WCHAR* ptr = dstFlag == TIME_ZONE_ID_DAYLIGHT ? tzInfo.DaylightName : tzInfo.StandardName; -#if defined(POCO_WIN32_UTF8) - UnicodeConverter::toUTF8(ptr, result); -#else char buffer[256]; DWORD rc = WideCharToMultiByte(CP_ACP, 0, ptr, -1, buffer, sizeof(buffer), NULL, NULL); if (rc) result = buffer; -#endif return result; } @@ -70,13 +66,9 @@ std::string Timezone::standardName() TIME_ZONE_INFORMATION tzInfo; DWORD dstFlag = GetTimeZoneInformation(&tzInfo); WCHAR* ptr = tzInfo.StandardName; -#if defined(POCO_WIN32_UTF8) - UnicodeConverter::toUTF8(ptr, result); -#else char buffer[256]; DWORD rc = WideCharToMultiByte(CP_ACP, 0, ptr, -1, buffer, sizeof(buffer), NULL, NULL); if (rc) result = buffer; -#endif return result; } @@ -87,13 +79,9 @@ std::string Timezone::dstName() TIME_ZONE_INFORMATION tzInfo; DWORD dstFlag = GetTimeZoneInformation(&tzInfo); WCHAR* ptr = tzInfo.DaylightName; -#if defined(POCO_WIN32_UTF8) - UnicodeConverter::toUTF8(ptr, result); -#else char buffer[256]; DWORD rc = WideCharToMultiByte(CP_ACP, 0, ptr, -1, buffer, sizeof(buffer), NULL, NULL); if (rc) result = buffer; -#endif return result; } diff --git a/base/poco/Foundation/src/WindowsConsoleChannel.cpp b/base/poco/Foundation/src/WindowsConsoleChannel.cpp index 07e352935f2..48665bb178a 100644 --- a/base/poco/Foundation/src/WindowsConsoleChannel.cpp +++ b/base/poco/Foundation/src/WindowsConsoleChannel.cpp @@ -14,9 +14,6 @@ #include "Poco/WindowsConsoleChannel.h" #include "Poco/Message.h" -#if defined(POCO_WIN32_UTF8) -#include "Poco/UnicodeConverter.h" -#endif #include "Poco/String.h" #include "Poco/Exception.h" @@ -45,23 +42,8 @@ void WindowsConsoleChannel::log(const Message& msg) std::string text = msg.getText(); text += "\r\n"; -#if defined(POCO_WIN32_UTF8) - if (_isFile) - { - DWORD written; - WriteFile(_hConsole, text.data(), static_cast(text.size()), &written, NULL); - } - else - { - std::wstring utext; - UnicodeConverter::toUTF16(text, utext); - DWORD written; - WriteConsoleW(_hConsole, utext.data(), static_cast(utext.size()), &written, NULL); - } -#else DWORD written; WriteFile(_hConsole, text.data(), text.size(), &written, NULL); -#endif } @@ -96,23 +78,8 @@ void WindowsColorConsoleChannel::log(const Message& msg) SetConsoleTextAttribute(_hConsole, attr); } -#if defined(POCO_WIN32_UTF8) - if (_isFile) - { - DWORD written; - WriteFile(_hConsole, text.data(), static_cast(text.size()), &written, NULL); - } - else - { - std::wstring utext; - UnicodeConverter::toUTF16(text, utext); - DWORD written; - WriteConsoleW(_hConsole, utext.data(), static_cast(utext.size()), &written, NULL); - } -#else DWORD written; WriteFile(_hConsole, text.data(), text.size(), &written, NULL); -#endif if (_enableColors && !_isFile) { diff --git a/base/poco/Util/include/Poco/Util/Application.h b/base/poco/Util/include/Poco/Util/Application.h index 7743ad1126c..c8d18e1bce9 100644 --- a/base/poco/Util/include/Poco/Util/Application.h +++ b/base/poco/Util/include/Poco/Util/Application.h @@ -138,18 +138,6 @@ namespace Util /// Note that as of release 1.3.7, init() no longer /// calls initialize(). This is now called from run(). -#if defined(POCO_WIN32_UTF8) && !defined(POCO_NO_WSTRING) - void init(int argc, wchar_t * argv[]); - /// Processes the application's command line arguments - /// and sets the application's properties (e.g., - /// "application.path", "application.name", etc.). - /// - /// Note that as of release 1.3.7, init() no longer - /// calls initialize(). This is now called from run(). - /// - /// This Windows-specific version of init is used for passing - /// Unicode command line arguments from wmain(). -#endif void init(const ArgVec & args); /// Processes the application's command line arguments diff --git a/base/poco/Util/include/Poco/Util/ServerApplication.h b/base/poco/Util/include/Poco/Util/ServerApplication.h index 3e90f4505d7..5df4c28538a 100644 --- a/base/poco/Util/include/Poco/Util/ServerApplication.h +++ b/base/poco/Util/include/Poco/Util/ServerApplication.h @@ -143,14 +143,6 @@ namespace Util /// Runs the application by performing additional initializations /// and calling the main() method. -#if defined(POCO_WIN32_UTF8) && !defined(POCO_NO_WSTRING) - int run(int argc, wchar_t ** argv); - /// Runs the application by performing additional initializations - /// and calling the main() method. - /// - /// This Windows-specific version of init is used for passing - /// Unicode command line arguments from wmain(). -#endif static void terminate(); /// Sends a friendly termination request to the application. diff --git a/base/poco/Util/include/Poco/Util/WinService.h b/base/poco/Util/include/Poco/Util/WinService.h index a0f3d1c5796..52377dfb67b 100644 --- a/base/poco/Util/include/Poco/Util/WinService.h +++ b/base/poco/Util/include/Poco/Util/WinService.h @@ -22,11 +22,7 @@ #include "Poco/Util/Util.h" -#if defined(POCO_WIN32_UTF8) -# define POCO_LPQUERY_SERVICE_CONFIG LPQUERY_SERVICE_CONFIGW -#else # define POCO_LPQUERY_SERVICE_CONFIG LPQUERY_SERVICE_CONFIGA -#endif namespace Poco diff --git a/base/poco/Util/src/Application.cpp b/base/poco/Util/src/Application.cpp index e8b3a4feaa3..483315fda60 100644 --- a/base/poco/Util/src/Application.cpp +++ b/base/poco/Util/src/Application.cpp @@ -38,9 +38,6 @@ #if defined(POCO_OS_FAMILY_UNIX) && !defined(POCO_VXWORKS) #include "Poco/SignalHandler.h" #endif -#if defined(POCO_WIN32_UTF8) && !defined(POCO_NO_WSTRING) -#include "Poco/UnicodeConverter.h" -#endif using Poco::Logger; @@ -131,19 +128,6 @@ void Application::init(int argc, char* argv[]) } -#if defined(POCO_WIN32_UTF8) && !defined(POCO_NO_WSTRING) -void Application::init(int argc, wchar_t* argv[]) -{ - std::vector args; - for (int i = 0; i < argc; ++i) - { - std::string arg; - Poco::UnicodeConverter::toUTF8(argv[i], arg); - args.push_back(arg); - } - init(args); -} -#endif void Application::init(const ArgVec& args) diff --git a/base/poco/Util/src/ServerApplication.cpp b/base/poco/Util/src/ServerApplication.cpp index b6f8205e38e..091263ed34b 100644 --- a/base/poco/Util/src/ServerApplication.cpp +++ b/base/poco/Util/src/ServerApplication.cpp @@ -32,9 +32,6 @@ #include #include #endif -#if defined(POCO_WIN32_UTF8) && !defined(POCO_NO_WSTRING) -#include "Poco/UnicodeConverter.h" -#endif using Poco::NumberFormatter; From 8fc46f6ea174db8ae539eed287bfe656a7a71afc Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 15 Feb 2023 09:29:37 +0000 Subject: [PATCH 234/717] Remove POCO_DLL --- base/poco/Foundation/include/Poco/Foundation.h | 7 ------- base/poco/Foundation/src/EventLogChannel.cpp | 7 ------- base/poco/NetSSL_OpenSSL/include/Poco/Net/NetSSL.h | 7 ------- 3 files changed, 21 deletions(-) diff --git a/base/poco/Foundation/include/Poco/Foundation.h b/base/poco/Foundation/include/Poco/Foundation.h index 747cff7818c..34493041720 100644 --- a/base/poco/Foundation/include/Poco/Foundation.h +++ b/base/poco/Foundation/include/Poco/Foundation.h @@ -39,13 +39,6 @@ // Foundation_API functions as being imported from a DLL, whereas this DLL sees symbols // defined with this macro as being exported. // -#if (defined(_WIN32) || defined(_WIN32_WCE)) && defined(POCO_DLL) -# if defined(Foundation_EXPORTS) -# define Foundation_API __declspec(dllexport) -# else -# define Foundation_API __declspec(dllimport) -# endif -#endif #if !defined(Foundation_API) diff --git a/base/poco/Foundation/src/EventLogChannel.cpp b/base/poco/Foundation/src/EventLogChannel.cpp index 33a19cef967..ec35a9b5d28 100644 --- a/base/poco/Foundation/src/EventLogChannel.cpp +++ b/base/poco/Foundation/src/EventLogChannel.cpp @@ -184,13 +184,6 @@ void EventLogChannel::setUpRegistry() const if (disp == REG_CREATED_NEW_KEY) { std::string path; - #if defined(POCO_DLL) - #if defined(_DEBUG) - path = findLibrary("PocoFoundationd.dll"); - #else - path = findLibrary("PocoFoundation.dll"); - #endif - #endif if (path.empty()) path = findLibrary("PocoMsg.dll"); diff --git a/base/poco/NetSSL_OpenSSL/include/Poco/Net/NetSSL.h b/base/poco/NetSSL_OpenSSL/include/Poco/Net/NetSSL.h index d6c7eba1a3d..5c3b45be8c6 100644 --- a/base/poco/NetSSL_OpenSSL/include/Poco/Net/NetSSL.h +++ b/base/poco/NetSSL_OpenSSL/include/Poco/Net/NetSSL.h @@ -32,13 +32,6 @@ // NetSSL_API functions as being imported from a DLL, whereas this DLL sees symbols // defined with this macro as being exported. // -#if (defined(_WIN32) || defined(__CYGWIN__)) && defined(POCO_DLL) -# if defined(NetSSL_EXPORTS) -# define NetSSL_API __declspec(dllexport) -# else -# define NetSSL_API __declspec(dllimport) -# endif -#endif #if !defined(NetSSL_API) From 44afecf083b2cfa3d77d2e227f99223133ab5aee Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 15 Feb 2023 09:32:03 +0000 Subject: [PATCH 235/717] Remove more unused files --- .../include/Poco/DirectoryIterator_WIN32.h | 74 --- .../include/Poco/DirectoryIterator_WIN32U.h | 74 --- .../Foundation/include/Poco/Environment_VX.h | 57 -- .../include/Poco/Environment_WIN32.h | 49 -- .../include/Poco/Environment_WIN32U.h | 49 -- .../include/Poco/Environment_WINCE.h | 60 --- base/poco/Foundation/include/Poco/Event_VX.h | 50 -- .../Foundation/include/Poco/Event_WIN32.h | 69 --- .../include/Poco/FPEnvironment_DEC.h | 74 --- .../include/Poco/FPEnvironment_QNX.h | 134 ----- .../include/Poco/FPEnvironment_SUN.h | 75 --- .../include/Poco/FPEnvironment_WIN32.h | 151 ------ base/poco/Foundation/include/Poco/File_VX.h | 85 --- .../poco/Foundation/include/Poco/File_WIN32.h | 88 ---- .../Foundation/include/Poco/File_WIN32U.h | 92 ---- .../poco/Foundation/include/Poco/File_WINCE.h | 91 ---- .../Foundation/include/Poco/LogFile_WIN32.h | 56 -- .../Foundation/include/Poco/LogFile_WIN32U.h | 56 -- base/poco/Foundation/include/Poco/Mutex_VX.h | 81 --- .../Foundation/include/Poco/Mutex_WIN32.h | 86 --- .../Foundation/include/Poco/Mutex_WINCE.h | 51 -- .../include/Poco/NamedEvent_WIN32.h | 46 -- .../include/Poco/NamedEvent_WIN32U.h | 47 -- .../include/Poco/NamedMutex_WIN32.h | 47 -- .../include/Poco/NamedMutex_WIN32U.h | 48 -- .../poco/Foundation/include/Poco/Path_WIN32.h | 50 -- .../Foundation/include/Poco/Path_WIN32U.h | 55 -- .../poco/Foundation/include/Poco/Path_WINCE.h | 55 -- .../Foundation/include/Poco/PipeImpl_WIN32.h | 55 -- .../Foundation/include/Poco/Platform_VX.h | 29 -- .../Foundation/include/Poco/Platform_WIN32.h | 168 ------ .../poco/Foundation/include/Poco/Process_VX.h | 79 --- .../Foundation/include/Poco/Process_WIN32.h | 84 --- .../Foundation/include/Poco/Process_WIN32U.h | 84 --- .../Foundation/include/Poco/Process_WINCE.h | 84 --- base/poco/Foundation/include/Poco/RWLock_VX.h | 91 ---- .../Foundation/include/Poco/RWLock_WIN32.h | 58 --- .../Foundation/include/Poco/RWLock_WINCE.h | 62 --- .../Foundation/include/Poco/Semaphore_VX.h | 57 -- .../Foundation/include/Poco/Semaphore_WIN32.h | 59 --- .../include/Poco/SharedLibrary_HPUX.h | 53 -- .../include/Poco/SharedLibrary_VX.h | 53 -- .../include/Poco/SharedLibrary_WIN32.h | 52 -- .../include/Poco/SharedLibrary_WIN32U.h | 52 -- .../include/Poco/SharedMemory_WIN32.h | 104 ---- base/poco/Foundation/include/Poco/Thread_VX.h | 167 ------ .../Foundation/include/Poco/Thread_WIN32.h | 175 ------- .../Foundation/include/Poco/Thread_WINCE.h | 171 ------ .../src/DirectoryIterator_WIN32.cpp | 66 --- .../src/DirectoryIterator_WIN32U.cpp | 71 --- base/poco/Foundation/src/Environment_VX.cpp | 157 ------ .../poco/Foundation/src/Environment_WIN32.cpp | 221 -------- .../Foundation/src/Environment_WIN32U.cpp | 235 --------- .../poco/Foundation/src/Environment_WINCE.cpp | 243 --------- base/poco/Foundation/src/Event_VX.cpp | 79 --- base/poco/Foundation/src/Event_WIN32.cpp | 61 --- .../poco/Foundation/src/FPEnvironment_SUN.cpp | 139 ----- .../Foundation/src/FPEnvironment_WIN32.cpp | 76 --- base/poco/Foundation/src/FileStream_WIN32.cpp | 199 ------- base/poco/Foundation/src/File_VX.cpp | 408 --------------- base/poco/Foundation/src/File_WIN32.cpp | 465 ----------------- base/poco/Foundation/src/File_WIN32U.cpp | 492 ------------------ base/poco/Foundation/src/File_WINCE.cpp | 441 ---------------- base/poco/Foundation/src/LogFile_WIN32.cpp | 125 ----- base/poco/Foundation/src/LogFile_WIN32U.cpp | 130 ----- base/poco/Foundation/src/Mutex_VX.cpp | 68 --- base/poco/Foundation/src/Mutex_WIN32.cpp | 59 --- base/poco/Foundation/src/Mutex_WINCE.cpp | 80 --- base/poco/Foundation/src/NamedEvent_WIN32.cpp | 61 --- .../poco/Foundation/src/NamedEvent_WIN32U.cpp | 63 --- base/poco/Foundation/src/NamedMutex_WIN32.cpp | 73 --- .../poco/Foundation/src/NamedMutex_WIN32U.cpp | 75 --- base/poco/Foundation/src/Path_WIN32.cpp | 203 -------- base/poco/Foundation/src/Path_WIN32U.cpp | 222 -------- base/poco/Foundation/src/Path_WINCE.cpp | 144 ----- base/poco/Foundation/src/PipeImpl_WIN32.cpp | 97 ---- base/poco/Foundation/src/Process_VX.cpp | 100 ---- base/poco/Foundation/src/Process_WIN32.cpp | 349 ------------- base/poco/Foundation/src/Process_WIN32U.cpp | 371 ------------- base/poco/Foundation/src/Process_WINCE.cpp | 223 -------- base/poco/Foundation/src/RWLock_VX.cpp | 41 -- base/poco/Foundation/src/RWLock_WIN32.cpp | 207 -------- base/poco/Foundation/src/RWLock_WINCE.cpp | 174 ------- .../Foundation/src/SharedLibrary_HPUX.cpp | 100 ---- base/poco/Foundation/src/SharedLibrary_VX.cpp | 141 ----- .../Foundation/src/SharedLibrary_WIN32.cpp | 108 ---- .../Foundation/src/SharedLibrary_WIN32U.cpp | 113 ---- .../Foundation/src/SharedMemory_WIN32.cpp | 145 ------ base/poco/Foundation/src/Thread_VX.cpp | 273 ---------- base/poco/Foundation/src/Thread_WIN32.cpp | 223 -------- base/poco/Foundation/src/Thread_WINCE.cpp | 169 ------ base/poco/Foundation/src/Timezone_WIN32.cpp | 89 ---- base/poco/Foundation/src/Timezone_WINCE.cpp | 84 --- 93 files changed, 11350 deletions(-) delete mode 100644 base/poco/Foundation/include/Poco/DirectoryIterator_WIN32.h delete mode 100644 base/poco/Foundation/include/Poco/DirectoryIterator_WIN32U.h delete mode 100644 base/poco/Foundation/include/Poco/Environment_VX.h delete mode 100644 base/poco/Foundation/include/Poco/Environment_WIN32.h delete mode 100644 base/poco/Foundation/include/Poco/Environment_WIN32U.h delete mode 100644 base/poco/Foundation/include/Poco/Environment_WINCE.h delete mode 100644 base/poco/Foundation/include/Poco/Event_VX.h delete mode 100644 base/poco/Foundation/include/Poco/Event_WIN32.h delete mode 100644 base/poco/Foundation/include/Poco/FPEnvironment_DEC.h delete mode 100644 base/poco/Foundation/include/Poco/FPEnvironment_QNX.h delete mode 100644 base/poco/Foundation/include/Poco/FPEnvironment_SUN.h delete mode 100644 base/poco/Foundation/include/Poco/FPEnvironment_WIN32.h delete mode 100644 base/poco/Foundation/include/Poco/File_VX.h delete mode 100644 base/poco/Foundation/include/Poco/File_WIN32.h delete mode 100644 base/poco/Foundation/include/Poco/File_WIN32U.h delete mode 100644 base/poco/Foundation/include/Poco/File_WINCE.h delete mode 100644 base/poco/Foundation/include/Poco/LogFile_WIN32.h delete mode 100644 base/poco/Foundation/include/Poco/LogFile_WIN32U.h delete mode 100644 base/poco/Foundation/include/Poco/Mutex_VX.h delete mode 100644 base/poco/Foundation/include/Poco/Mutex_WIN32.h delete mode 100644 base/poco/Foundation/include/Poco/Mutex_WINCE.h delete mode 100644 base/poco/Foundation/include/Poco/NamedEvent_WIN32.h delete mode 100644 base/poco/Foundation/include/Poco/NamedEvent_WIN32U.h delete mode 100644 base/poco/Foundation/include/Poco/NamedMutex_WIN32.h delete mode 100644 base/poco/Foundation/include/Poco/NamedMutex_WIN32U.h delete mode 100644 base/poco/Foundation/include/Poco/Path_WIN32.h delete mode 100644 base/poco/Foundation/include/Poco/Path_WIN32U.h delete mode 100644 base/poco/Foundation/include/Poco/Path_WINCE.h delete mode 100644 base/poco/Foundation/include/Poco/PipeImpl_WIN32.h delete mode 100644 base/poco/Foundation/include/Poco/Platform_VX.h delete mode 100644 base/poco/Foundation/include/Poco/Platform_WIN32.h delete mode 100644 base/poco/Foundation/include/Poco/Process_VX.h delete mode 100644 base/poco/Foundation/include/Poco/Process_WIN32.h delete mode 100644 base/poco/Foundation/include/Poco/Process_WIN32U.h delete mode 100644 base/poco/Foundation/include/Poco/Process_WINCE.h delete mode 100644 base/poco/Foundation/include/Poco/RWLock_VX.h delete mode 100644 base/poco/Foundation/include/Poco/RWLock_WIN32.h delete mode 100644 base/poco/Foundation/include/Poco/RWLock_WINCE.h delete mode 100644 base/poco/Foundation/include/Poco/Semaphore_VX.h delete mode 100644 base/poco/Foundation/include/Poco/Semaphore_WIN32.h delete mode 100644 base/poco/Foundation/include/Poco/SharedLibrary_HPUX.h delete mode 100644 base/poco/Foundation/include/Poco/SharedLibrary_VX.h delete mode 100644 base/poco/Foundation/include/Poco/SharedLibrary_WIN32.h delete mode 100644 base/poco/Foundation/include/Poco/SharedLibrary_WIN32U.h delete mode 100644 base/poco/Foundation/include/Poco/SharedMemory_WIN32.h delete mode 100644 base/poco/Foundation/include/Poco/Thread_VX.h delete mode 100644 base/poco/Foundation/include/Poco/Thread_WIN32.h delete mode 100644 base/poco/Foundation/include/Poco/Thread_WINCE.h delete mode 100644 base/poco/Foundation/src/DirectoryIterator_WIN32.cpp delete mode 100644 base/poco/Foundation/src/DirectoryIterator_WIN32U.cpp delete mode 100644 base/poco/Foundation/src/Environment_VX.cpp delete mode 100644 base/poco/Foundation/src/Environment_WIN32.cpp delete mode 100644 base/poco/Foundation/src/Environment_WIN32U.cpp delete mode 100644 base/poco/Foundation/src/Environment_WINCE.cpp delete mode 100644 base/poco/Foundation/src/Event_VX.cpp delete mode 100644 base/poco/Foundation/src/Event_WIN32.cpp delete mode 100644 base/poco/Foundation/src/FPEnvironment_SUN.cpp delete mode 100644 base/poco/Foundation/src/FPEnvironment_WIN32.cpp delete mode 100644 base/poco/Foundation/src/FileStream_WIN32.cpp delete mode 100644 base/poco/Foundation/src/File_VX.cpp delete mode 100644 base/poco/Foundation/src/File_WIN32.cpp delete mode 100644 base/poco/Foundation/src/File_WIN32U.cpp delete mode 100644 base/poco/Foundation/src/File_WINCE.cpp delete mode 100644 base/poco/Foundation/src/LogFile_WIN32.cpp delete mode 100644 base/poco/Foundation/src/LogFile_WIN32U.cpp delete mode 100644 base/poco/Foundation/src/Mutex_VX.cpp delete mode 100644 base/poco/Foundation/src/Mutex_WIN32.cpp delete mode 100644 base/poco/Foundation/src/Mutex_WINCE.cpp delete mode 100644 base/poco/Foundation/src/NamedEvent_WIN32.cpp delete mode 100644 base/poco/Foundation/src/NamedEvent_WIN32U.cpp delete mode 100644 base/poco/Foundation/src/NamedMutex_WIN32.cpp delete mode 100644 base/poco/Foundation/src/NamedMutex_WIN32U.cpp delete mode 100644 base/poco/Foundation/src/Path_WIN32.cpp delete mode 100644 base/poco/Foundation/src/Path_WIN32U.cpp delete mode 100644 base/poco/Foundation/src/Path_WINCE.cpp delete mode 100644 base/poco/Foundation/src/PipeImpl_WIN32.cpp delete mode 100644 base/poco/Foundation/src/Process_VX.cpp delete mode 100644 base/poco/Foundation/src/Process_WIN32.cpp delete mode 100644 base/poco/Foundation/src/Process_WIN32U.cpp delete mode 100644 base/poco/Foundation/src/Process_WINCE.cpp delete mode 100644 base/poco/Foundation/src/RWLock_VX.cpp delete mode 100644 base/poco/Foundation/src/RWLock_WIN32.cpp delete mode 100644 base/poco/Foundation/src/RWLock_WINCE.cpp delete mode 100644 base/poco/Foundation/src/SharedLibrary_HPUX.cpp delete mode 100644 base/poco/Foundation/src/SharedLibrary_VX.cpp delete mode 100644 base/poco/Foundation/src/SharedLibrary_WIN32.cpp delete mode 100644 base/poco/Foundation/src/SharedLibrary_WIN32U.cpp delete mode 100644 base/poco/Foundation/src/SharedMemory_WIN32.cpp delete mode 100644 base/poco/Foundation/src/Thread_VX.cpp delete mode 100644 base/poco/Foundation/src/Thread_WIN32.cpp delete mode 100644 base/poco/Foundation/src/Thread_WINCE.cpp delete mode 100644 base/poco/Foundation/src/Timezone_WIN32.cpp delete mode 100644 base/poco/Foundation/src/Timezone_WINCE.cpp diff --git a/base/poco/Foundation/include/Poco/DirectoryIterator_WIN32.h b/base/poco/Foundation/include/Poco/DirectoryIterator_WIN32.h deleted file mode 100644 index 81b1dd06d35..00000000000 --- a/base/poco/Foundation/include/Poco/DirectoryIterator_WIN32.h +++ /dev/null @@ -1,74 +0,0 @@ -// -// DirectoryIterator_WIN32.h -// -// Library: Foundation -// Package: Filesystem -// Module: DirectoryIterator -// -// Definition of the DirectoryIteratorImpl class for WIN32. -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_DirectoryIterator_WIN32_INCLUDED -#define Foundation_DirectoryIterator_WIN32_INCLUDED - - -#include "Poco/Foundation.h" -#include "Poco/UnWindows.h" - - -namespace Poco -{ - - -class Foundation_API DirectoryIteratorImpl -{ -public: - DirectoryIteratorImpl(const std::string & path); - ~DirectoryIteratorImpl(); - - void duplicate(); - void release(); - - const std::string & get() const; - const std::string & next(); - -private: - HANDLE _fh; - WIN32_FIND_DATA _fd; - std::string _current; - int _rc; -}; - - -// -// inlines -// -const std::string & DirectoryIteratorImpl::get() const -{ - return _current; -} - - -inline void DirectoryIteratorImpl::duplicate() -{ - ++_rc; -} - - -inline void DirectoryIteratorImpl::release() -{ - if (--_rc == 0) - delete this; -} - - -} // namespace Poco - - -#endif // Foundation_DirectoryIterator_WIN32_INCLUDED diff --git a/base/poco/Foundation/include/Poco/DirectoryIterator_WIN32U.h b/base/poco/Foundation/include/Poco/DirectoryIterator_WIN32U.h deleted file mode 100644 index dc358c620a6..00000000000 --- a/base/poco/Foundation/include/Poco/DirectoryIterator_WIN32U.h +++ /dev/null @@ -1,74 +0,0 @@ -// -// DirectoryIterator_WIN32U.h -// -// Library: Foundation -// Package: Filesystem -// Module: DirectoryIterator -// -// Definition of the DirectoryIteratorImpl class for WIN32. -// -// Copyright (c) 2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_DirectoryIterator_WIN32U_INCLUDED -#define Foundation_DirectoryIterator_WIN32U_INCLUDED - - -#include "Poco/Foundation.h" -#include "Poco/UnWindows.h" - - -namespace Poco -{ - - -class Foundation_API DirectoryIteratorImpl -{ -public: - DirectoryIteratorImpl(const std::string & path); - ~DirectoryIteratorImpl(); - - void duplicate(); - void release(); - - const std::string & get() const; - const std::string & next(); - -private: - HANDLE _fh; - WIN32_FIND_DATAW _fd; - std::string _current; - int _rc; -}; - - -// -// inlines -// -const std::string & DirectoryIteratorImpl::get() const -{ - return _current; -} - - -inline void DirectoryIteratorImpl::duplicate() -{ - ++_rc; -} - - -inline void DirectoryIteratorImpl::release() -{ - if (--_rc == 0) - delete this; -} - - -} // namespace Poco - - -#endif // Foundation_DirectoryIterator_WIN32U_INCLUDED diff --git a/base/poco/Foundation/include/Poco/Environment_VX.h b/base/poco/Foundation/include/Poco/Environment_VX.h deleted file mode 100644 index 583103ac620..00000000000 --- a/base/poco/Foundation/include/Poco/Environment_VX.h +++ /dev/null @@ -1,57 +0,0 @@ -// -// Environment_VX.h -// -// Library: Foundation -// Package: Core -// Module: Environment -// -// Definition of the EnvironmentImpl class for VxWorks. -// -// Copyright (c) 2004-2011, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_Environment_VX_INCLUDED -#define Foundation_Environment_VX_INCLUDED - - -#include -#include "Poco/Foundation.h" -#include "Poco/Mutex.h" - - -namespace Poco -{ - - -class Foundation_API EnvironmentImpl -{ -public: - typedef UInt8 NodeId[6]; /// Ethernet address. - - static std::string getImpl(const std::string & name); - static bool hasImpl(const std::string & name); - static void setImpl(const std::string & name, const std::string & value); - static std::string osNameImpl(); - static std::string osDisplayNameImpl(); - static std::string osVersionImpl(); - static std::string osArchitectureImpl(); - static std::string nodeNameImpl(); - static void nodeIdImpl(NodeId & id); - static unsigned processorCountImpl(); - -private: - typedef std::map StringMap; - - static StringMap _map; - static FastMutex _mutex; -}; - - -} // namespace Poco - - -#endif // Foundation_Environment_VX_INCLUDED diff --git a/base/poco/Foundation/include/Poco/Environment_WIN32.h b/base/poco/Foundation/include/Poco/Environment_WIN32.h deleted file mode 100644 index 451a27a5792..00000000000 --- a/base/poco/Foundation/include/Poco/Environment_WIN32.h +++ /dev/null @@ -1,49 +0,0 @@ -// -// Environment_WIN32.h -// -// Library: Foundation -// Package: Core -// Module: Environment -// -// Definition of the EnvironmentImpl class for WIN32. -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_Environment_WIN32_INCLUDED -#define Foundation_Environment_WIN32_INCLUDED - - -#include "Poco/Foundation.h" - - -namespace Poco -{ - - -class Foundation_API EnvironmentImpl -{ -public: - typedef UInt8 NodeId[6]; /// Ethernet address. - - static std::string getImpl(const std::string & name); - static bool hasImpl(const std::string & name); - static void setImpl(const std::string & name, const std::string & value); - static std::string osNameImpl(); - static std::string osDisplayNameImpl(); - static std::string osVersionImpl(); - static std::string osArchitectureImpl(); - static std::string nodeNameImpl(); - static void nodeIdImpl(NodeId & id); - static unsigned processorCountImpl(); -}; - - -} // namespace Poco - - -#endif // Foundation_Environment_WIN32_INCLUDED diff --git a/base/poco/Foundation/include/Poco/Environment_WIN32U.h b/base/poco/Foundation/include/Poco/Environment_WIN32U.h deleted file mode 100644 index 95ef6aac7bf..00000000000 --- a/base/poco/Foundation/include/Poco/Environment_WIN32U.h +++ /dev/null @@ -1,49 +0,0 @@ -// -// Environment_WIN32U.h -// -// Library: Foundation -// Package: Core -// Module: Environment -// -// Definition of the EnvironmentImpl class for WIN32. -// -// Copyright (c) 2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_Environment_WIN32U_INCLUDED -#define Foundation_Environment_WIN32U_INCLUDED - - -#include "Poco/Foundation.h" - - -namespace Poco -{ - - -class Foundation_API EnvironmentImpl -{ -public: - typedef UInt8 NodeId[6]; /// Ethernet address. - - static std::string getImpl(const std::string & name); - static bool hasImpl(const std::string & name); - static void setImpl(const std::string & name, const std::string & value); - static std::string osNameImpl(); - static std::string osDisplayNameImpl(); - static std::string osVersionImpl(); - static std::string osArchitectureImpl(); - static std::string nodeNameImpl(); - static void nodeIdImpl(NodeId & id); - static unsigned processorCountImpl(); -}; - - -} // namespace Poco - - -#endif // Foundation_Environment_WIN32U_INCLUDED diff --git a/base/poco/Foundation/include/Poco/Environment_WINCE.h b/base/poco/Foundation/include/Poco/Environment_WINCE.h deleted file mode 100644 index a9bf71181e7..00000000000 --- a/base/poco/Foundation/include/Poco/Environment_WINCE.h +++ /dev/null @@ -1,60 +0,0 @@ -// -// Environment_WINCE.h -// -// Library: Foundation -// Package: Core -// Module: Environment -// -// Definition of the EnvironmentImpl class for WINCE. -// -// Copyright (c) 2009-2010, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_Environment_WINCE_INCLUDED -#define Foundation_Environment_WINCE_INCLUDED - - -#include "Poco/Foundation.h" - - -namespace Poco -{ - - -class Foundation_API EnvironmentImpl -{ -public: - typedef UInt8 NodeId[6]; /// Ethernet address. - - static std::string getImpl(const std::string & name); - static bool hasImpl(const std::string & name); - static void setImpl(const std::string & name, const std::string & value); - static std::string osNameImpl(); - static std::string osDisplayNameImpl(); - static std::string osVersionImpl(); - static std::string osArchitectureImpl(); - static std::string nodeNameImpl(); - static void nodeIdImpl(NodeId & id); - static unsigned processorCountImpl(); - -private: - static bool envVar(const std::string & name, std::string * value); - - static const std::string TEMP; - static const std::string TMP; - static const std::string HOMEPATH; - static const std::string COMPUTERNAME; - static const std::string OS; - static const std::string NUMBER_OF_PROCESSORS; - static const std::string PROCESSOR_ARCHITECTURE; -}; - - -} // namespace Poco - - -#endif // Foundation_Environment_WINCE_INCLUDED diff --git a/base/poco/Foundation/include/Poco/Event_VX.h b/base/poco/Foundation/include/Poco/Event_VX.h deleted file mode 100644 index adee455d6c7..00000000000 --- a/base/poco/Foundation/include/Poco/Event_VX.h +++ /dev/null @@ -1,50 +0,0 @@ -// -// Event_VX.h -// -// Library: Foundation -// Package: Threading -// Module: Event -// -// Definition of the EventImpl class for VxWorks. -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_Event_VX_INCLUDED -#define Foundation_Event_VX_INCLUDED - - -#include -#include "Poco/Exception.h" -#include "Poco/Foundation.h" - - -namespace Poco -{ - - -class Foundation_API EventImpl -{ -protected: - EventImpl(bool autoReset); - ~EventImpl(); - void setImpl(); - void waitImpl(); - bool waitImpl(long milliseconds); - void resetImpl(); - -private: - bool _auto; - volatile bool _state; - SEM_ID _sem; -}; - - -} // namespace Poco - - -#endif // Foundation_Event_VX_INCLUDED diff --git a/base/poco/Foundation/include/Poco/Event_WIN32.h b/base/poco/Foundation/include/Poco/Event_WIN32.h deleted file mode 100644 index abda1d5686d..00000000000 --- a/base/poco/Foundation/include/Poco/Event_WIN32.h +++ /dev/null @@ -1,69 +0,0 @@ -// -// Event_WIN32.h -// -// Library: Foundation -// Package: Threading -// Module: Event -// -// Definition of the EventImpl class for WIN32. -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_Event_WIN32_INCLUDED -#define Foundation_Event_WIN32_INCLUDED - - -#include "Poco/Exception.h" -#include "Poco/Foundation.h" -#include "Poco/UnWindows.h" - - -namespace Poco -{ - - -class Foundation_API EventImpl -{ -protected: - EventImpl(bool autoReset); - ~EventImpl(); - void setImpl(); - void waitImpl(); - bool waitImpl(long milliseconds); - void resetImpl(); - -private: - HANDLE _event; -}; - - -// -// inlines -// -inline void EventImpl::setImpl() -{ - if (!SetEvent(_event)) - { - throw SystemException("cannot signal event"); - } -} - - -inline void EventImpl::resetImpl() -{ - if (!ResetEvent(_event)) - { - throw SystemException("cannot reset event"); - } -} - - -} // namespace Poco - - -#endif // Foundation_Event_WIN32_INCLUDED diff --git a/base/poco/Foundation/include/Poco/FPEnvironment_DEC.h b/base/poco/Foundation/include/Poco/FPEnvironment_DEC.h deleted file mode 100644 index 8f764750b5e..00000000000 --- a/base/poco/Foundation/include/Poco/FPEnvironment_DEC.h +++ /dev/null @@ -1,74 +0,0 @@ -// -// FPEnvironment_DEC.h -// -// Library: Foundation -// Package: Core -// Module: FPEnvironment -// -// Definitions of class FPEnvironmentImpl for Tru64 and OpenVMS Alpha. -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_FPEnvironment_DEC_INCLUDED -#define Foundation_FPEnvironment_DEC_INCLUDED - - -#include "Poco/Foundation.h" -# include - - -namespace Poco -{ - - -class FPEnvironmentImpl -{ -protected: - enum RoundingModeImpl - { - FP_ROUND_DOWNWARD_IMPL = 0, - FP_ROUND_UPWARD_IMPL = 0, - FP_ROUND_TONEAREST_IMPL = 0, - FP_ROUND_TOWARDZERO_IMPL = 0 - }; - enum FlagImpl - { - FP_DIVIDE_BY_ZERO_IMPL = IEEE_STATUS_DZE, - FP_INEXACT_IMPL = IEEE_STATUS_INE, - FP_OVERFLOW_IMPL = IEEE_STATUS_OVF, - FP_UNDERFLOW_IMPL = IEEE_STATUS_UNF, - FP_INVALID_IMPL = IEEE_STATUS_INV - }; - FPEnvironmentImpl(); - FPEnvironmentImpl(const FPEnvironmentImpl & env); - ~FPEnvironmentImpl(); - FPEnvironmentImpl & operator=(const FPEnvironmentImpl & env); - void keepCurrentImpl(); - static void clearFlagsImpl(); - static bool isFlagImpl(FlagImpl flag); - static void setRoundingModeImpl(RoundingModeImpl mode); - static RoundingModeImpl getRoundingModeImpl(); - static bool isInfiniteImpl(float value); - static bool isInfiniteImpl(double value); - static bool isInfiniteImpl(long double value); - static bool isNaNImpl(float value); - static bool isNaNImpl(double value); - static bool isNaNImpl(long double value); - static float copySignImpl(float target, float source); - static double copySignImpl(double target, double source); - static long double copySignImpl(long double target, long double source); - -private: - unsigned long _env; -}; - - -} // namespace Poco - - -#endif // Foundation_FPEnvironment_DEC_INCLUDED diff --git a/base/poco/Foundation/include/Poco/FPEnvironment_QNX.h b/base/poco/Foundation/include/Poco/FPEnvironment_QNX.h deleted file mode 100644 index 360abf8d31c..00000000000 --- a/base/poco/Foundation/include/Poco/FPEnvironment_QNX.h +++ /dev/null @@ -1,134 +0,0 @@ -// -// FPEnvironment_QNX.h -// -// Library: Foundation -// Package: Core -// Module: FPEnvironment -// -// Definitions of class FPEnvironmentImpl for QNX. -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_FPEnvironment_QNX_INCLUDED -#define Foundation_FPEnvironment_QNX_INCLUDED - - -#include -#include -#include "Poco/Foundation.h" - - -namespace Poco -{ - - -class FPEnvironmentImpl -{ -protected: - enum RoundingModeImpl - { - FP_ROUND_DOWNWARD_IMPL = FE_DOWNWARD, - FP_ROUND_UPWARD_IMPL = FE_UPWARD, - FP_ROUND_TONEAREST_IMPL = FE_TONEAREST, - FP_ROUND_TOWARDZERO_IMPL = FE_TOWARDZERO - }; - enum FlagImpl - { - FP_DIVIDE_BY_ZERO_IMPL = FE_DIVBYZERO, - FP_INEXACT_IMPL = FE_INEXACT, - FP_OVERFLOW_IMPL = FE_OVERFLOW, - FP_UNDERFLOW_IMPL = FE_UNDERFLOW, - FP_INVALID_IMPL = FE_INVALID - }; - FPEnvironmentImpl(); - FPEnvironmentImpl(const FPEnvironmentImpl & env); - ~FPEnvironmentImpl(); - FPEnvironmentImpl & operator=(const FPEnvironmentImpl & env); - void keepCurrentImpl(); - static void clearFlagsImpl(); - static bool isFlagImpl(FlagImpl flag); - static void setRoundingModeImpl(RoundingModeImpl mode); - static RoundingModeImpl getRoundingModeImpl(); - static bool isInfiniteImpl(float value); - static bool isInfiniteImpl(double value); - static bool isInfiniteImpl(long double value); - static bool isNaNImpl(float value); - static bool isNaNImpl(double value); - static bool isNaNImpl(long double value); - static float copySignImpl(float target, float source); - static double copySignImpl(double target, double source); - static long double copySignImpl(long double target, long double source); - -private: - fenv_t _env; -}; - - -// -// inlines -// -inline bool FPEnvironmentImpl::isInfiniteImpl(float value) -{ - using namespace std; - return isinf(value) != 0; -} - - -inline bool FPEnvironmentImpl::isInfiniteImpl(double value) -{ - using namespace std; - return isinf(value) != 0; -} - - -inline bool FPEnvironmentImpl::isInfiniteImpl(long double value) -{ - using namespace std; - return isinf((double)value) != 0; -} - - -inline bool FPEnvironmentImpl::isNaNImpl(float value) -{ - using namespace std; - return isnan(value) != 0; -} - - -inline bool FPEnvironmentImpl::isNaNImpl(double value) -{ - using namespace std; - return isnan(value) != 0; -} - - -inline bool FPEnvironmentImpl::isNaNImpl(long double value) -{ - using namespace std; - return isnan((double)value) != 0; -} - - -inline float FPEnvironmentImpl::copySignImpl(float target, float source) -{ - using namespace std; - return copysignf(target, source); -} - - -inline double FPEnvironmentImpl::copySignImpl(double target, double source) -{ - using namespace std; - return copysign(target, source); -} - - -} // namespace Poco - - -#endif // Foundation_FPEnvironment_QNX_INCLUDED diff --git a/base/poco/Foundation/include/Poco/FPEnvironment_SUN.h b/base/poco/Foundation/include/Poco/FPEnvironment_SUN.h deleted file mode 100644 index 7b31307e1ca..00000000000 --- a/base/poco/Foundation/include/Poco/FPEnvironment_SUN.h +++ /dev/null @@ -1,75 +0,0 @@ -// -// FPEnvironment_SUN.h -// -// Library: Foundation -// Package: Core -// Module: FPEnvironment -// -// Definitions of class FPEnvironmentImpl for Solaris. -// -// Copyright (c) 2005-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_FPEnvironment_SUN_INCLUDED -#define Foundation_FPEnvironment_SUN_INCLUDED - - -#include -#include "Poco/Foundation.h" - - -namespace Poco -{ - - -class FPEnvironmentImpl -{ -protected: - enum RoundingModeImpl - { - FP_ROUND_DOWNWARD_IMPL = FP_RM, - FP_ROUND_UPWARD_IMPL = FP_RP, - FP_ROUND_TONEAREST_IMPL = FP_RN, - FP_ROUND_TOWARDZERO_IMPL = FP_RZ - }; - enum FlagImpl - { - FP_DIVIDE_BY_ZERO_IMPL = FP_X_DZ, - FP_INEXACT_IMPL = FP_X_IMP, - FP_OVERFLOW_IMPL = FP_X_OFL, - FP_UNDERFLOW_IMPL = FP_X_UFL, - FP_INVALID_IMPL = FP_X_INV - }; - FPEnvironmentImpl(); - FPEnvironmentImpl(const FPEnvironmentImpl & env); - ~FPEnvironmentImpl(); - FPEnvironmentImpl & operator=(const FPEnvironmentImpl & env); - void keepCurrentImpl(); - static void clearFlagsImpl(); - static bool isFlagImpl(FlagImpl flag); - static void setRoundingModeImpl(RoundingModeImpl mode); - static RoundingModeImpl getRoundingModeImpl(); - static bool isInfiniteImpl(float value); - static bool isInfiniteImpl(double value); - static bool isInfiniteImpl(long double value); - static bool isNaNImpl(float value); - static bool isNaNImpl(double value); - static bool isNaNImpl(long double value); - static float copySignImpl(float target, float source); - static double copySignImpl(double target, double source); - static long double copySignImpl(long double target, long double source); - -private: - fp_rnd _rnd; - fp_except _exc; -}; - - -} // namespace Poco - - -#endif // Foundation_FPEnvironment_SUN_INCLUDED diff --git a/base/poco/Foundation/include/Poco/FPEnvironment_WIN32.h b/base/poco/Foundation/include/Poco/FPEnvironment_WIN32.h deleted file mode 100644 index 0a36bde9712..00000000000 --- a/base/poco/Foundation/include/Poco/FPEnvironment_WIN32.h +++ /dev/null @@ -1,151 +0,0 @@ -// -// FPEnvironment_WIN32.h -// -// Library: Foundation -// Package: Core -// Module: FPEnvironment -// -// Definitions of class FPEnvironmentImpl for WIN32. -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_FPEnvironment_WIN32_INCLUDED -#define Foundation_FPEnvironment_WIN32_INCLUDED - - -#include -#include -#include "Poco/Foundation.h" - -#ifndef _SW_INEXACT -# define _SW_INEXACT 0x00000001 // inexact (precision) -#endif -#ifndef _SW_UNDERFLOW -# define _SW_UNDERFLOW 0x00000002 // underflow -#endif -#ifndef _SW_OVERFLOW -# define _SW_OVERFLOW 0x00000004 // overflow -#endif -#ifndef _SW_ZERODIVIDE -# define _SW_ZERODIVIDE 0x00000008 // zero divide -#endif -#ifndef _SW_INVALID -# define _SW_INVALID 0x00000010 // invalid -#endif -#ifndef _SW_DENORMAL -# define _SW_DENORMAL 0x00080000 // denormal status bit -#endif - - -namespace Poco -{ - - -class Foundation_API FPEnvironmentImpl -{ -protected: - enum RoundingModeImpl - { - FP_ROUND_DOWNWARD_IMPL = _RC_DOWN, - FP_ROUND_UPWARD_IMPL = _RC_UP, - FP_ROUND_TONEAREST_IMPL = _RC_NEAR, - FP_ROUND_TOWARDZERO_IMPL = _RC_CHOP - }; - enum FlagImpl - { - FP_DIVIDE_BY_ZERO_IMPL = _SW_ZERODIVIDE, - FP_INEXACT_IMPL = _SW_INEXACT, - FP_OVERFLOW_IMPL = _SW_OVERFLOW, - FP_UNDERFLOW_IMPL = _SW_UNDERFLOW, - FP_INVALID_IMPL = _SW_INVALID - }; - FPEnvironmentImpl(); - FPEnvironmentImpl(const FPEnvironmentImpl & env); - ~FPEnvironmentImpl(); - FPEnvironmentImpl & operator=(const FPEnvironmentImpl & env); - void keepCurrentImpl(); - static void clearFlagsImpl(); - static bool isFlagImpl(FlagImpl flag); - static void setRoundingModeImpl(RoundingModeImpl mode); - static RoundingModeImpl getRoundingModeImpl(); - static bool isInfiniteImpl(float value); - static bool isInfiniteImpl(double value); - static bool isInfiniteImpl(long double value); - static bool isNaNImpl(float value); - static bool isNaNImpl(double value); - static bool isNaNImpl(long double value); - static float copySignImpl(float target, float source); - static double copySignImpl(double target, double source); - static long double copySignImpl(long double target, long double source); - -private: - unsigned _env; -}; - - -// -// inlines -// -inline bool FPEnvironmentImpl::isInfiniteImpl(float value) -{ - return _finite(value) == 0; -} - - -inline bool FPEnvironmentImpl::isInfiniteImpl(double value) -{ - return _finite(value) == 0; -} - - -inline bool FPEnvironmentImpl::isInfiniteImpl(long double value) -{ - return _finite(value) == 0; -} - - -inline bool FPEnvironmentImpl::isNaNImpl(float value) -{ - return _isnan(value) != 0; -} - - -inline bool FPEnvironmentImpl::isNaNImpl(double value) -{ - return _isnan(value) != 0; -} - - -inline bool FPEnvironmentImpl::isNaNImpl(long double value) -{ - return _isnan(value) != 0; -} - - -inline float FPEnvironmentImpl::copySignImpl(float target, float source) -{ - return float(_copysign(target, source)); -} - - -inline double FPEnvironmentImpl::copySignImpl(double target, double source) -{ - return _copysign(target, source); -} - - -inline long double FPEnvironmentImpl::copySignImpl(long double target, long double source) -{ - return (source > 0 && target > 0) || (source < 0 && target < 0) ? target : -target; -} - - -} // namespace Poco - - -#endif // Foundation_FPEnvironment_WIN32_INCLUDED diff --git a/base/poco/Foundation/include/Poco/File_VX.h b/base/poco/Foundation/include/Poco/File_VX.h deleted file mode 100644 index f329d0a6d41..00000000000 --- a/base/poco/Foundation/include/Poco/File_VX.h +++ /dev/null @@ -1,85 +0,0 @@ -// -// File_VX.h -// -// Library: Foundation -// Package: Filesystem -// Module: File -// -// Definition of the FileImpl class for VxWorks. -// -// Copyright (c) 2004-2011, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_File_VX_INCLUDED -#define Foundation_File_VX_INCLUDED - - -#include "Poco/Foundation.h" - - -namespace Poco -{ - - -class FileImpl -{ -protected: - typedef UInt64 FileSizeImpl; - - FileImpl(); - FileImpl(const std::string & path); - virtual ~FileImpl(); - void swapImpl(FileImpl & file); - void setPathImpl(const std::string & path); - const std::string & getPathImpl() const; - bool existsImpl() const; - bool canReadImpl() const; - bool canWriteImpl() const; - bool canExecuteImpl() const; - bool isFileImpl() const; - bool isDirectoryImpl() const; - bool isLinkImpl() const; - bool isDeviceImpl() const; - bool isHiddenImpl() const; - Timestamp createdImpl() const; - Timestamp getLastModifiedImpl() const; - void setLastModifiedImpl(const Timestamp & ts); - FileSizeImpl getSizeImpl() const; - void setSizeImpl(FileSizeImpl size); - void setWriteableImpl(bool flag = true); - void setExecutableImpl(bool flag = true); - void copyToImpl(const std::string & path) const; - void renameToImpl(const std::string & path); - void linkToImpl(const std::string & path, int type) const; - void removeImpl(); - bool createFileImpl(); - bool createDirectoryImpl(); - FileSizeImpl totalSpaceImpl() const; - FileSizeImpl usableSpaceImpl() const; - FileSizeImpl freeSpaceImpl() const; - static void handleLastErrorImpl(const std::string & path); - -private: - std::string _path; - - friend class DirectoryIteratorImpl; -}; - - -// -// inlines -// -inline const std::string & FileImpl::getPathImpl() const -{ - return _path; -} - - -} // namespace Poco - - -#endif // Foundation_File_VX_INCLUDED diff --git a/base/poco/Foundation/include/Poco/File_WIN32.h b/base/poco/Foundation/include/Poco/File_WIN32.h deleted file mode 100644 index 236abb0ff5e..00000000000 --- a/base/poco/Foundation/include/Poco/File_WIN32.h +++ /dev/null @@ -1,88 +0,0 @@ -// -// File_WIN32.h -// -// Library: Foundation -// Package: Filesystem -// Module: File -// -// Definition of the FileImpl class for WIN32. -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_File_WIN32_INCLUDED -#define Foundation_File_WIN32_INCLUDED - - -#include "Poco/Foundation.h" -#include "Poco/Timestamp.h" - - -namespace Poco -{ - - -class Foundation_API FileImpl -{ -protected: - typedef UInt64 FileSizeImpl; - - FileImpl(); - FileImpl(const std::string & path); - virtual ~FileImpl(); - void swapImpl(FileImpl & file); - void setPathImpl(const std::string & path); - const std::string & getPathImpl() const; - bool existsImpl() const; - bool canReadImpl() const; - bool canWriteImpl() const; - bool canExecuteImpl() const; - bool isFileImpl() const; - bool isDirectoryImpl() const; - bool isLinkImpl() const; - bool isDeviceImpl() const; - bool isHiddenImpl() const; - Timestamp createdImpl() const; - Timestamp getLastModifiedImpl() const; - void setLastModifiedImpl(const Timestamp & ts); - FileSizeImpl getSizeImpl() const; - void setSizeImpl(FileSizeImpl size); - void setWriteableImpl(bool flag = true); - void setExecutableImpl(bool flag = true); - void copyToImpl(const std::string & path) const; - void renameToImpl(const std::string & path); - void linkToImpl(const std::string & path, int type) const; - void removeImpl(); - bool createFileImpl(); - bool createDirectoryImpl(); - FileSizeImpl totalSpaceImpl() const; - FileSizeImpl usableSpaceImpl() const; - FileSizeImpl freeSpaceImpl() const; - static void handleLastErrorImpl(const std::string & path); - -private: - std::string _path; - - friend class FileHandle; - friend class DirectoryIteratorImpl; - friend class WindowsDirectoryWatcherStrategy; -}; - - -// -// inlines -// -inline const std::string & FileImpl::getPathImpl() const -{ - return _path; -} - - -} // namespace Poco - - -#endif // Foundation_File_WIN32_INCLUDED diff --git a/base/poco/Foundation/include/Poco/File_WIN32U.h b/base/poco/Foundation/include/Poco/File_WIN32U.h deleted file mode 100644 index 87a4f3e7e11..00000000000 --- a/base/poco/Foundation/include/Poco/File_WIN32U.h +++ /dev/null @@ -1,92 +0,0 @@ -// -// File_WIN32U.h -// -// Library: Foundation -// Package: Filesystem -// Module: File -// -// Definition of the Unicode FileImpl class for WIN32. -// -// Copyright (c) 2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_File_WIN32U_INCLUDED -#define Foundation_File_WIN32U_INCLUDED - - -#include "Poco/Foundation.h" -#include "Poco/Timestamp.h" - - -namespace Poco -{ - - -class Foundation_API FileImpl -{ -protected: - typedef UInt64 FileSizeImpl; - - FileImpl(); - FileImpl(const std::string & path); - virtual ~FileImpl(); - void swapImpl(FileImpl & file); - void setPathImpl(const std::string & path); - const std::string & getPathImpl() const; - bool existsImpl() const; - bool canReadImpl() const; - bool canWriteImpl() const; - bool canExecuteImpl() const; - bool isFileImpl() const; - bool isDirectoryImpl() const; - bool isLinkImpl() const; - bool isDeviceImpl() const; - bool isHiddenImpl() const; - Timestamp createdImpl() const; - Timestamp getLastModifiedImpl() const; - void setLastModifiedImpl(const Timestamp & ts); - FileSizeImpl getSizeImpl() const; - void setSizeImpl(FileSizeImpl size); - void setWriteableImpl(bool flag = true); - void setExecutableImpl(bool flag = true); - void copyToImpl(const std::string & path) const; - void renameToImpl(const std::string & path); - void linkToImpl(const std::string & path, int type) const; - void removeImpl(); - bool createFileImpl(); - bool createDirectoryImpl(); - FileSizeImpl totalSpaceImpl() const; - FileSizeImpl usableSpaceImpl() const; - FileSizeImpl freeSpaceImpl() const; - static void handleLastErrorImpl(const std::string & path); - static void convertPath(const std::string & utf8Path, std::wstring & utf16Path); - -private: - std::string _path; - std::wstring _upath; - - friend class FileHandle; - friend class DirectoryIteratorImpl; - friend class WindowsDirectoryWatcherStrategy; - friend class FileStreamBuf; - friend class LogFileImpl; -}; - - -// -// inlines -// -inline const std::string & FileImpl::getPathImpl() const -{ - return _path; -} - - -} // namespace Poco - - -#endif // Foundation_File_WIN32U_INCLUDED diff --git a/base/poco/Foundation/include/Poco/File_WINCE.h b/base/poco/Foundation/include/Poco/File_WINCE.h deleted file mode 100644 index 457e2c94538..00000000000 --- a/base/poco/Foundation/include/Poco/File_WINCE.h +++ /dev/null @@ -1,91 +0,0 @@ -// -// File_WIN32U.h -// -// Library: Foundation -// Package: Filesystem -// Module: File -// -// Definition of the Unicode FileImpl class for WIN32. -// -// Copyright (c) 2006-2010, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_File_WINCE_INCLUDED -#define Foundation_File_WINCE_INCLUDED - - -#include "Poco/Foundation.h" -#include "Poco/Timestamp.h" - - -namespace Poco -{ - - -class Foundation_API FileImpl -{ -protected: - typedef UInt64 FileSizeImpl; - - FileImpl(); - FileImpl(const std::string & path); - virtual ~FileImpl(); - void swapImpl(FileImpl & file); - void setPathImpl(const std::string & path); - const std::string & getPathImpl() const; - bool existsImpl() const; - bool canReadImpl() const; - bool canWriteImpl() const; - bool canExecuteImpl() const; - bool isFileImpl() const; - bool isDirectoryImpl() const; - bool isLinkImpl() const; - bool isDeviceImpl() const; - bool isHiddenImpl() const; - Timestamp createdImpl() const; - Timestamp getLastModifiedImpl() const; - void setLastModifiedImpl(const Timestamp & ts); - FileSizeImpl getSizeImpl() const; - void setSizeImpl(FileSizeImpl size); - void setWriteableImpl(bool flag = true); - void setExecutableImpl(bool flag = true); - void copyToImpl(const std::string & path) const; - void renameToImpl(const std::string & path); - void linkToImpl(const std::string & path, int type) const; - void removeImpl(); - bool createFileImpl(); - bool createDirectoryImpl(); - FileSizeImpl totalSpaceImpl() const; - FileSizeImpl usableSpaceImpl() const; - FileSizeImpl freeSpaceImpl() const; - static void handleLastErrorImpl(const std::string & path); - static void convertPath(const std::string & utf8Path, std::wstring & utf16Path); - -private: - std::string _path; - std::wstring _upath; - - friend class FileHandle; - friend class DirectoryIteratorImpl; - friend class FileStreamBuf; - friend class LogFileImpl; -}; - - -// -// inlines -// -inline const std::string & FileImpl::getPathImpl() const -{ - return _path; -} - - -} // namespace Poco - - -#endif // Foundation_File_WINCE_INCLUDED diff --git a/base/poco/Foundation/include/Poco/LogFile_WIN32.h b/base/poco/Foundation/include/Poco/LogFile_WIN32.h deleted file mode 100644 index f95b973e2c1..00000000000 --- a/base/poco/Foundation/include/Poco/LogFile_WIN32.h +++ /dev/null @@ -1,56 +0,0 @@ -// -// LogFile_WIN32.h -// -// Library: Foundation -// Package: Logging -// Module: LogFile -// -// Definition of the LogFileImpl class using the Windows file APIs. -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_LogFile_WIN32_INCLUDED -#define Foundation_LogFile_WIN32_INCLUDED - - -#include "Poco/Foundation.h" -#include "Poco/Timestamp.h" -#include "Poco/UnWindows.h" - - -namespace Poco -{ - - -class Foundation_API LogFileImpl -/// The implementation of LogFile for Windows. -/// The native filesystem APIs are used for -/// total control over locking behavior. -{ -public: - LogFileImpl(const std::string & path); - ~LogFileImpl(); - void writeImpl(const std::string & text, bool flush); - void writeBinaryImpl(const char * data, size_t size, bool flush); - UInt64 sizeImpl() const; - Timestamp creationDateImpl() const; - const std::string & pathImpl() const; - -private: - void createFile(); - - std::string _path; - HANDLE _hFile; - Timestamp _creationDate; -}; - - -} // namespace Poco - - -#endif // Foundation_LogFile_WIN32_INCLUDED diff --git a/base/poco/Foundation/include/Poco/LogFile_WIN32U.h b/base/poco/Foundation/include/Poco/LogFile_WIN32U.h deleted file mode 100644 index a9aba7729ee..00000000000 --- a/base/poco/Foundation/include/Poco/LogFile_WIN32U.h +++ /dev/null @@ -1,56 +0,0 @@ -// -// LogFile_WIN32U.h -// -// Library: Foundation -// Package: Logging -// Module: LogFile -// -// Definition of the LogFileImpl class using the Windows file APIs. -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_LogFile_WIN32U_INCLUDED -#define Foundation_LogFile_WIN32U_INCLUDED - - -#include "Poco/Foundation.h" -#include "Poco/Timestamp.h" -#include "Poco/UnWindows.h" - - -namespace Poco -{ - - -class Foundation_API LogFileImpl -/// The implementation of LogFile for Windows. -/// The native filesystem APIs are used for -/// total control over locking behavior. -{ -public: - LogFileImpl(const std::string & path); - ~LogFileImpl(); - void writeImpl(const std::string & text, bool flush); - void writeBinaryImpl(const char * data, size_t size, bool flush); - UInt64 sizeImpl() const; - Timestamp creationDateImpl() const; - const std::string & pathImpl() const; - -private: - void createFile(); - - std::string _path; - HANDLE _hFile; - Timestamp _creationDate; -}; - - -} // namespace Poco - - -#endif // Foundation_LogFile_WIN32U_INCLUDED diff --git a/base/poco/Foundation/include/Poco/Mutex_VX.h b/base/poco/Foundation/include/Poco/Mutex_VX.h deleted file mode 100644 index 50569672c89..00000000000 --- a/base/poco/Foundation/include/Poco/Mutex_VX.h +++ /dev/null @@ -1,81 +0,0 @@ -// -// Mutex_VX.h -// -// Library: Foundation -// Package: Threading -// Module: Mutex -// -// Definition of the MutexImpl and FastMutexImpl classes for VxWorks. -// -// Copyright (c) 2004-2011, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_Mutex_VX_INCLUDED -#define Foundation_Mutex_VX_INCLUDED - - -#include -#include -#include "Poco/Exception.h" -#include "Poco/Foundation.h" - - -namespace Poco -{ - - -class Foundation_API MutexImpl -{ -protected: - MutexImpl(); - MutexImpl(bool fast); - ~MutexImpl(); - void lockImpl(); - bool tryLockImpl(); - bool tryLockImpl(long milliseconds); - void unlockImpl(); - -private: - SEM_ID _sem; -}; - - -class Foundation_API FastMutexImpl : public MutexImpl -{ -protected: - FastMutexImpl(); - ~FastMutexImpl(); -}; - - -// -// inlines -// -inline void MutexImpl::lockImpl() -{ - if (semTake(_sem, WAIT_FOREVER) != OK) - throw SystemException("cannot lock mutex"); -} - - -inline bool MutexImpl::tryLockImpl() -{ - return semTake(_sem, NO_WAIT) == OK; -} - - -inline void MutexImpl::unlockImpl() -{ - if (semGive(_sem) != OK) - throw SystemException("cannot unlock mutex"); -} - - -} // namespace Poco - - -#endif // Foundation_Mutex_VX_INCLUDED diff --git a/base/poco/Foundation/include/Poco/Mutex_WIN32.h b/base/poco/Foundation/include/Poco/Mutex_WIN32.h deleted file mode 100644 index bfcd481fbe0..00000000000 --- a/base/poco/Foundation/include/Poco/Mutex_WIN32.h +++ /dev/null @@ -1,86 +0,0 @@ -// -// Mutex_WIN32.h -// -// Library: Foundation -// Package: Threading -// Module: Mutex -// -// Definition of the MutexImpl and FastMutexImpl classes for WIN32. -// -// Copyright (c) 2004-2008, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_Mutex_WIN32_INCLUDED -#define Foundation_Mutex_WIN32_INCLUDED - - -#include "Poco/Exception.h" -#include "Poco/Foundation.h" -#include "Poco/UnWindows.h" - - -namespace Poco -{ - - -class Foundation_API MutexImpl -{ -protected: - MutexImpl(); - ~MutexImpl(); - void lockImpl(); - bool tryLockImpl(); - bool tryLockImpl(long milliseconds); - void unlockImpl(); - -private: - CRITICAL_SECTION _cs; -}; - - -typedef MutexImpl FastMutexImpl; - - -// -// inlines -// -inline void MutexImpl::lockImpl() -{ - try - { - EnterCriticalSection(&_cs); - } - catch (...) - { - throw SystemException("cannot lock mutex"); - } -} - - -inline bool MutexImpl::tryLockImpl() -{ - try - { - return TryEnterCriticalSection(&_cs) != 0; - } - catch (...) - { - } - throw SystemException("cannot lock mutex"); -} - - -inline void MutexImpl::unlockImpl() -{ - LeaveCriticalSection(&_cs); -} - - -} // namespace Poco - - -#endif // Foundation_Mutex_WIN32_INCLUDED diff --git a/base/poco/Foundation/include/Poco/Mutex_WINCE.h b/base/poco/Foundation/include/Poco/Mutex_WINCE.h deleted file mode 100644 index 01b903a2d93..00000000000 --- a/base/poco/Foundation/include/Poco/Mutex_WINCE.h +++ /dev/null @@ -1,51 +0,0 @@ -// -// Mutex_WIN32.h -// -// Library: Foundation -// Package: Threading -// Module: Mutex -// -// Definition of the MutexImpl and FastMutexImpl classes for WIN32. -// -// Copyright (c) 2004-2010, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_Mutex_WINCE_INCLUDED -#define Foundation_Mutex_WINCE_INCLUDED - - -#include "Poco/Exception.h" -#include "Poco/Foundation.h" -#include "Poco/UnWindows.h" - - -namespace Poco -{ - - -class Foundation_API MutexImpl -{ -protected: - MutexImpl(); - ~MutexImpl(); - void lockImpl(); - bool tryLockImpl(); - bool tryLockImpl(long milliseconds); - void unlockImpl(); - -private: - HANDLE _mutex; -}; - - -typedef MutexImpl FastMutexImpl; - - -} // namespace Poco - - -#endif // Foundation_Mutex_WINCE_INCLUDED diff --git a/base/poco/Foundation/include/Poco/NamedEvent_WIN32.h b/base/poco/Foundation/include/Poco/NamedEvent_WIN32.h deleted file mode 100644 index 68ebd24c29c..00000000000 --- a/base/poco/Foundation/include/Poco/NamedEvent_WIN32.h +++ /dev/null @@ -1,46 +0,0 @@ -// -// NamedEvent_WIN32.h -// -// Library: Foundation -// Package: Processes -// Module: NamedEvent -// -// Definition of the NamedEventImpl class for Windows. -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_NamedEvent_WIN32_INCLUDED -#define Foundation_NamedEvent_WIN32_INCLUDED - - -#include "Poco/Foundation.h" -#include "Poco/UnWindows.h" - - -namespace Poco -{ - - -class Foundation_API NamedEventImpl -{ -protected: - NamedEventImpl(const std::string & name); - ~NamedEventImpl(); - void setImpl(); - void waitImpl(); - -private: - std::string _name; - HANDLE _event; -}; - - -} // namespace Poco - - -#endif // Foundation_NamedEvent_WIN32_INCLUDED diff --git a/base/poco/Foundation/include/Poco/NamedEvent_WIN32U.h b/base/poco/Foundation/include/Poco/NamedEvent_WIN32U.h deleted file mode 100644 index f97daed7847..00000000000 --- a/base/poco/Foundation/include/Poco/NamedEvent_WIN32U.h +++ /dev/null @@ -1,47 +0,0 @@ -// -// NamedEvent_WIN32U.h -// -// Library: Foundation -// Package: Processes -// Module: NamedEvent -// -// Definition of the NamedEventImpl class for Windows. -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_NamedEvent_WIN32U_INCLUDED -#define Foundation_NamedEvent_WIN32U_INCLUDED - - -#include "Poco/Foundation.h" -#include "Poco/UnWindows.h" - - -namespace Poco -{ - - -class Foundation_API NamedEventImpl -{ -protected: - NamedEventImpl(const std::string & name); - ~NamedEventImpl(); - void setImpl(); - void waitImpl(); - -private: - std::string _name; - std::wstring _uname; - HANDLE _event; -}; - - -} // namespace Poco - - -#endif // Foundation_NamedEvent_WIN32U_INCLUDED diff --git a/base/poco/Foundation/include/Poco/NamedMutex_WIN32.h b/base/poco/Foundation/include/Poco/NamedMutex_WIN32.h deleted file mode 100644 index 5b33d1b2d35..00000000000 --- a/base/poco/Foundation/include/Poco/NamedMutex_WIN32.h +++ /dev/null @@ -1,47 +0,0 @@ -// -// NamedMutex_WIN32.h -// -// Library: Foundation -// Package: Processes -// Module: NamedMutex -// -// Definition of the NamedMutexImpl class for Windows. -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_NamedMutex_WIN32_INCLUDED -#define Foundation_NamedMutex_WIN32_INCLUDED - - -#include "Poco/Foundation.h" -#include "Poco/UnWindows.h" - - -namespace Poco -{ - - -class Foundation_API NamedMutexImpl -{ -protected: - NamedMutexImpl(const std::string & name); - ~NamedMutexImpl(); - void lockImpl(); - bool tryLockImpl(); - void unlockImpl(); - -private: - std::string _name; - HANDLE _mutex; -}; - - -} // namespace Poco - - -#endif // Foundation_NamedMutex_WIN32_INCLUDED diff --git a/base/poco/Foundation/include/Poco/NamedMutex_WIN32U.h b/base/poco/Foundation/include/Poco/NamedMutex_WIN32U.h deleted file mode 100644 index 50b3a328302..00000000000 --- a/base/poco/Foundation/include/Poco/NamedMutex_WIN32U.h +++ /dev/null @@ -1,48 +0,0 @@ -// -// NamedMutex_WIN32U.h -// -// Library: Foundation -// Package: Processes -// Module: NamedMutex -// -// Definition of the NamedMutexImpl class for Windows. -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_NamedMutex_WIN32U_INCLUDED -#define Foundation_NamedMutex_WIN32U_INCLUDED - - -#include "Poco/Foundation.h" -#include "Poco/UnWindows.h" - - -namespace Poco -{ - - -class Foundation_API NamedMutexImpl -{ -protected: - NamedMutexImpl(const std::string & name); - ~NamedMutexImpl(); - void lockImpl(); - bool tryLockImpl(); - void unlockImpl(); - -private: - std::string _name; - std::wstring _uname; - HANDLE _mutex; -}; - - -} // namespace Poco - - -#endif // Foundation_NamedMutex_WIN32U_INCLUDED diff --git a/base/poco/Foundation/include/Poco/Path_WIN32.h b/base/poco/Foundation/include/Poco/Path_WIN32.h deleted file mode 100644 index d75f154cb47..00000000000 --- a/base/poco/Foundation/include/Poco/Path_WIN32.h +++ /dev/null @@ -1,50 +0,0 @@ -// -// Path_WIN32.h -// -// Library: Foundation -// Package: Filesystem -// Module: Path -// -// Definition of the PathImpl class for WIN32. -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_Path_WIN32_INCLUDED -#define Foundation_Path_WIN32_INCLUDED - - -#include -#include "Poco/Foundation.h" - - -namespace Poco -{ - - -class Foundation_API PathImpl -{ -public: - static std::string currentImpl(); - static std::string homeImpl(); - static std::string configHomeImpl(); - static std::string dataHomeImpl(); - static std::string cacheHomeImpl(); - static std::string tempHomeImpl(); - static std::string tempImpl(); - static std::string configImpl(); - static std::string nullImpl(); - static std::string systemImpl(); - static std::string expandImpl(const std::string & path); - static void listRootsImpl(std::vector & roots); -}; - - -} // namespace Poco - - -#endif // Foundation_Path_WIN32_INCLUDED diff --git a/base/poco/Foundation/include/Poco/Path_WIN32U.h b/base/poco/Foundation/include/Poco/Path_WIN32U.h deleted file mode 100644 index 7c20999cd1b..00000000000 --- a/base/poco/Foundation/include/Poco/Path_WIN32U.h +++ /dev/null @@ -1,55 +0,0 @@ -// -// Path_WIN32U.h -// -// Library: Foundation -// Package: Filesystem -// Module: Path -// -// Definition of the PathImpl class for WIN32. -// -// Copyright (c) 2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_Path_WIN32U_INCLUDED -#define Foundation_Path_WIN32U_INCLUDED - - -#include -#include "Poco/Foundation.h" - - -namespace Poco -{ - - -class Foundation_API PathImpl -{ -public: - static std::string currentImpl(); - static std::string homeImpl(); - static std::string configHomeImpl(); - static std::string dataHomeImpl(); - static std::string cacheHomeImpl(); - static std::string tempHomeImpl(); - static std::string tempImpl(); - static std::string configImpl(); - static std::string nullImpl(); - static std::string systemImpl(); - static std::string expandImpl(const std::string & path); - static void listRootsImpl(std::vector & roots); - - enum - { - MAX_PATH_LEN = 32767 - }; -}; - - -} // namespace Poco - - -#endif // Foundation_Path_WIN32U_INCLUDED diff --git a/base/poco/Foundation/include/Poco/Path_WINCE.h b/base/poco/Foundation/include/Poco/Path_WINCE.h deleted file mode 100644 index d615e2452ef..00000000000 --- a/base/poco/Foundation/include/Poco/Path_WINCE.h +++ /dev/null @@ -1,55 +0,0 @@ -// -// Path_WINCE.h -// -// Library: Foundation -// Package: Filesystem -// Module: Path -// -// Definition of the PathImpl class for WIN32. -// -// Copyright (c) 2006-2010, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_Path_WINCE_INCLUDED -#define Foundation_Path_WINCE_INCLUDED - - -#include -#include "Poco/Foundation.h" - - -namespace Poco -{ - - -class Foundation_API PathImpl -{ -public: - static std::string currentImpl(); - static std::string homeImpl(); - static std::string configHomeImpl(); - static std::string dataHomeImpl(); - static std::string cacheHomeImpl(); - static std::string tempHomeImpl(); - static std::string tempImpl(); - static std::string configImpl(); - static std::string nullImpl(); - static std::string systemImpl(); - static std::string expandImpl(const std::string & path); - static void listRootsImpl(std::vector & roots); - - enum - { - MAX_PATH_LEN = 32767 - }; -}; - - -} // namespace Poco - - -#endif // Foundation_Path_WINCE_INCLUDED diff --git a/base/poco/Foundation/include/Poco/PipeImpl_WIN32.h b/base/poco/Foundation/include/Poco/PipeImpl_WIN32.h deleted file mode 100644 index 08ea26e8c79..00000000000 --- a/base/poco/Foundation/include/Poco/PipeImpl_WIN32.h +++ /dev/null @@ -1,55 +0,0 @@ -// -// PipeImpl_WIN32.h -// -// Library: Foundation -// Package: Processes -// Module: PipeImpl -// -// Definition of the PipeImpl class for WIN32. -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_PipeImpl_WIN32_INCLUDED -#define Foundation_PipeImpl_WIN32_INCLUDED - - -#include "Poco/Foundation.h" -#include "Poco/RefCountedObject.h" -#include "Poco/UnWindows.h" - - -namespace Poco -{ - - -class Foundation_API PipeImpl : public RefCountedObject -/// A dummy implementation of PipeImpl for platforms -/// that do not support pipes. -{ -public: - typedef HANDLE Handle; - - PipeImpl(); - ~PipeImpl(); - int writeBytes(const void * buffer, int length); - int readBytes(void * buffer, int length); - Handle readHandle() const; - Handle writeHandle() const; - void closeRead(); - void closeWrite(); - -private: - HANDLE _readHandle; - HANDLE _writeHandle; -}; - - -} // namespace Poco - - -#endif // Foundation_PipeImpl_WIN32_INCLUDED diff --git a/base/poco/Foundation/include/Poco/Platform_VX.h b/base/poco/Foundation/include/Poco/Platform_VX.h deleted file mode 100644 index fbe566f7da8..00000000000 --- a/base/poco/Foundation/include/Poco/Platform_VX.h +++ /dev/null @@ -1,29 +0,0 @@ -// -// Platform_VX.h -// -// Library: Foundation -// Package: Core -// Module: Platform -// -// Platform and architecture identification macros -// and platform-specific definitions for VxWorks -// -// Copyright (c) 2004-2011, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_Platform_VX_INCLUDED -#define Foundation_Platform_VX_INCLUDED - - -#define POCO_NO_SYS_SELECT_H -#define POCO_NO_FPENVIRONMENT -#define POCO_NO_WSTRING -#define POCO_NO_SHAREDMEMORY -#define POCO_NO_SYSLOGCHANNEL - - -#endif // Foundation_Platform_VX_INCLUDED diff --git a/base/poco/Foundation/include/Poco/Platform_WIN32.h b/base/poco/Foundation/include/Poco/Platform_WIN32.h deleted file mode 100644 index 8f77b171dac..00000000000 --- a/base/poco/Foundation/include/Poco/Platform_WIN32.h +++ /dev/null @@ -1,168 +0,0 @@ -// -// Platform_WIN32.h -// -// Library: Foundation -// Package: Core -// Module: Platform -// -// Platform and architecture identification macros -// and platform-specific definitions for Windows. -// -// Copyright (c) 2004-2007, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_Platform_WIN32_INCLUDED -#define Foundation_Platform_WIN32_INCLUDED - - -#include "Poco/UnWindows.h" - - -#ifndef POCO_FORCE_MIN_WINDOWS_OS_SUPPORT -// Determine the real version. -// This setting can be forced from UnWindows.h -# if defined(_WIN32_WINNT_WINBLUE) -//Windows 8.1 _WIN32_WINNT_WINBLUE (0x0602) -# ifdef _WIN32_WINNT -# undef _WIN32_WINNT -# endif -# define _WIN32_WINNT _WIN32_WINNT_WINBLUE -# ifdef NTDDI_VERSION -# undef NTDDI_VERSION -# endif -# define NTDDI_VERSION NTDDI_WINBLUE -# elif defined(_WIN32_WINNT_WIN8) -//Windows 8 _WIN32_WINNT_WIN8 (0x0602) -# ifdef _WIN32_WINNT -# undef _WIN32_WINNT -# endif -# define _WIN32_WINNT _WIN32_WINNT_WIN8 -# ifdef NTDDI_VERSION -# undef NTDDI_VERSION -# endif -# define NTDDI_VERSION NTDDI_WIN8 -# elif defined(_WIN32_WINNT_WIN7) -//Windows 7 _WIN32_WINNT_WIN7 (0x0601) -# ifdef _WIN32_WINNT -# undef _WIN32_WINNT -# endif -# define _WIN32_WINNT _WIN32_WINNT_WIN7 -# ifdef NTDDI_VERSION -# undef NTDDI_VERSION -# endif -# define NTDDI_VERSION NTDDI_WIN7 -# elif defined(_WIN32_WINNT_WS08) -//Windows Server 2008 _WIN32_WINNT_WS08 (0x0600) -# ifdef _WIN32_WINNT -# undef _WIN32_WINNT -# endif -# define _WIN32_WINNT _WIN32_WINNT_WS08 -# ifdef NTDDI_VERSION -# undef NTDDI_VERSION -# endif -# define NTDDI_VERSION NTDDI_WS08 -# elif defined(_WIN32_WINNT_VISTA) -//Windows Vista _WIN32_WINNT_VISTA (0x0600) -# ifdef _WIN32_WINNT -# undef _WIN32_WINNT -# endif -# define _WIN32_WINNT _WIN32_WINNT_VISTA -# ifdef NTDDI_VERSION -# undef NTDDI_VERSION -# endif -# define NTDDI_VERSION NTDDI_VISTA -# elif defined(_WIN32_WINNT_LONGHORN) -//Windows Vista and server 2008 Development _WIN32_WINNT_LONGHORN (0x0600) -# ifdef _WIN32_WINNT -# undef _WIN32_WINNT -# endif -# define _WIN32_WINNT _WIN32_WINNT_LONGHORN -# ifdef NTDDI_VERSION -# undef NTDDI_VERSION -# endif -# define NTDDI_VERSION 0x06000000 // hardcoded, VS90 can't find NTDDI_* macros -# elif defined(_WIN32_WINNT_WS03) -//Windows Server 2003 with SP1, -//Windows XP with SP2 _WIN32_WINNT_WS03 (0x0502) -# ifdef _WIN32_WINNT -# undef _WIN32_WINNT -# endif -# define _WIN32_WINNT _WIN32_WINNT_WS03 -# ifdef NTDDI_VERSION -# undef NTDDI_VERSION -# endif -# define NTDDI_VERSION NTDDI_WS03 -# elif defined(_WIN32_WINNT_WINXP) -//Windows Server 2003, Windows XP _WIN32_WINNT_WINXP (0x0501) -# ifdef _WIN32_WINNT -# undef _WIN32_WINNT -# endif -# define _WIN32_WINNT _WIN32_WINNT_WINXP -# ifdef NTDDI_VERSION -# undef NTDDI_VERSION -# endif -# define NTDDI_VERSION NTDDI_WINXP -# elif defined(_WIN32_WINNT_WIN2K) -//Windows 2000 _WIN32_WINNT_WIN2K (0x0500) -# ifdef _WIN32_WINNT -# undef _WIN32_WINNT -# endif -# define _WIN32_WINNT _WIN32_WINNT_WIN2K -# elif defined(WINVER) -// fail back on WINVER -# ifdef _WIN32_WINNT -# undef _WIN32_WINNT -# endif -# define _WIN32_WINNT WINVER -# elif !defined(_WIN32_WINNT) -// last resort = Win XP, SP2 is minimum supported -# define _WIN32_WINNT 0x0502 -# ifdef NTDDI_VERSION -# undef NTDDI_VERSION -# endif -# define NTDDI_VERSION 0x05020000 -# endif -#endif // POCO_FORCE_MIN_WINDOWS_OS_SUPPORT - - - - -// Verify that we're built with the multithreaded -// versions of the runtime libraries - - -// Check debug/release settings consistency -#if defined(NDEBUG) && defined(_DEBUG) -# error Inconsistent build settings (check for /MD[d]) -#endif - - - - -// Unicode Support -#if defined(UNICODE) && !defined(POCO_WIN32_UTF8) -# define POCO_WIN32_UTF8 -#endif - - -# pragma message("Compiling POCO on Windows without #define POCO_WIN32_UTF8 is deprecated.") - - -// Turn off some annoying warnings - - -// Enable C++11 support for VS 2010 and newer - - -#if defined(__INTEL_COMPILER) -# pragma warning(disable : 1738) // base class dllexport/dllimport specification differs from that of the derived class -# pragma warning(disable : 1478) // function ... was declared "deprecated" -# pragma warning(disable : 1744) // field of class type without a DLL interface used in a class with a DLL interface -#endif - - -#endif // Foundation_Platform_WIN32_INCLUDED diff --git a/base/poco/Foundation/include/Poco/Process_VX.h b/base/poco/Foundation/include/Poco/Process_VX.h deleted file mode 100644 index 01ba9c89630..00000000000 --- a/base/poco/Foundation/include/Poco/Process_VX.h +++ /dev/null @@ -1,79 +0,0 @@ -// -// Process_VX.h -// -// Library: Foundation -// Package: Processes -// Module: Process -// -// Definition of the ProcessImpl class for VxWorks. -// -// Copyright (c) 2004-20011, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_Process_VX_INCLUDED -#define Foundation_Process_VX_INCLUDED - - -#include -#include -#include "Poco/Foundation.h" -#include "Poco/RefCountedObject.h" - - -#undef PID - - -namespace Poco -{ - - -class Pipe; - - -class Foundation_API ProcessHandleImpl : public RefCountedObject -{ -public: - ProcessHandleImpl(int pid); - ~ProcessHandleImpl(); - - int id() const; - int wait() const; - -private: - int _pid; -}; - - -class Foundation_API ProcessImpl -{ -public: - typedef int PIDImpl; - typedef std::vector ArgsImpl; - typedef std::map EnvImpl; - - static PIDImpl idImpl(); - static void timesImpl(long & userTime, long & kernelTime); - static ProcessHandleImpl * launchImpl( - const std::string & command, - const ArgsImpl & args, - const std::string & initialDirectory, - Pipe * inPipe, - Pipe * outPipe, - Pipe * errPipe, - const EnvImpl & env); - static void killImpl(ProcessHandleImpl & handle); - static void killImpl(PIDImpl pid); - static bool isRunningImpl(const ProcessHandleImpl & handle); - static bool isRunningImpl(PIDImpl pid); - static void requestTerminationImpl(PIDImpl pid); -}; - - -} // namespace Poco - - -#endif // Foundation_Process_UNIX_INCLUDED diff --git a/base/poco/Foundation/include/Poco/Process_WIN32.h b/base/poco/Foundation/include/Poco/Process_WIN32.h deleted file mode 100644 index a516260f64b..00000000000 --- a/base/poco/Foundation/include/Poco/Process_WIN32.h +++ /dev/null @@ -1,84 +0,0 @@ -// -// Process_WIN32.h -// -// Library: Foundation -// Package: Processes -// Module: Process -// -// Definition of the ProcessImpl class for WIN32. -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_Process_WIN32_INCLUDED -#define Foundation_Process_WIN32_INCLUDED - - -#include -#include -#include "Poco/Foundation.h" -#include "Poco/RefCountedObject.h" -#include "Poco/UnWindows.h" - - -namespace Poco -{ - - -class Pipe; - - -class Foundation_API ProcessHandleImpl : public RefCountedObject -{ -public: - ProcessHandleImpl(HANDLE _hProcess, UInt32 pid); - ~ProcessHandleImpl(); - - UInt32 id() const; - HANDLE process() const; - int wait() const; - void closeHandle(); - -private: - HANDLE _hProcess; - UInt32 _pid; - - ProcessHandleImpl(const ProcessHandleImpl &); - ProcessHandleImpl & operator=(const ProcessHandleImpl &); -}; - - -class Foundation_API ProcessImpl -{ -public: - typedef UInt32 PIDImpl; - typedef std::vector ArgsImpl; - typedef std::map EnvImpl; - - static PIDImpl idImpl(); - static void timesImpl(long & userTime, long & kernelTime); - static ProcessHandleImpl * launchImpl( - const std::string & command, - const ArgsImpl & args, - const std::string & initialDirectory, - Pipe * inPipe, - Pipe * outPipe, - Pipe * errPipe, - const EnvImpl & env); - static void killImpl(ProcessHandleImpl & handle); - static void killImpl(PIDImpl pid); - static bool isRunningImpl(const ProcessHandleImpl & handle); - static bool isRunningImpl(PIDImpl pid); - static void requestTerminationImpl(PIDImpl pid); - static std::string terminationEventName(PIDImpl pid); -}; - - -} // namespace Poco - - -#endif // Foundation_Process_WIN32_INCLUDED diff --git a/base/poco/Foundation/include/Poco/Process_WIN32U.h b/base/poco/Foundation/include/Poco/Process_WIN32U.h deleted file mode 100644 index e886b464858..00000000000 --- a/base/poco/Foundation/include/Poco/Process_WIN32U.h +++ /dev/null @@ -1,84 +0,0 @@ -// -// Process_WIN32U.h -// -// Library: Foundation -// Package: Processes -// Module: Process -// -// Definition of the ProcessImpl class for WIN32. -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_Process_WIN32U_INCLUDED -#define Foundation_Process_WIN32U_INCLUDED - - -#include -#include -#include "Poco/Foundation.h" -#include "Poco/RefCountedObject.h" -#include "Poco/UnWindows.h" - - -namespace Poco -{ - - -class Pipe; - - -class Foundation_API ProcessHandleImpl : public RefCountedObject -{ -public: - ProcessHandleImpl(HANDLE _hProcess, UInt32 pid); - ~ProcessHandleImpl(); - - UInt32 id() const; - HANDLE process() const; - int wait() const; - void closeHandle(); - -private: - HANDLE _hProcess; - UInt32 _pid; - - ProcessHandleImpl(const ProcessHandleImpl &); - ProcessHandleImpl & operator=(const ProcessHandleImpl &); -}; - - -class Foundation_API ProcessImpl -{ -public: - typedef UInt32 PIDImpl; - typedef std::vector ArgsImpl; - typedef std::map EnvImpl; - - static PIDImpl idImpl(); - static void timesImpl(long & userTime, long & kernelTime); - static ProcessHandleImpl * launchImpl( - const std::string & command, - const ArgsImpl & args, - const std::string & initialDirectory, - Pipe * inPipe, - Pipe * outPipe, - Pipe * errPipe, - const EnvImpl & env); - static void killImpl(ProcessHandleImpl & handle); - static void killImpl(PIDImpl pid); - static bool isRunningImpl(const ProcessHandleImpl & handle); - static bool isRunningImpl(PIDImpl pid); - static void requestTerminationImpl(PIDImpl pid); - static std::string terminationEventName(PIDImpl pid); -}; - - -} // namespace Poco - - -#endif // Foundation_Process_WIN32U_INCLUDED diff --git a/base/poco/Foundation/include/Poco/Process_WINCE.h b/base/poco/Foundation/include/Poco/Process_WINCE.h deleted file mode 100644 index 853fc8c3f9f..00000000000 --- a/base/poco/Foundation/include/Poco/Process_WINCE.h +++ /dev/null @@ -1,84 +0,0 @@ -// -// Process_WINCE.h -// -// Library: Foundation -// Package: Processes -// Module: Process -// -// Definition of the ProcessImpl class for WIN32. -// -// Copyright (c) 2004-2010, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_Process_WINCE_INCLUDED -#define Foundation_Process_WINCE_INCLUDED - - -#include -#include -#include "Poco/Foundation.h" -#include "Poco/RefCountedObject.h" -#include "Poco/UnWindows.h" - - -namespace Poco -{ - - -class Pipe; - - -class Foundation_API ProcessHandleImpl : public RefCountedObject -{ -public: - ProcessHandleImpl(HANDLE _hProcess, UInt32 pid); - ~ProcessHandleImpl(); - - UInt32 id() const; - HANDLE process() const; - int wait() const; - void closeHandle(); - -private: - HANDLE _hProcess; - UInt32 _pid; - - ProcessHandleImpl(const ProcessHandleImpl &); - ProcessHandleImpl & operator=(const ProcessHandleImpl &); -}; - - -class Foundation_API ProcessImpl -{ -public: - typedef UInt32 PIDImpl; - typedef std::vector ArgsImpl; - typedef std::map EnvImpl; - - static PIDImpl idImpl(); - static void timesImpl(long & userTime, long & kernelTime); - static ProcessHandleImpl * launchImpl( - const std::string & command, - const ArgsImpl & args, - const std::string & initialDirectory, - Pipe * inPipe, - Pipe * outPipe, - Pipe * errPipe, - const EnvImpl & env); - static void killImpl(ProcessHandleImpl & handle); - static void killImpl(PIDImpl pid); - static bool isRunningImpl(const ProcessHandleImpl & handle); - static bool isRunningImpl(PIDImpl pid); - static void requestTerminationImpl(PIDImpl pid); - static std::string terminationEventName(PIDImpl pid); -}; - - -} // namespace Poco - - -#endif // Foundation_Process_WINCE_INCLUDED diff --git a/base/poco/Foundation/include/Poco/RWLock_VX.h b/base/poco/Foundation/include/Poco/RWLock_VX.h deleted file mode 100644 index 1d31bdd0cde..00000000000 --- a/base/poco/Foundation/include/Poco/RWLock_VX.h +++ /dev/null @@ -1,91 +0,0 @@ -// -// RWLock_VX.h -// -// Library: Foundation -// Package: Threading -// Module: RWLock -// -// Definition of the RWLockImpl class for POSIX Threads (VxWorks). -// -// Copyright (c) 2004-2011, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_RWLock_VX_INCLUDED -#define Foundation_RWLock_VX_INCLUDED - - -#include -#include -#include "Poco/Exception.h" -#include "Poco/Foundation.h" - - -namespace Poco -{ - - -class Foundation_API RWLockImpl -{ -protected: - RWLockImpl(); - ~RWLockImpl(); - void readLockImpl(); - bool tryReadLockImpl(); - void writeLockImpl(); - bool tryWriteLockImpl(); - void unlockImpl(); - -private: - pthread_mutex_t _mutex; -}; - - -// -// inlines -// -inline void RWLockImpl::readLockImpl() -{ - if (pthread_mutex_lock(&_mutex)) - throw SystemException("cannot lock mutex"); -} - - -inline bool RWLockImpl::tryReadLockImpl() -{ - int rc = pthread_mutex_trylock(&_mutex); - if (rc == 0) - return true; - else if (rc == EBUSY) - return false; - else - throw SystemException("cannot lock mutex"); -} - - -inline void RWLockImpl::writeLockImpl() -{ - readLockImpl(); -} - - -inline bool RWLockImpl::tryWriteLockImpl() -{ - return tryReadLockImpl(); -} - - -inline void RWLockImpl::unlockImpl() -{ - if (pthread_mutex_unlock(&_mutex)) - throw SystemException("cannot unlock mutex"); -} - - -} // namespace Poco - - -#endif // Foundation_RWLock_VX_INCLUDED diff --git a/base/poco/Foundation/include/Poco/RWLock_WIN32.h b/base/poco/Foundation/include/Poco/RWLock_WIN32.h deleted file mode 100644 index 9d2e3823c3c..00000000000 --- a/base/poco/Foundation/include/Poco/RWLock_WIN32.h +++ /dev/null @@ -1,58 +0,0 @@ -// -// RWLock_WIN32.h -// -// Library: Foundation -// Package: Threading -// Module: RWLock -// -// Definition of the RWLockImpl class for WIN32. -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_RWLock_WIN32_INCLUDED -#define Foundation_RWLock_WIN32_INCLUDED - - -#include "Poco/Exception.h" -#include "Poco/Foundation.h" -#include "Poco/UnWindows.h" - - -namespace Poco -{ - - -class Foundation_API RWLockImpl -{ -protected: - RWLockImpl(); - ~RWLockImpl(); - void readLockImpl(); - bool tryReadLockImpl(); - void writeLockImpl(); - bool tryWriteLockImpl(); - void unlockImpl(); - -private: - void addWriter(); - void removeWriter(); - DWORD tryReadLockOnce(); - - HANDLE _mutex; - HANDLE _readEvent; - HANDLE _writeEvent; - unsigned _readers; - unsigned _writersWaiting; - unsigned _writers; -}; - - -} // namespace Poco - - -#endif // Foundation_RWLock_WIN32_INCLUDED diff --git a/base/poco/Foundation/include/Poco/RWLock_WINCE.h b/base/poco/Foundation/include/Poco/RWLock_WINCE.h deleted file mode 100644 index e7347f145a8..00000000000 --- a/base/poco/Foundation/include/Poco/RWLock_WINCE.h +++ /dev/null @@ -1,62 +0,0 @@ -// -// RWLock_WINCE.h -// -// Library: Foundation -// Package: Threading -// Module: RWLock -// -// Definition of the RWLockImpl class for WINCE. -// -// Copyright (c) 2009-2010, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_RWLock_WINCE_INCLUDED -#define Foundation_RWLock_WINCE_INCLUDED - - -#include "Poco/Exception.h" -#include "Poco/Foundation.h" -#include "Poco/UnWindows.h" - - -namespace Poco -{ - - -class Foundation_API RWLockImpl -/// This implementation is based on the one from Stone Steps Inc, -/// licensed under the BSD license. -/// http://forums.stonesteps.ca/thread.asp?t=105 -/// -/// Note that with this implementation, writers always take -/// precedence over readers. -{ -protected: - RWLockImpl(); - ~RWLockImpl(); - void readLockImpl(); - bool tryReadLockImpl(DWORD timeout = 1); - void writeLockImpl(); - bool tryWriteLockImpl(DWORD timeout = 1); - void unlockImpl(); - -private: - DWORD _readerCount; - DWORD _readerWaiting; - DWORD _writerCount; - DWORD _writerWaiting; - HANDLE _readerGreen; - HANDLE _writerGreen; - CRITICAL_SECTION _cs; - bool _writeLock; -}; - - -} // namespace Poco - - -#endif // Foundation_RWLock_WINCE_INCLUDED diff --git a/base/poco/Foundation/include/Poco/Semaphore_VX.h b/base/poco/Foundation/include/Poco/Semaphore_VX.h deleted file mode 100644 index 0b9e784fbaf..00000000000 --- a/base/poco/Foundation/include/Poco/Semaphore_VX.h +++ /dev/null @@ -1,57 +0,0 @@ -// -// Semaphore_VX.h -// -// Library: Foundation -// Package: Threading -// Module: Semaphore -// -// Definition of the SemaphoreImpl class for VxWorks. -// -// Copyright (c) 2004-20011, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_Semaphore_VX_INCLUDED -#define Foundation_Semaphore_VX_INCLUDED - - -#include -#include "Poco/Exception.h" -#include "Poco/Foundation.h" - - -namespace Poco -{ - - -class Foundation_API SemaphoreImpl -{ -protected: - SemaphoreImpl(int n, int max); - ~SemaphoreImpl(); - void setImpl(); - void waitImpl(); - bool waitImpl(long milliseconds); - -private: - SEM_ID _sem; -}; - - -// -// inlines -// -inline void SemaphoreImpl::setImpl() -{ - if (semGive(_sem) != OK) - throw SystemException("cannot signal semaphore"); -} - - -} // namespace Poco - - -#endif // Foundation_Semaphore_VX_INCLUDED diff --git a/base/poco/Foundation/include/Poco/Semaphore_WIN32.h b/base/poco/Foundation/include/Poco/Semaphore_WIN32.h deleted file mode 100644 index 399a350ef88..00000000000 --- a/base/poco/Foundation/include/Poco/Semaphore_WIN32.h +++ /dev/null @@ -1,59 +0,0 @@ -// -// Semaphore_WIN32.h -// -// Library: Foundation -// Package: Threading -// Module: Semaphore -// -// Definition of the SemaphoreImpl class for WIN32. -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_Semaphore_WIN32_INCLUDED -#define Foundation_Semaphore_WIN32_INCLUDED - - -#include "Poco/Exception.h" -#include "Poco/Foundation.h" -#include "Poco/UnWindows.h" - - -namespace Poco -{ - - -class Foundation_API SemaphoreImpl -{ -protected: - SemaphoreImpl(int n, int max); - ~SemaphoreImpl(); - void setImpl(); - void waitImpl(); - bool waitImpl(long milliseconds); - -private: - HANDLE _sema; -}; - - -// -// inlines -// -inline void SemaphoreImpl::setImpl() -{ - if (!ReleaseSemaphore(_sema, 1, NULL)) - { - throw SystemException("cannot signal semaphore"); - } -} - - -} // namespace Poco - - -#endif // Foundation_Semaphore_WIN32_INCLUDED diff --git a/base/poco/Foundation/include/Poco/SharedLibrary_HPUX.h b/base/poco/Foundation/include/Poco/SharedLibrary_HPUX.h deleted file mode 100644 index 63c6e639d01..00000000000 --- a/base/poco/Foundation/include/Poco/SharedLibrary_HPUX.h +++ /dev/null @@ -1,53 +0,0 @@ -// -// SharedLibrary_HPUX.h -// -// Library: Foundation -// Package: SharedLibrary -// Module: SharedLibrary -// -// Definition of the SharedLibraryImpl class for HP-UX. -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_SharedLibrary_HPUX_INCLUDED -#define Foundation_SharedLibrary_HPUX_INCLUDED - - -#include -#include "Poco/Foundation.h" -#include "Poco/Mutex.h" - - -namespace Poco -{ - - -class Foundation_API SharedLibraryImpl -{ -protected: - SharedLibraryImpl(); - ~SharedLibraryImpl(); - void loadImpl(const std::string & path, int flags); - void unloadImpl(); - bool isLoadedImpl() const; - void * findSymbolImpl(const std::string & name); - const std::string & getPathImpl() const; - static std::string suffixImpl(); - static bool setSearchPathImpl(const std::string & path); - -private: - std::string _path; - shl_t _handle; - static FastMutex _mutex; -}; - - -} // namespace Poco - - -#endif // Foundation_SharedLibrary_HPUX_INCLUDED diff --git a/base/poco/Foundation/include/Poco/SharedLibrary_VX.h b/base/poco/Foundation/include/Poco/SharedLibrary_VX.h deleted file mode 100644 index dc7cd1fe473..00000000000 --- a/base/poco/Foundation/include/Poco/SharedLibrary_VX.h +++ /dev/null @@ -1,53 +0,0 @@ -// -// SharedLibrary_VX.h -// -// Library: Foundation -// Package: SharedLibrary -// Module: SharedLibrary -// -// Definition of the SharedLibraryImpl class for VxWorks. -// -// Copyright (c) 2004-2011, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_SharedLibrary_VX_INCLUDED -#define Foundation_SharedLibrary_VX_INCLUDED - - -#include -#include "Poco/Foundation.h" -#include "Poco/Mutex.h" - - -namespace Poco -{ - - -class Foundation_API SharedLibraryImpl -{ -protected: - SharedLibraryImpl(); - ~SharedLibraryImpl(); - void loadImpl(const std::string & path, int flags); - void unloadImpl(); - bool isLoadedImpl() const; - void * findSymbolImpl(const std::string & name); - const std::string & getPathImpl() const; - static std::string suffixImpl(); - static bool setSearchPathImpl(const std::string & path); - -private: - std::string _path; - MODULE_ID _moduleId; - static FastMutex _mutex; -}; - - -} // namespace Poco - - -#endif // Foundation_SharedLibrary_VX_INCLUDED diff --git a/base/poco/Foundation/include/Poco/SharedLibrary_WIN32.h b/base/poco/Foundation/include/Poco/SharedLibrary_WIN32.h deleted file mode 100644 index 86fa35213d0..00000000000 --- a/base/poco/Foundation/include/Poco/SharedLibrary_WIN32.h +++ /dev/null @@ -1,52 +0,0 @@ -// -// SharedLibrary_WIN32.h -// -// Library: Foundation -// Package: SharedLibrary -// Module: SharedLibrary -// -// Definition of the SharedLibraryImpl class for Win32. -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_SharedLibrary_WIN32_INCLUDED -#define Foundation_SharedLibrary_WIN32_INCLUDED - - -#include "Poco/Foundation.h" -#include "Poco/Mutex.h" - - -namespace Poco -{ - - -class Foundation_API SharedLibraryImpl -{ -protected: - SharedLibraryImpl(); - ~SharedLibraryImpl(); - void loadImpl(const std::string & path, int flags); - void unloadImpl(); - bool isLoadedImpl() const; - void * findSymbolImpl(const std::string & name); - const std::string & getPathImpl() const; - static std::string suffixImpl(); - static bool setSearchPathImpl(const std::string & path); - -private: - std::string _path; - void * _handle; - static FastMutex _mutex; -}; - - -} // namespace Poco - - -#endif // Foundation_SharedLibrary_WIN32_INCLUDED diff --git a/base/poco/Foundation/include/Poco/SharedLibrary_WIN32U.h b/base/poco/Foundation/include/Poco/SharedLibrary_WIN32U.h deleted file mode 100644 index 03b85655efc..00000000000 --- a/base/poco/Foundation/include/Poco/SharedLibrary_WIN32U.h +++ /dev/null @@ -1,52 +0,0 @@ -// -// SharedLibrary_WIN32U.h -// -// Library: Foundation -// Package: SharedLibrary -// Module: SharedLibrary -// -// Definition of the SharedLibraryImpl class for Win32. -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_SharedLibrary_WIN32U_INCLUDED -#define Foundation_SharedLibrary_WIN32U_INCLUDED - - -#include "Poco/Foundation.h" -#include "Poco/Mutex.h" - - -namespace Poco -{ - - -class Foundation_API SharedLibraryImpl -{ -protected: - SharedLibraryImpl(); - ~SharedLibraryImpl(); - void loadImpl(const std::string & path, int flags); - void unloadImpl(); - bool isLoadedImpl() const; - void * findSymbolImpl(const std::string & name); - const std::string & getPathImpl() const; - static std::string suffixImpl(); - static bool setSearchPathImpl(const std::string & path); - -private: - std::string _path; - void * _handle; - static FastMutex _mutex; -}; - - -} // namespace Poco - - -#endif // Foundation_SharedLibrary_WIN32U_INCLUDED diff --git a/base/poco/Foundation/include/Poco/SharedMemory_WIN32.h b/base/poco/Foundation/include/Poco/SharedMemory_WIN32.h deleted file mode 100644 index ba90c874f68..00000000000 --- a/base/poco/Foundation/include/Poco/SharedMemory_WIN32.h +++ /dev/null @@ -1,104 +0,0 @@ -// -// SharedMemoryImpl.h -// -// Library: Foundation -// Package: Processes -// Module: SharedMemoryImpl -// -// Definition of the SharedMemoryImpl class. -// -// Copyright (c) 2007, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_SharedMemoryImpl_INCLUDED -#define Foundation_SharedMemoryImpl_INCLUDED - - -#include "Poco/Poco.h" -#include "Poco/RefCountedObject.h" -#include "Poco/SharedMemory.h" - - -namespace Poco -{ - - -class Foundation_API SharedMemoryImpl : public RefCountedObject -/// Shared memory implementation for Windows platforms. -{ -public: - SharedMemoryImpl(const std::string & name, std::size_t size, SharedMemory::AccessMode mode, const void * addrHint, bool server); - /// Creates or connects to a shared memory object with the given name. - /// - /// For maximum portability, name should be a valid Unix filename and not - /// contain any slashes or backslashes. - /// - /// An address hint can be passed to the system, specifying the desired - /// start address of the shared memory area. Whether the hint - /// is actually honored is, however, up to the system. Windows platform - /// will generally ignore the hint. - - SharedMemoryImpl(const Poco::File & file, SharedMemory::AccessMode mode, const void * addrHint); - /// Maps the entire contents of file into a shared memory segment. - /// - /// An address hint can be passed to the system, specifying the desired - /// start address of the shared memory area. Whether the hint - /// is actually honored is, however, up to the system. Windows platform - /// will generally ignore the hint. - - char * begin() const; - /// Returns the begin address of the SharedMemory segment. Will be null for illegal segments. - - char * end() const; - /// Points past the last byte of the end address of the SharedMemory segment. Will be null for illegal segments. - -protected: - void map(); - /// Maps the shared memory object. - - void unmap(); - /// Unmaps the shared memory object. - - void close(); - /// Releases the handle for the shared memory segment. - - ~SharedMemoryImpl(); - /// Destroys the SharedMemoryImpl. - -private: - SharedMemoryImpl(); - SharedMemoryImpl(const SharedMemoryImpl &); - SharedMemoryImpl & operator=(const SharedMemoryImpl &); - - std::string _name; - HANDLE _memHandle; - HANDLE _fileHandle; - DWORD _size; - DWORD _mode; - char * _address; -}; - - -// -// inlines -// -inline char * SharedMemoryImpl::begin() const -{ - return _address; -} - - -inline char * SharedMemoryImpl::end() const -{ - return _address + _size; -} - - -} // namespace Poco - - -#endif // Foundation_SharedMemoryImpl_INCLUDED diff --git a/base/poco/Foundation/include/Poco/Thread_VX.h b/base/poco/Foundation/include/Poco/Thread_VX.h deleted file mode 100644 index 381a4932762..00000000000 --- a/base/poco/Foundation/include/Poco/Thread_VX.h +++ /dev/null @@ -1,167 +0,0 @@ -// -// Thread_VX.h -// -// Library: Foundation -// Package: Threading -// Module: Thread -// -// Definition of the ThreadImpl class for VxWorks tasks. -// -// Copyright (c) 2004-2011, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_Thread_VX_INCLUDED -#define Foundation_Thread_VX_INCLUDED - - -#include -#include -#include "Poco/AutoPtr.h" -#include "Poco/Event.h" -#include "Poco/Foundation.h" -#include "Poco/RefCountedObject.h" -#include "Poco/Runnable.h" -#include "Poco/SignalHandler.h" - - -namespace Poco -{ - - -class Foundation_API ThreadImpl -{ -public: - typedef int TIDImpl; - typedef void (*Callable)(void *); - - enum Priority - { - PRIO_LOWEST_IMPL, - PRIO_LOW_IMPL, - PRIO_NORMAL_IMPL, - PRIO_HIGH_IMPL, - PRIO_HIGHEST_IMPL - }; - - enum Policy - { - POLICY_DEFAULT_IMPL = 0 - }; - - enum - { - DEFAULT_THREAD_STACK_SIZE = 65536 - }; - - struct CallbackData : public RefCountedObject - { - CallbackData() : callback(0), pData(0) { } - - Callable callback; - void * pData; - }; - - ThreadImpl(); - ~ThreadImpl(); - - TIDImpl tidImpl() const; - void setPriorityImpl(int prio); - int getPriorityImpl() const; - void setOSPriorityImpl(int prio, int policy = 0); - int getOSPriorityImpl() const; - static int getMinOSPriorityImpl(int policy); - static int getMaxOSPriorityImpl(int policy); - void setStackSizeImpl(int size); - int getStackSizeImpl() const; - void startImpl(Runnable & target); - void startImpl(Callable target, void * pData = 0); - - void joinImpl(); - bool joinImpl(long milliseconds); - bool isRunningImpl() const; - static void sleepImpl(long milliseconds); - static void yieldImpl(); - static ThreadImpl * currentImpl(); - static TIDImpl currentTidImpl(); - -protected: - static void runnableEntry(void * pThread, int, int, int, int, int, int, int, int, int); - static void callableEntry(void * pThread, int, int, int, int, int, int, int, int, int); - static int mapPrio(int prio); - static int reverseMapPrio(int osPrio); - - struct ThreadData : public RefCountedObject - { - ThreadData() - : pRunnableTarget(0) - , pCallbackTarget(0) - , task(0) - , prio(PRIO_NORMAL_IMPL) - , osPrio(127) - , done(false) - , stackSize(POCO_THREAD_STACK_SIZE) - { - } - - Runnable * pRunnableTarget; - AutoPtr pCallbackTarget; - int task; - int prio; - int osPrio; - Event done; - int stackSize; - }; - -private: - AutoPtr _pData; - static ThreadImpl * _pCurrent; -}; - - -// -// inlines -// -inline int ThreadImpl::getPriorityImpl() const -{ - return _pData->prio; -} - - -inline int ThreadImpl::getOSPriorityImpl() const -{ - return _pData->osPrio; -} - - -inline bool ThreadImpl::isRunningImpl() const -{ - return _pData->pRunnableTarget != 0 || (_pData->pCallbackTarget.get() != 0 && _pData->pCallbackTarget->callback != 0); -} - - -inline void ThreadImpl::yieldImpl() -{ - taskDelay(0); -} - - -inline int ThreadImpl::getStackSizeImpl() const -{ - return _pData->stackSize; -} - - -inline ThreadImpl::TIDImpl ThreadImpl::tidImpl() const -{ - return _pData->task; -} - - -} // namespace Poco - - -#endif // Foundation_Thread_VX_INCLUDED diff --git a/base/poco/Foundation/include/Poco/Thread_WIN32.h b/base/poco/Foundation/include/Poco/Thread_WIN32.h deleted file mode 100644 index c5f4b6ffcc6..00000000000 --- a/base/poco/Foundation/include/Poco/Thread_WIN32.h +++ /dev/null @@ -1,175 +0,0 @@ -// -// Thread_WIN32.h -// -// Library: Foundation -// Package: Threading -// Module: Thread -// -// Definition of the ThreadImpl class for WIN32. -// -// Copyright (c) 2004-2009, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_Thread_WIN32_INCLUDED -#define Foundation_Thread_WIN32_INCLUDED - - -#include "Poco/Foundation.h" -#include "Poco/Runnable.h" -#include "Poco/SharedPtr.h" -#include "Poco/UnWindows.h" - - -namespace Poco -{ - - -class Foundation_API ThreadImpl -{ -public: - typedef DWORD TIDImpl; - typedef void (*Callable)(void *); - -#if defined(_DLL) - typedef DWORD(WINAPI * Entry)(LPVOID); -#else - typedef unsigned(__stdcall * Entry)(void *); -#endif - - enum Priority - { - PRIO_LOWEST_IMPL = THREAD_PRIORITY_LOWEST, - PRIO_LOW_IMPL = THREAD_PRIORITY_BELOW_NORMAL, - PRIO_NORMAL_IMPL = THREAD_PRIORITY_NORMAL, - PRIO_HIGH_IMPL = THREAD_PRIORITY_ABOVE_NORMAL, - PRIO_HIGHEST_IMPL = THREAD_PRIORITY_HIGHEST - }; - - enum Policy - { - POLICY_DEFAULT_IMPL = 0 - }; - - ThreadImpl(); - ~ThreadImpl(); - - TIDImpl tidImpl() const; - void setPriorityImpl(int prio); - int getPriorityImpl() const; - void setOSPriorityImpl(int prio, int policy = 0); - int getOSPriorityImpl() const; - static int getMinOSPriorityImpl(int policy); - static int getMaxOSPriorityImpl(int policy); - void setStackSizeImpl(int size); - int getStackSizeImpl() const; - void startImpl(SharedPtr pTarget); - void joinImpl(); - bool joinImpl(long milliseconds); - bool isRunningImpl() const; - static void sleepImpl(long milliseconds); - static void yieldImpl(); - static ThreadImpl * currentImpl(); - static TIDImpl currentTidImpl(); - -protected: -#if defined(_DLL) - static DWORD WINAPI runnableEntry(LPVOID pThread); -#else - static unsigned __stdcall runnableEntry(void * pThread); -#endif - - void createImpl(Entry ent, void * pData); - void threadCleanup(); - -private: - class CurrentThreadHolder - { - public: - CurrentThreadHolder() : _slot(TlsAlloc()) - { - if (_slot == TLS_OUT_OF_INDEXES) - throw SystemException("cannot allocate thread context key"); - } - ~CurrentThreadHolder() { TlsFree(_slot); } - ThreadImpl * get() const { return reinterpret_cast(TlsGetValue(_slot)); } - void set(ThreadImpl * pThread) { TlsSetValue(_slot, pThread); } - - private: - DWORD _slot; - }; - - SharedPtr _pRunnableTarget; - HANDLE _thread; - DWORD _threadId; - int _prio; - int _stackSize; - - static CurrentThreadHolder _currentThreadHolder; -}; - - -// -// inlines -// -inline int ThreadImpl::getPriorityImpl() const -{ - return _prio; -} - - -inline int ThreadImpl::getOSPriorityImpl() const -{ - return _prio; -} - - -inline int ThreadImpl::getMinOSPriorityImpl(int /* policy */) -{ - return PRIO_LOWEST_IMPL; -} - - -inline int ThreadImpl::getMaxOSPriorityImpl(int /* policy */) -{ - return PRIO_HIGHEST_IMPL; -} - - -inline void ThreadImpl::sleepImpl(long milliseconds) -{ - Sleep(DWORD(milliseconds)); -} - - -inline void ThreadImpl::yieldImpl() -{ - Sleep(0); -} - - -inline void ThreadImpl::setStackSizeImpl(int size) -{ - _stackSize = size; -} - - -inline int ThreadImpl::getStackSizeImpl() const -{ - return _stackSize; -} - - -inline ThreadImpl::TIDImpl ThreadImpl::tidImpl() const -{ - return _threadId; -} - - -} // namespace Poco - - -#endif // Foundation_Thread_WIN32_INCLUDED diff --git a/base/poco/Foundation/include/Poco/Thread_WINCE.h b/base/poco/Foundation/include/Poco/Thread_WINCE.h deleted file mode 100644 index 5e5d64c1f4f..00000000000 --- a/base/poco/Foundation/include/Poco/Thread_WINCE.h +++ /dev/null @@ -1,171 +0,0 @@ -// -// Thread_WINCE.h -// -// Library: Foundation -// Package: Threading -// Module: Thread -// -// Definition of the ThreadImpl class for WIN32. -// -// Copyright (c) 2004-2010, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Foundation_Thread_WINCE_INCLUDED -#define Foundation_Thread_WINCE_INCLUDED - - -#include "Poco/Foundation.h" -#include "Poco/Runnable.h" -#include "Poco/SharedPtr.h" -#include "Poco/UnWindows.h" - - -#if !defined(TLS_OUT_OF_INDEXES) // Windows CE 5.x does not define this -# define TLS_OUT_OF_INDEXES 0xFFFFFFFF -#endif - - -namespace Poco -{ - - -class Foundation_API ThreadImpl -{ -public: - typedef DWORD TIDImpl; - typedef void (*Callable)(void *); - typedef DWORD(WINAPI * Entry)(LPVOID); - - enum Priority - { - PRIO_LOWEST_IMPL = THREAD_PRIORITY_LOWEST, - PRIO_LOW_IMPL = THREAD_PRIORITY_BELOW_NORMAL, - PRIO_NORMAL_IMPL = THREAD_PRIORITY_NORMAL, - PRIO_HIGH_IMPL = THREAD_PRIORITY_ABOVE_NORMAL, - PRIO_HIGHEST_IMPL = THREAD_PRIORITY_HIGHEST - }; - - enum Policy - { - POLICY_DEFAULT_IMPL = 0 - }; - - ThreadImpl(); - ~ThreadImpl(); - - TIDImpl tidImpl() const; - void setPriorityImpl(int prio); - int getPriorityImpl() const; - void setOSPriorityImpl(int prio, int policy = 0); - int getOSPriorityImpl() const; - static int getMinOSPriorityImpl(int policy); - static int getMaxOSPriorityImpl(int policy); - void setStackSizeImpl(int size); - int getStackSizeImpl() const; - void startImpl(SharedPtr pTarget); - void joinImpl(); - bool joinImpl(long milliseconds); - bool isRunningImpl() const; - static void sleepImpl(long milliseconds); - static void yieldImpl(); - static ThreadImpl * currentImpl(); - static TIDImpl currentTidImpl(); - -protected: - static DWORD WINAPI runnableEntry(LPVOID pThread); - - void createImpl(Entry ent, void * pData); - void threadCleanup(); - -private: - class CurrentThreadHolder - { - public: - CurrentThreadHolder() : _slot(TlsAlloc()) - { - if (_slot == TLS_OUT_OF_INDEXES) - throw SystemException("cannot allocate thread context key"); - } - ~CurrentThreadHolder() { TlsFree(_slot); } - ThreadImpl * get() const { return reinterpret_cast(TlsGetValue(_slot)); } - void set(ThreadImpl * pThread) { TlsSetValue(_slot, pThread); } - - private: - DWORD _slot; - }; - - SharedPtr _pRunnableTarget; - HANDLE _thread; - DWORD _threadId; - int _prio; - int _stackSize; - - static CurrentThreadHolder _currentThreadHolder; -}; - - -// -// inlines -// -inline int ThreadImpl::getPriorityImpl() const -{ - return _prio; -} - - -inline int ThreadImpl::getOSPriorityImpl() const -{ - return _prio; -} - - -inline int ThreadImpl::getMinOSPriorityImpl(int /* policy */) -{ - return PRIO_LOWEST_IMPL; -} - - -inline int ThreadImpl::getMaxOSPriorityImpl(int /* policy */) -{ - return PRIO_HIGHEST_IMPL; -} - - -inline void ThreadImpl::sleepImpl(long milliseconds) -{ - Sleep(DWORD(milliseconds)); -} - - -inline void ThreadImpl::yieldImpl() -{ - Sleep(0); -} - - -inline void ThreadImpl::setStackSizeImpl(int size) -{ - _stackSize = size; -} - - -inline int ThreadImpl::getStackSizeImpl() const -{ - return _stackSize; -} - - -inline ThreadImpl::TIDImpl ThreadImpl::tidImpl() const -{ - return _threadId; -} - - -} // namespace Poco - - -#endif // Foundation_Thread_WINCE_INCLUDED diff --git a/base/poco/Foundation/src/DirectoryIterator_WIN32.cpp b/base/poco/Foundation/src/DirectoryIterator_WIN32.cpp deleted file mode 100644 index e7ebfc859b0..00000000000 --- a/base/poco/Foundation/src/DirectoryIterator_WIN32.cpp +++ /dev/null @@ -1,66 +0,0 @@ -// -// DirectoryIterator_WIN32.cpp -// -// Library: Foundation -// Package: Filesystem -// Module: DirectoryIterator -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/DirectoryIterator_WIN32.h" -#include "Poco/File_WIN32.h" -#include "Poco/Path.h" - - -namespace Poco { - - -DirectoryIteratorImpl::DirectoryIteratorImpl(const std::string& path): _fh(INVALID_HANDLE_VALUE), _rc(1) -{ - Path p(path); - p.makeDirectory(); - std::string findPath = p.toString(); - findPath.append("*"); - - _fh = FindFirstFile(findPath.c_str(), &_fd); - if (_fh == INVALID_HANDLE_VALUE) - { - if (GetLastError() != ERROR_NO_MORE_FILES) - File::handleLastError(path); - } - else - { - _current = _fd.cFileName; - if (_current == "." || _current == "..") - next(); - } -} - - -DirectoryIteratorImpl::~DirectoryIteratorImpl() -{ - if (_fh != INVALID_HANDLE_VALUE) - FindClose(_fh); -} - - -const std::string& DirectoryIteratorImpl::next() -{ - do - { - if (FindNextFile(_fh, &_fd) != 0) - _current = _fd.cFileName; - else - _current.clear(); - } - while (_current == "." || _current == ".."); - return _current; -} - - -} // namespace Poco diff --git a/base/poco/Foundation/src/DirectoryIterator_WIN32U.cpp b/base/poco/Foundation/src/DirectoryIterator_WIN32U.cpp deleted file mode 100644 index e72136bd478..00000000000 --- a/base/poco/Foundation/src/DirectoryIterator_WIN32U.cpp +++ /dev/null @@ -1,71 +0,0 @@ -// -// DirectoryIterator_WIN32U.cpp -// -// Library: Foundation -// Package: Filesystem -// Module: DirectoryIterator -// -// Copyright (c) 2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/DirectoryIterator_WIN32U.h" -#include "Poco/File_WIN32U.h" -#include "Poco/Path.h" -#include "Poco/UnicodeConverter.h" -#include - - -namespace Poco { - - -DirectoryIteratorImpl::DirectoryIteratorImpl(const std::string& path): _fh(INVALID_HANDLE_VALUE), _rc(1) -{ - Path p(path); - p.makeDirectory(); - std::string findPath = p.toString(); - findPath.append("*"); - std::wstring uFindPath; - FileImpl::convertPath(findPath, uFindPath); - - _fh = FindFirstFileW(uFindPath.c_str(), &_fd); - if (_fh == INVALID_HANDLE_VALUE) - { - if (GetLastError() != ERROR_NO_MORE_FILES) - File::handleLastError(path); - } - else - { - UnicodeConverter::toUTF8(_fd.cFileName, _current); - if (_current == "." || _current == "..") - next(); - } -} - - -DirectoryIteratorImpl::~DirectoryIteratorImpl() -{ - if (_fh != INVALID_HANDLE_VALUE) - FindClose(_fh); -} - - -const std::string& DirectoryIteratorImpl::next() -{ - do - { - _current.clear(); - if (FindNextFileW(_fh, &_fd) != 0) - { - UnicodeConverter::toUTF8(_fd.cFileName, _current); - } - } - while (_current == "." || _current == ".."); - return _current; -} - - -} // namespace Poco diff --git a/base/poco/Foundation/src/Environment_VX.cpp b/base/poco/Foundation/src/Environment_VX.cpp deleted file mode 100644 index 44dc6907688..00000000000 --- a/base/poco/Foundation/src/Environment_VX.cpp +++ /dev/null @@ -1,157 +0,0 @@ - -// Environment_VX.cpp -// -// Library: Foundation -// Package: Core -// Module: Environment -// -// Copyright (c) 2004-2011, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Environment_VX.h" -#include "Poco/Exception.h" -#include "Poco/Buffer.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -namespace Poco { - - -EnvironmentImpl::StringMap EnvironmentImpl::_map; -FastMutex EnvironmentImpl::_mutex; - - -std::string EnvironmentImpl::getImpl(const std::string& name) -{ - FastMutex::ScopedLock lock(_mutex); - - const char* val = getenv(name.c_str()); - if (val) - return std::string(val); - else - throw NotFoundException(name); -} - - -bool EnvironmentImpl::hasImpl(const std::string& name) -{ - FastMutex::ScopedLock lock(_mutex); - - return getenv(name.c_str()) != 0; -} - - -void EnvironmentImpl::setImpl(const std::string& name, const std::string& value) -{ - FastMutex::ScopedLock lock(_mutex); - - std::string var = name; - var.append("="); - var.append(value); - std::swap(_map[name], var); - if (putenv((char*) _map[name].c_str())) - { - std::string msg = "cannot set environment variable: "; - msg.append(name); - throw SystemException(msg); - } -} - - -std::string EnvironmentImpl::osNameImpl() -{ - return runtimeName; -} - - -std::string EnvironmentImpl::osDisplayNameImpl() -{ - return osNameImpl(); -} - - -std::string EnvironmentImpl::osVersionImpl() -{ - return runtimeVersion; -} - - -std::string EnvironmentImpl::osArchitectureImpl() -{ -#if POCO_ARCH == POCO_ARCH_IA32 - return "i386"; -#elif POCO_ARCH == POCO_ARCH_MIPS - return "mips"; -#elif POCO_ARCH == POCO_ARCH_PPC - return "ppc"; -#elif POCO_ARCH == POCO_ARCH_ARM - return "arm"; -#elif POCO_ARCH == POCO_ARCH_SH - return "sh"; -#else - return "unknown"; -#endif -} - - -std::string EnvironmentImpl::nodeNameImpl() -{ - char buffer[64]; - if (gethostname(buffer, sizeof(buffer)) == OK) - return buffer; - else - return "unknown"; -} - - -unsigned EnvironmentImpl::processorCountImpl() -{ - return 1; -} - - -void EnvironmentImpl::nodeIdImpl(NodeId& id) -{ - std::memset(&id, 0, sizeof(id)); - - int ifIndex = 1; - char ifName[32]; - for (;;) - { - if (ifIndexToIfName(ifIndex, ifName) == OK) - { - struct ifnet* pIf = ifunit(ifName); - if (pIf) - { - std::memcpy(&id, ((struct arpcom *) pIf)->ac_enaddr, sizeof(id)); - return; - } - } - else break; - ++ifIndex; - } - throw SystemException("cannot get Ethernet hardware address"); -} - - -} // namespace Poco diff --git a/base/poco/Foundation/src/Environment_WIN32.cpp b/base/poco/Foundation/src/Environment_WIN32.cpp deleted file mode 100644 index 62845fab029..00000000000 --- a/base/poco/Foundation/src/Environment_WIN32.cpp +++ /dev/null @@ -1,221 +0,0 @@ -// -// Environment_WIN32.cpp -// -// Library: Foundation -// Package: Core -// Module: Environment -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Environment_WIN32.h" -#include "Poco/Exception.h" -#include -#include -#include "Poco/UnWindows.h" -#include -#include -#include -#include - - -namespace Poco { - - -std::string EnvironmentImpl::getImpl(const std::string& name) -{ - DWORD len = GetEnvironmentVariableA(name.c_str(), 0, 0); - if (len == 0) throw NotFoundException(name); - char* buffer = new char[len]; - GetEnvironmentVariableA(name.c_str(), buffer, len); - std::string result(buffer); - delete [] buffer; - return result; -} - - -bool EnvironmentImpl::hasImpl(const std::string& name) -{ - DWORD len = GetEnvironmentVariableA(name.c_str(), 0, 0); - return len > 0; -} - - -void EnvironmentImpl::setImpl(const std::string& name, const std::string& value) -{ - if (SetEnvironmentVariableA(name.c_str(), value.c_str()) == 0) - { - std::string msg = "cannot set environment variable: "; - msg.append(name); - throw SystemException(msg); - } -} - - -std::string EnvironmentImpl::osNameImpl() -{ - OSVERSIONINFO vi; - vi.dwOSVersionInfoSize = sizeof(vi); - if (GetVersionEx(&vi) == 0) throw SystemException("Cannot get OS version information"); - switch (vi.dwPlatformId) - { - case VER_PLATFORM_WIN32s: - return "Windows 3.x"; - case VER_PLATFORM_WIN32_WINDOWS: - return vi.dwMinorVersion == 0 ? "Windows 95" : "Windows 98"; - case VER_PLATFORM_WIN32_NT: - return "Windows NT"; - default: - return "Unknown"; - } -} - - -std::string EnvironmentImpl::osDisplayNameImpl() -{ - OSVERSIONINFOEX vi; // OSVERSIONINFOEX is supported starting at Windows 2000 - vi.dwOSVersionInfoSize = sizeof(vi); - if (GetVersionEx((OSVERSIONINFO*) &vi) == 0) throw SystemException("Cannot get OS version information"); - switch (vi.dwMajorVersion) - { - case 10: - switch (vi.dwMinorVersion) - { - case 0: - return vi.wProductType == VER_NT_WORKSTATION ? "Windows 10" : "Windows Server 2016"; - } - case 6: - switch (vi.dwMinorVersion) - { - case 0: - return vi.wProductType == VER_NT_WORKSTATION ? "Windows Vista" : "Windows Server 2008"; - case 1: - return vi.wProductType == VER_NT_WORKSTATION ? "Windows 7" : "Windows Server 2008 R2"; - case 2: - return vi.wProductType == VER_NT_WORKSTATION ? "Windows 8" : "Windows Server 2012"; - case 3: - return vi.wProductType == VER_NT_WORKSTATION ? "Windows 8.1" : "Windows Server 2012 R2"; - default: - return "Unknown"; - } - case 5: - switch (vi.dwMinorVersion) - { - case 0: - return "Windows 2000"; - case 1: - return "Windows XP"; - case 2: - return "Windows Server 2003/Windows Server 2003 R2"; - default: - return "Unknown"; - } - default: - return "Unknown"; - } -} - - -std::string EnvironmentImpl::osVersionImpl() -{ - OSVERSIONINFO vi; - vi.dwOSVersionInfoSize = sizeof(vi); - if (GetVersionEx(&vi) == 0) throw SystemException("Cannot get OS version information"); - std::ostringstream str; - str << vi.dwMajorVersion << "." << vi.dwMinorVersion << " (Build " << (vi.dwBuildNumber & 0xFFFF); - if (vi.szCSDVersion[0]) str << ": " << vi.szCSDVersion; - str << ")"; - return str.str(); -} - - -std::string EnvironmentImpl::osArchitectureImpl() -{ - SYSTEM_INFO si; - GetSystemInfo(&si); - switch (si.wProcessorArchitecture) - { - case PROCESSOR_ARCHITECTURE_INTEL: - return "IA32"; - case PROCESSOR_ARCHITECTURE_MIPS: - return "MIPS"; - case PROCESSOR_ARCHITECTURE_ALPHA: - return "ALPHA"; - case PROCESSOR_ARCHITECTURE_PPC: - return "PPC"; - case PROCESSOR_ARCHITECTURE_IA64: - return "IA64"; -#ifdef PROCESSOR_ARCHITECTURE_IA32_ON_WIN64 - case PROCESSOR_ARCHITECTURE_IA32_ON_WIN64: - return "IA64/32"; -#endif -#ifdef PROCESSOR_ARCHITECTURE_AMD64 - case PROCESSOR_ARCHITECTURE_AMD64: - return "AMD64"; -#endif - default: - return "Unknown"; - } -} - - -std::string EnvironmentImpl::nodeNameImpl() -{ - char name[MAX_COMPUTERNAME_LENGTH + 1]; - DWORD size = sizeof(name); - if (GetComputerNameA(name, &size) == 0) throw SystemException("Cannot get computer name"); - return std::string(name); -} - - -void EnvironmentImpl::nodeIdImpl(NodeId& id) -{ - std::memset(&id, 0, sizeof(id)); - - PIP_ADAPTER_INFO pAdapterInfo; - PIP_ADAPTER_INFO pAdapter = 0; - ULONG len = sizeof(IP_ADAPTER_INFO); - pAdapterInfo = reinterpret_cast(new char[len]); - // Make an initial call to GetAdaptersInfo to get - // the necessary size into len - DWORD rc = GetAdaptersInfo(pAdapterInfo, &len); - if (rc == ERROR_BUFFER_OVERFLOW) - { - delete [] reinterpret_cast(pAdapterInfo); - pAdapterInfo = reinterpret_cast(new char[len]); - } - else if (rc != ERROR_SUCCESS) - { - return; - } - if (GetAdaptersInfo(pAdapterInfo, &len) == NO_ERROR) - { - pAdapter = pAdapterInfo; - bool found = false; - while (pAdapter && !found) - { - if (pAdapter->Type == MIB_IF_TYPE_ETHERNET && pAdapter->AddressLength == sizeof(id)) - { - found = true; - std::memcpy(&id, pAdapter->Address, pAdapter->AddressLength); - } - pAdapter = pAdapter->Next; - } - } - delete [] reinterpret_cast(pAdapterInfo); -} - - -unsigned EnvironmentImpl::processorCountImpl() -{ - SYSTEM_INFO si; - GetSystemInfo(&si); - return si.dwNumberOfProcessors; -} - - -} // namespace Poco diff --git a/base/poco/Foundation/src/Environment_WIN32U.cpp b/base/poco/Foundation/src/Environment_WIN32U.cpp deleted file mode 100644 index 624ff83c8ce..00000000000 --- a/base/poco/Foundation/src/Environment_WIN32U.cpp +++ /dev/null @@ -1,235 +0,0 @@ -// -// Environment_WIN32U.cpp -// -// Library: Foundation -// Package: Core -// Module: Environment -// -// Copyright (c) 2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Environment_WIN32U.h" -#include "Poco/Exception.h" -#include "Poco/UnicodeConverter.h" -#include "Poco/Buffer.h" -#include -#include -#include "Poco/UnWindows.h" -#include -#include -#include -#include - - -namespace Poco { - - -std::string EnvironmentImpl::getImpl(const std::string& name) -{ - std::wstring uname; - UnicodeConverter::toUTF16(name, uname); - DWORD len = GetEnvironmentVariableW(uname.c_str(), 0, 0); - if (len == 0) throw NotFoundException(name); - Buffer buffer(len); - GetEnvironmentVariableW(uname.c_str(), buffer.begin(), len); - std::string result; - UnicodeConverter::toUTF8(buffer.begin(), len - 1, result); - return result; -} - - -bool EnvironmentImpl::hasImpl(const std::string& name) -{ - std::wstring uname; - UnicodeConverter::toUTF16(name, uname); - DWORD len = GetEnvironmentVariableW(uname.c_str(), 0, 0); - return len > 0; -} - - -void EnvironmentImpl::setImpl(const std::string& name, const std::string& value) -{ - std::wstring uname; - std::wstring uvalue; - UnicodeConverter::toUTF16(name, uname); - UnicodeConverter::toUTF16(value, uvalue); - if (SetEnvironmentVariableW(uname.c_str(), uvalue.c_str()) == 0) - { - std::string msg = "cannot set environment variable: "; - msg.append(name); - throw SystemException(msg); - } -} - - -std::string EnvironmentImpl::osNameImpl() -{ - OSVERSIONINFO vi; - vi.dwOSVersionInfoSize = sizeof(vi); - if (GetVersionEx(&vi) == 0) throw SystemException("Cannot get OS version information"); - switch (vi.dwPlatformId) - { - case VER_PLATFORM_WIN32s: - return "Windows 3.x"; - case VER_PLATFORM_WIN32_WINDOWS: - return vi.dwMinorVersion == 0 ? "Windows 95" : "Windows 98"; - case VER_PLATFORM_WIN32_NT: - return "Windows NT"; - default: - return "Unknown"; - } -} - - -std::string EnvironmentImpl::osDisplayNameImpl() -{ - OSVERSIONINFOEX vi; // OSVERSIONINFOEX is supported starting at Windows 2000 - vi.dwOSVersionInfoSize = sizeof(vi); - if (GetVersionEx((OSVERSIONINFO*) &vi) == 0) throw SystemException("Cannot get OS version information"); - switch (vi.dwMajorVersion) - { - case 10: - switch (vi.dwMinorVersion) - { - case 0: - return vi.wProductType == VER_NT_WORKSTATION ? "Windows 10" : "Windows Server 2016"; - } - case 6: - switch (vi.dwMinorVersion) - { - case 0: - return vi.wProductType == VER_NT_WORKSTATION ? "Windows Vista" : "Windows Server 2008"; - case 1: - return vi.wProductType == VER_NT_WORKSTATION ? "Windows 7" : "Windows Server 2008 R2"; - case 2: - return vi.wProductType == VER_NT_WORKSTATION ? "Windows 8" : "Windows Server 2012"; - case 3: - return vi.wProductType == VER_NT_WORKSTATION ? "Windows 8.1" : "Windows Server 2012 R2"; - default: - return "Unknown"; - } - case 5: - switch (vi.dwMinorVersion) - { - case 0: - return "Windows 2000"; - case 1: - return "Windows XP"; - case 2: - return "Windows Server 2003/Windows Server 2003 R2"; - default: - return "Unknown"; - } - default: - return "Unknown"; - } -} - - -std::string EnvironmentImpl::osVersionImpl() -{ - OSVERSIONINFOW vi; - vi.dwOSVersionInfoSize = sizeof(vi); - if (GetVersionExW(&vi) == 0) throw SystemException("Cannot get OS version information"); - std::ostringstream str; - str << vi.dwMajorVersion << "." << vi.dwMinorVersion << " (Build " << (vi.dwBuildNumber & 0xFFFF); - std::string version; - UnicodeConverter::toUTF8(vi.szCSDVersion, version); - if (!version.empty()) str << ": " << version; - str << ")"; - return str.str(); -} - - -std::string EnvironmentImpl::osArchitectureImpl() -{ - SYSTEM_INFO si; - GetSystemInfo(&si); - switch (si.wProcessorArchitecture) - { - case PROCESSOR_ARCHITECTURE_INTEL: - return "IA32"; - case PROCESSOR_ARCHITECTURE_MIPS: - return "MIPS"; - case PROCESSOR_ARCHITECTURE_ALPHA: - return "ALPHA"; - case PROCESSOR_ARCHITECTURE_PPC: - return "PPC"; - case PROCESSOR_ARCHITECTURE_IA64: - return "IA64"; -#ifdef PROCESSOR_ARCHITECTURE_IA32_ON_WIN64 - case PROCESSOR_ARCHITECTURE_IA32_ON_WIN64: - return "IA64/32"; -#endif -#ifdef PROCESSOR_ARCHITECTURE_AMD64 - case PROCESSOR_ARCHITECTURE_AMD64: - return "AMD64"; -#endif - default: - return "Unknown"; - } -} - - -std::string EnvironmentImpl::nodeNameImpl() -{ - wchar_t name[MAX_COMPUTERNAME_LENGTH + 1]; - DWORD size = MAX_COMPUTERNAME_LENGTH + 1; - if (GetComputerNameW(name, &size) == 0) throw SystemException("Cannot get computer name"); - std::string result; - UnicodeConverter::toUTF8(name, result); - return result; -} - - -void EnvironmentImpl::nodeIdImpl(NodeId& id) -{ - std::memset(&id, 0, sizeof(id)); - - PIP_ADAPTER_INFO pAdapterInfo; - PIP_ADAPTER_INFO pAdapter = 0; - ULONG len = sizeof(IP_ADAPTER_INFO); - pAdapterInfo = reinterpret_cast(new char[len]); - // Make an initial call to GetAdaptersInfo to get - // the necessary size into len - DWORD rc = GetAdaptersInfo(pAdapterInfo, &len); - if (rc == ERROR_BUFFER_OVERFLOW) - { - delete [] reinterpret_cast(pAdapterInfo); - pAdapterInfo = reinterpret_cast(new char[len]); - } - else if (rc != ERROR_SUCCESS) - { - return; - } - if (GetAdaptersInfo(pAdapterInfo, &len) == NO_ERROR) - { - pAdapter = pAdapterInfo; - bool found = false; - while (pAdapter && !found) - { - if (pAdapter->Type == MIB_IF_TYPE_ETHERNET && pAdapter->AddressLength == sizeof(id)) - { - found = true; - std::memcpy(&id, pAdapter->Address, pAdapter->AddressLength); - } - pAdapter = pAdapter->Next; - } - } - delete [] reinterpret_cast(pAdapterInfo); -} - - -unsigned EnvironmentImpl::processorCountImpl() -{ - SYSTEM_INFO si; - GetSystemInfo(&si); - return si.dwNumberOfProcessors; -} - - -} // namespace Poco diff --git a/base/poco/Foundation/src/Environment_WINCE.cpp b/base/poco/Foundation/src/Environment_WINCE.cpp deleted file mode 100644 index afa59b68d35..00000000000 --- a/base/poco/Foundation/src/Environment_WINCE.cpp +++ /dev/null @@ -1,243 +0,0 @@ -// -// Environment_WINCE.cpp -// -// Library: Foundation -// Package: Core -// Module: Environment -// -// Copyright (c) 2009-2010, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Environment_WINCE.h" -#include "Poco/Exception.h" -#include "Poco/UnicodeConverter.h" -#include "Poco/String.h" -#include "Poco/Path.h" -#include "Poco/NumberFormatter.h" -#include -#include -#include -#include - - -namespace Poco { - - -const std::string EnvironmentImpl::TEMP("TEMP"); -const std::string EnvironmentImpl::TMP("TMP"); -const std::string EnvironmentImpl::HOMEPATH("HOMEPATH"); -const std::string EnvironmentImpl::COMPUTERNAME("COMPUTERNAME"); -const std::string EnvironmentImpl::OS("OS"); -const std::string EnvironmentImpl::NUMBER_OF_PROCESSORS("NUMBER_OF_PROCESSORS"); -const std::string EnvironmentImpl::PROCESSOR_ARCHITECTURE("PROCESSOR_ARCHITECTURE"); - - -std::string EnvironmentImpl::getImpl(const std::string& name) -{ - std::string value; - if (!envVar(name, &value)) throw NotFoundException(name); - return value; -} - - -bool EnvironmentImpl::hasImpl(const std::string& name) -{ - return envVar(name, 0); -} - - -void EnvironmentImpl::setImpl(const std::string& name, const std::string& value) -{ - throw NotImplementedException("Cannot set environment variables on Windows CE"); -} - - -std::string EnvironmentImpl::osNameImpl() -{ - return "Windows CE"; -} - - -std::string EnvironmentImpl::osDisplayNameImpl() -{ - return osNameImpl(); -} - - -std::string EnvironmentImpl::osVersionImpl() -{ - OSVERSIONINFOW vi; - vi.dwOSVersionInfoSize = sizeof(vi); - if (GetVersionExW(&vi) == 0) throw SystemException("Cannot get OS version information"); - std::ostringstream str; - str << vi.dwMajorVersion << "." << vi.dwMinorVersion << " (Build " << (vi.dwBuildNumber & 0xFFFF); - std::string version; - UnicodeConverter::toUTF8(vi.szCSDVersion, version); - if (!version.empty()) str << ": " << version; - str << ")"; - return str.str(); -} - - -std::string EnvironmentImpl::osArchitectureImpl() -{ - SYSTEM_INFO si; - GetSystemInfo(&si); - switch (si.wProcessorArchitecture) - { - case PROCESSOR_ARCHITECTURE_INTEL: - return "IA32"; - case PROCESSOR_ARCHITECTURE_MIPS: - return "MIPS"; - case PROCESSOR_ARCHITECTURE_ALPHA: - return "ALPHA"; - case PROCESSOR_ARCHITECTURE_PPC: - return "PPC"; - case PROCESSOR_ARCHITECTURE_IA64: - return "IA64"; -#ifdef PROCESSOR_ARCHITECTURE_IA32_ON_WIN64 - case PROCESSOR_ARCHITECTURE_IA32_ON_WIN64: - return "IA64/32"; -#endif -#ifdef PROCESSOR_ARCHITECTURE_AMD64 - case PROCESSOR_ARCHITECTURE_AMD64: - return "AMD64"; -#endif - case PROCESSOR_ARCHITECTURE_SHX: - return "SHX"; - case PROCESSOR_ARCHITECTURE_ARM: - return "ARM"; - default: - return "Unknown"; - } -} - - -std::string EnvironmentImpl::nodeNameImpl() -{ - HKEY hKey; - DWORD dwDisposition; - if (RegCreateKeyExW(HKEY_LOCAL_MACHINE, L"\\Ident", 0, 0, 0, 0, 0, &hKey, &dwDisposition) != ERROR_SUCCESS) - throw SystemException("Cannot get node name", "registry key not found"); - - std::string value; - DWORD dwType; - BYTE bData[1026]; - DWORD dwData = sizeof(bData); - if (RegQueryValueExW(hKey, L"Name", 0, &dwType, bData, &dwData) == ERROR_SUCCESS) - { - switch (dwType) - { - case REG_SZ: - UnicodeConverter::toUTF8(reinterpret_cast(bData), value); - break; - - default: - RegCloseKey(hKey); - throw SystemException("Cannot get node name", "registry value has wrong type"); - } - } - else - { - RegCloseKey(hKey); - throw SystemException("Cannot get node name", "registry value not found"); - } - RegCloseKey(hKey); - return value; -} - - -void EnvironmentImpl::nodeIdImpl(NodeId& id) -{ - PIP_ADAPTER_INFO pAdapterInfo; - PIP_ADAPTER_INFO pAdapter = 0; - ULONG len = sizeof(IP_ADAPTER_INFO); - pAdapterInfo = reinterpret_cast(new char[len]); - // Make an initial call to GetAdaptersInfo to get - // the necessary size into len - DWORD rc = GetAdaptersInfo(pAdapterInfo, &len); - if (rc == ERROR_BUFFER_OVERFLOW) - { - delete [] reinterpret_cast(pAdapterInfo); - pAdapterInfo = reinterpret_cast(new char[len]); - } - else if (rc != ERROR_SUCCESS) - { - throw SystemException("cannot get network adapter list"); - } - try - { - bool found = false; - if (GetAdaptersInfo(pAdapterInfo, &len) == NO_ERROR) - { - pAdapter = pAdapterInfo; - while (pAdapter && !found) - { - if (pAdapter->Type == MIB_IF_TYPE_ETHERNET && pAdapter->AddressLength == sizeof(id)) - { - std::memcpy(&id, pAdapter->Address, pAdapter->AddressLength); - found = true; - } - pAdapter = pAdapter->Next; - } - } - else throw SystemException("cannot get network adapter list"); - if (!found) throw SystemException("no Ethernet adapter found"); - } - catch (Exception&) - { - delete [] reinterpret_cast(pAdapterInfo); - throw; - } - delete [] reinterpret_cast(pAdapterInfo); -} - - -unsigned EnvironmentImpl::processorCountImpl() -{ - SYSTEM_INFO si; - GetSystemInfo(&si); - return si.dwNumberOfProcessors; -} - - -bool EnvironmentImpl::envVar(const std::string& name, std::string* value) -{ - if (icompare(name, TEMP) == 0) - { - if (value) *value = Path::temp(); - } - else if (icompare(name, TMP) == 0) - { - if (value) *value = Path::temp(); - } - else if (icompare(name, HOMEPATH) == 0) - { - if (value) *value = Path::home(); - } - else if (icompare(name, COMPUTERNAME) == 0) - { - if (value) *value = nodeNameImpl(); - } - else if (icompare(name, OS) == 0) - { - if (value) *value = osNameImpl(); - } - else if (icompare(name, NUMBER_OF_PROCESSORS) == 0) - { - if (value) *value = NumberFormatter::format(processorCountImpl()); - } - else if (icompare(name, PROCESSOR_ARCHITECTURE) == 0) - { - if (value) *value = osArchitectureImpl(); - } - else return false; - return true; -} - - -} // namespace Poco diff --git a/base/poco/Foundation/src/Event_VX.cpp b/base/poco/Foundation/src/Event_VX.cpp deleted file mode 100644 index 18f667156ed..00000000000 --- a/base/poco/Foundation/src/Event_VX.cpp +++ /dev/null @@ -1,79 +0,0 @@ -// -// Event_POSIX.cpp -// -// Library: Foundation -// Package: Threading -// Module: Event -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Event_VX.h" -#include - - -namespace Poco { - - -EventImpl::EventImpl(bool autoReset): _auto(autoReset), _state(false) -{ - _sem = semCCreate(SEM_Q_PRIORITY, 0); - if (_sem == 0) - throw Poco::SystemException("cannot create event"); -} - - -EventImpl::~EventImpl() -{ - semDelete(_sem); -} - - -void EventImpl::setImpl() -{ - if (_auto) - { - if (semGive(_sem) != OK) - throw SystemException("cannot set event"); - } - else - { - _state = true; - if (semFlush(_sem) != OK) - throw SystemException("cannot set event"); - } -} - - -void EventImpl::resetImpl() -{ - _state = false; -} - - -void EventImpl::waitImpl() -{ - if (!_state) - { - if (semTake(_sem, WAIT_FOREVER) != OK) - throw SystemException("cannot wait for event"); - } -} - - -bool EventImpl::waitImpl(long milliseconds) -{ - if (!_state) - { - int ticks = milliseconds*sysClkRateGet()/1000; - return semTake(_sem, ticks) == OK; - } - else return true; -} - - -} // namespace Poco diff --git a/base/poco/Foundation/src/Event_WIN32.cpp b/base/poco/Foundation/src/Event_WIN32.cpp deleted file mode 100644 index 256951870a9..00000000000 --- a/base/poco/Foundation/src/Event_WIN32.cpp +++ /dev/null @@ -1,61 +0,0 @@ -// -// Event_WIN32.cpp -// -// Library: Foundation -// Package: Threading -// Module: Event -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Event_WIN32.h" - - -namespace Poco { - - -EventImpl::EventImpl(bool autoReset) -{ - _event = CreateEventW(NULL, autoReset ? FALSE : TRUE, FALSE, NULL); - if (!_event) - throw SystemException("cannot create event"); -} - - -EventImpl::~EventImpl() -{ - CloseHandle(_event); -} - - -void EventImpl::waitImpl() -{ - switch (WaitForSingleObject(_event, INFINITE)) - { - case WAIT_OBJECT_0: - return; - default: - throw SystemException("wait for event failed"); - } -} - - -bool EventImpl::waitImpl(long milliseconds) -{ - switch (WaitForSingleObject(_event, milliseconds + 1)) - { - case WAIT_TIMEOUT: - return false; - case WAIT_OBJECT_0: - return true; - default: - throw SystemException("wait for event failed"); - } -} - - -} // namespace Poco diff --git a/base/poco/Foundation/src/FPEnvironment_SUN.cpp b/base/poco/Foundation/src/FPEnvironment_SUN.cpp deleted file mode 100644 index 36ee36431df..00000000000 --- a/base/poco/Foundation/src/FPEnvironment_SUN.cpp +++ /dev/null @@ -1,139 +0,0 @@ -// -// FPEnvironment_SUN.cpp -// -// Library: Foundation -// Package: Core -// Module: FPEnvironment -// -// Copyright (c) 2005-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include -#include "Poco/FPEnvironment_SUN.h" - - -namespace Poco { - - -FPEnvironmentImpl::FPEnvironmentImpl() -{ - _rnd = fpgetround(); - _exc = fpgetmask(); -} - - -FPEnvironmentImpl::FPEnvironmentImpl(const FPEnvironmentImpl& env) -{ - _rnd = env._rnd; - _exc = env._exc; -} - - -FPEnvironmentImpl::~FPEnvironmentImpl() -{ - fpsetround(_rnd); - fpsetmask(_exc); -} - - -FPEnvironmentImpl& FPEnvironmentImpl::operator = (const FPEnvironmentImpl& env) -{ - _rnd = env._rnd; - _exc = env._exc; - return *this; -} - - -bool FPEnvironmentImpl::isInfiniteImpl(float value) -{ - int cls = fpclass(value); - return cls == FP_PINF || cls == FP_NINF; -} - - -bool FPEnvironmentImpl::isInfiniteImpl(double value) -{ - int cls = fpclass(value); - return cls == FP_PINF || cls == FP_NINF; -} - - -bool FPEnvironmentImpl::isInfiniteImpl(long double value) -{ - int cls = fpclass(value); - return cls == FP_PINF || cls == FP_NINF; -} - - -bool FPEnvironmentImpl::isNaNImpl(float value) -{ - return isnanf(value) != 0; -} - - -bool FPEnvironmentImpl::isNaNImpl(double value) -{ - return isnan(value) != 0; -} - - -bool FPEnvironmentImpl::isNaNImpl(long double value) -{ - return isnan((double) value) != 0; -} - - -float FPEnvironmentImpl::copySignImpl(float target, float source) -{ - return (float) copysign(target, source); -} - - -double FPEnvironmentImpl::copySignImpl(double target, double source) -{ - return (float) copysign(target, source); -} - - -long double FPEnvironmentImpl::copySignImpl(long double target, long double source) -{ - return (source > 0 && target > 0) || (source < 0 && target < 0) ? target : -target; -} - - -void FPEnvironmentImpl::keepCurrentImpl() -{ - fpsetround(_rnd); - fpsetmask(_exc); -} - - -void FPEnvironmentImpl::clearFlagsImpl() -{ - fpsetsticky(0); -} - - -bool FPEnvironmentImpl::isFlagImpl(FlagImpl flag) -{ - return (fpgetsticky() & flag) != 0; -} - - -void FPEnvironmentImpl::setRoundingModeImpl(RoundingModeImpl mode) -{ - fpsetround((fp_rnd) mode); -} - - -FPEnvironmentImpl::RoundingModeImpl FPEnvironmentImpl::getRoundingModeImpl() -{ - return (FPEnvironmentImpl::RoundingModeImpl) fpgetround(); -} - - -} // namespace Poco diff --git a/base/poco/Foundation/src/FPEnvironment_WIN32.cpp b/base/poco/Foundation/src/FPEnvironment_WIN32.cpp deleted file mode 100644 index 32c7f65e7b7..00000000000 --- a/base/poco/Foundation/src/FPEnvironment_WIN32.cpp +++ /dev/null @@ -1,76 +0,0 @@ -// -// FPEnvironment_WIN32.cpp -// -// Library: Foundation -// Package: Core -// Module: FPEnvironment -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/FPEnvironment_WIN32.h" - - -namespace Poco { - - -FPEnvironmentImpl::FPEnvironmentImpl() -{ - _env = _controlfp(0, 0); -} - - -FPEnvironmentImpl::FPEnvironmentImpl(const FPEnvironmentImpl& env) -{ - _env = env._env; -} - - -FPEnvironmentImpl::~FPEnvironmentImpl() -{ - _controlfp(_env, _MCW_RC); -} - - -FPEnvironmentImpl& FPEnvironmentImpl::operator = (const FPEnvironmentImpl& env) -{ - _env = env._env; - return *this; -} - - -void FPEnvironmentImpl::keepCurrentImpl() -{ - _env = _controlfp(0, 0); -} - - -void FPEnvironmentImpl::clearFlagsImpl() -{ - _clearfp(); -} - - -bool FPEnvironmentImpl::isFlagImpl(FlagImpl flag) -{ - return (_statusfp() & flag) != 0; -} - - -void FPEnvironmentImpl::setRoundingModeImpl(RoundingModeImpl mode) -{ - _controlfp(mode, _MCW_RC); -} - - -FPEnvironmentImpl::RoundingModeImpl FPEnvironmentImpl::getRoundingModeImpl() -{ - return RoundingModeImpl(_controlfp(0, 0) & _MCW_RC); -} - - -} // namespace Poco diff --git a/base/poco/Foundation/src/FileStream_WIN32.cpp b/base/poco/Foundation/src/FileStream_WIN32.cpp deleted file mode 100644 index 0fc52f0ce01..00000000000 --- a/base/poco/Foundation/src/FileStream_WIN32.cpp +++ /dev/null @@ -1,199 +0,0 @@ -// -// FileStream.cpp -// -// Library: Foundation -// Package: Streams -// Module: FileStream -// -// Copyright (c) 2007, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/FileStream.h" -#include "Poco/File.h" -#include "Poco/Exception.h" - - -namespace Poco { - - -FileStreamBuf::FileStreamBuf(): - BufferedBidirectionalStreamBuf(BUFFER_SIZE, std::ios::in | std::ios::out), - _handle(INVALID_HANDLE_VALUE), - _pos(0) -{ -} - - -FileStreamBuf::~FileStreamBuf() -{ - close(); -} - - -void FileStreamBuf::open(const std::string& path, std::ios::openmode mode) -{ - poco_assert (_handle == INVALID_HANDLE_VALUE); - - _path = path; - _pos = 0; - setMode(mode); - resetBuffers(); - - DWORD access = 0; - if (mode & std::ios::in) - access |= GENERIC_READ; - if (mode & std::ios::out) - access |= GENERIC_WRITE; - - DWORD shareMode = FILE_SHARE_READ; - if (!(mode & std::ios::out)) - shareMode |= FILE_SHARE_WRITE; - - DWORD creationDisp = OPEN_EXISTING; - if (mode & std::ios::trunc) - creationDisp = CREATE_ALWAYS; - else if (mode & std::ios::out) - creationDisp = OPEN_ALWAYS; - - DWORD flags = FILE_ATTRIBUTE_NORMAL; - - _handle = CreateFileA(path.c_str(), access, shareMode, NULL, creationDisp, flags, NULL); - - if (_handle == INVALID_HANDLE_VALUE) - File::handleLastError(_path); - - if ((mode & std::ios::ate) || (mode & std::ios::app)) - seekoff(0, std::ios::end, mode); -} - - -int FileStreamBuf::readFromDevice(char* buffer, std::streamsize length) -{ - if (INVALID_HANDLE_VALUE == _handle || !(getMode() & std::ios::in)) - return -1; - - if (getMode() & std::ios::out) - sync(); - - DWORD bytesRead(0); - BOOL rc = ReadFile(_handle, buffer, static_cast(length), &bytesRead, NULL); - if (rc == 0) - File::handleLastError(_path); - - _pos += bytesRead; - - return static_cast(bytesRead); -} - - -int FileStreamBuf::writeToDevice(const char* buffer, std::streamsize length) -{ - if (INVALID_HANDLE_VALUE == _handle || !(getMode() & std::ios::out)) - return -1; - - if (getMode() & std::ios::app) - { - LARGE_INTEGER li; - li.QuadPart = 0; - li.LowPart = SetFilePointer(_handle, li.LowPart, &li.HighPart, FILE_END); - if (li.LowPart == INVALID_SET_FILE_POINTER && GetLastError() != NO_ERROR) - File::handleLastError(_path); - _pos = li.QuadPart; - } - - DWORD bytesWritten(0); - BOOL rc = WriteFile(_handle, buffer, static_cast(length), &bytesWritten, NULL); - if (rc == 0) - File::handleLastError(_path); - - _pos += bytesWritten; - - return static_cast(bytesWritten); -} - - -bool FileStreamBuf::close() -{ - bool success = true; - if (_handle != INVALID_HANDLE_VALUE) - { - try - { - if (getMode() & std::ios::out) - sync(); - } - catch (...) - { - success = false; - } - CloseHandle(_handle); - _handle = INVALID_HANDLE_VALUE; - } - return success; -} - - -std::streampos FileStreamBuf::seekoff(std::streamoff off, std::ios::seekdir dir, std::ios::openmode mode) -{ - if (INVALID_HANDLE_VALUE == _handle || !(getMode() & mode)) - return -1; - - if (getMode() & std::ios::out) - sync(); - - std::streamoff adj; - if (mode & std::ios::in) - adj = static_cast(egptr() - gptr()); - else - adj = 0; - - resetBuffers(); - - DWORD offset = FILE_BEGIN; - if (dir == std::ios::cur) - { - offset = FILE_CURRENT; - off -= adj; - } - else if (dir == std::ios::end) - { - offset = FILE_END; - } - - LARGE_INTEGER li; - li.QuadPart = off; - li.LowPart = SetFilePointer(_handle, li.LowPart, &li.HighPart, offset); - - if (li.LowPart == INVALID_SET_FILE_POINTER && GetLastError() != NO_ERROR) - File::handleLastError(_path); - _pos = li.QuadPart; - return std::streampos(static_cast(_pos)); -} - - -std::streampos FileStreamBuf::seekpos(std::streampos pos, std::ios::openmode mode) -{ - if (INVALID_HANDLE_VALUE == _handle || !(getMode() & mode)) - return -1; - - if (getMode() & std::ios::out) - sync(); - - resetBuffers(); - - LARGE_INTEGER li; - li.QuadPart = pos; - li.LowPart = SetFilePointer(_handle, li.LowPart, &li.HighPart, FILE_BEGIN); - - if (li.LowPart == INVALID_SET_FILE_POINTER && GetLastError() != NO_ERROR) - File::handleLastError(_path); - _pos = li.QuadPart; - return std::streampos(static_cast(_pos)); -} - - -} // namespace Poco diff --git a/base/poco/Foundation/src/File_VX.cpp b/base/poco/Foundation/src/File_VX.cpp deleted file mode 100644 index 9f59b478a88..00000000000 --- a/base/poco/Foundation/src/File_VX.cpp +++ /dev/null @@ -1,408 +0,0 @@ -// -// File_VX.cpp -// -// Library: Foundation -// Package: Filesystem -// Module: File -// -// Copyright (c) 2004-2011, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/File_VX.h" -#include "Poco/Buffer.h" -#include "Poco/Exception.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -namespace Poco { - - -FileImpl::FileImpl() -{ -} - - -FileImpl::FileImpl(const std::string& path): _path(path) -{ - std::string::size_type n = _path.size(); - if (n > 1 && _path[n - 1] == '/') - _path.resize(n - 1); -} - - -FileImpl::~FileImpl() -{ -} - - -void FileImpl::swapImpl(FileImpl& file) -{ - std::swap(_path, file._path); -} - - -void FileImpl::setPathImpl(const std::string& path) -{ - _path = path; - std::string::size_type n = _path.size(); - if (n > 1 && _path[n - 1] == '/') - _path.resize(n - 1); -} - - -bool FileImpl::existsImpl() const -{ - poco_assert (!_path.empty()); - - struct stat st; - return stat(const_cast(_path.c_str()), &st) == 0; -} - - -bool FileImpl::canReadImpl() const -{ - poco_assert (!_path.empty()); - - return true; -} - - -bool FileImpl::canWriteImpl() const -{ - poco_assert (!_path.empty()); - - return true; -} - - -bool FileImpl::canExecuteImpl() const -{ - return false; -} - - -bool FileImpl::isFileImpl() const -{ - poco_assert (!_path.empty()); - - struct stat st; - if (stat(const_cast(_path.c_str()), &st) == 0) - return S_ISREG(st.st_mode); - else - handleLastErrorImpl(_path); - return false; -} - - -bool FileImpl::isDirectoryImpl() const -{ - poco_assert (!_path.empty()); - - struct stat st; - if (stat(const_cast(_path.c_str()), &st) == 0) - return S_ISDIR(st.st_mode); - else - handleLastErrorImpl(_path); - return false; -} - - -bool FileImpl::isLinkImpl() const -{ - return false; -} - - -bool FileImpl::isDeviceImpl() const -{ - poco_assert (!_path.empty()); - - struct stat st; - if (stat(const_cast(_path.c_str()), &st) == 0) - return S_ISCHR(st.st_mode) || S_ISBLK(st.st_mode); - else - handleLastErrorImpl(_path); - return false; -} - - -bool FileImpl::isHiddenImpl() const -{ - poco_assert (!_path.empty()); - Path p(_path); - p.makeFile(); - - return p.getFileName()[0] == '.'; -} - - -Timestamp FileImpl::createdImpl() const -{ - poco_assert (!_path.empty()); - - struct stat st; - if (stat(const_cast(_path.c_str()), &st) == 0) - return Timestamp::fromEpochTime(st.st_ctime); - else - handleLastErrorImpl(_path); - return 0; -} - - -Timestamp FileImpl::getLastModifiedImpl() const -{ - poco_assert (!_path.empty()); - - struct stat st; - if (stat(const_cast(_path.c_str()), &st) == 0) - return Timestamp::fromEpochTime(st.st_mtime); - else - handleLastErrorImpl(_path); - return 0; -} - - -void FileImpl::setLastModifiedImpl(const Timestamp& ts) -{ - poco_assert (!_path.empty()); - - struct utimbuf tb; - tb.actime = ts.epochTime(); - tb.modtime = ts.epochTime(); - if (utime(const_cast(_path.c_str()), &tb) != 0) - handleLastErrorImpl(_path); -} - - -FileImpl::FileSizeImpl FileImpl::getSizeImpl() const -{ - poco_assert (!_path.empty()); - - struct stat st; - if (stat(const_cast(_path.c_str()), &st) == 0) - return st.st_size; - else - handleLastErrorImpl(_path); - return 0; -} - - -void FileImpl::setSizeImpl(FileSizeImpl size) -{ - poco_assert (!_path.empty()); - - int fd = open(_path.c_str(), O_WRONLY, S_IRWXU); - if (fd != -1) - { - try - { - if (ftruncate(fd, size) != 0) - handleLastErrorImpl(_path); - } - catch (...) - { - close(fd); - throw; - } - } -} - - -void FileImpl::setWriteableImpl(bool flag) -{ - poco_assert (!_path.empty()); -} - - -void FileImpl::setExecutableImpl(bool flag) -{ - poco_assert (!_path.empty()); -} - - -void FileImpl::copyToImpl(const std::string& path) const -{ - poco_assert (!_path.empty()); - - int sd = open(_path.c_str(), O_RDONLY, 0); - if (sd == -1) handleLastErrorImpl(_path); - - struct stat st; - if (fstat(sd, &st) != 0) - { - close(sd); - handleLastErrorImpl(_path); - } - const long blockSize = st.st_blksize; - - int dd = open(path.c_str(), O_CREAT | O_TRUNC | O_WRONLY, st.st_mode & S_IRWXU); - if (dd == -1) - { - close(sd); - handleLastErrorImpl(path); - } - Buffer buffer(blockSize); - try - { - int n; - while ((n = read(sd, buffer.begin(), blockSize)) > 0) - { - if (write(dd, buffer.begin(), n) != n) - handleLastErrorImpl(path); - } - if (n < 0) - handleLastErrorImpl(_path); - } - catch (...) - { - close(sd); - close(dd); - throw; - } - close(sd); - close(dd); -} - - -void FileImpl::renameToImpl(const std::string& path) -{ - poco_assert (!_path.empty()); - - if (rename(_path.c_str(), path.c_str()) != 0) - handleLastErrorImpl(_path); -} - - -void FileImpl::linkToImpl(const std::string& path, int type) const -{ - throw Poco::NotImplementedException("File::linkTo() is not available on this platform"); -} - - -void FileImpl::removeImpl() -{ - poco_assert (!_path.empty()); - - int rc; - if (!isLinkImpl() && isDirectoryImpl()) - rc = rmdir(_path.c_str()); - else - rc = unlink(const_cast(_path.c_str())); - if (rc) handleLastErrorImpl(_path); -} - - -bool FileImpl::createFileImpl() -{ - poco_assert (!_path.empty()); - - int n = open(_path.c_str(), O_WRONLY | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH); - if (n != -1) - { - close(n); - return true; - } - if (n == -1 && errno == EEXIST) - return false; - else - handleLastErrorImpl(_path); - return false; -} - - -bool FileImpl::createDirectoryImpl() -{ - poco_assert (!_path.empty()); - - if (existsImpl() && isDirectoryImpl()) - return false; - if (mkdir(_path.c_str()) != 0) - handleLastErrorImpl(_path); - return true; -} - - -FileImpl::FileSizeImpl FileImpl::totalSpaceImpl() const -{ - poco_assert(!_path.empty()); - - struct statfs stats; - if (statfs(_path.c_str(), &stats) != 0) - handleLastErrorImpl(_path); - - return (FileSizeImpl)stats.f_blocks * (FileSizeImpl)stats.f_bsize; -} - - -FileImpl::FileSizeImpl FileImpl::usableSpaceImpl() const -{ - poco_assert(!_path.empty()); - - struct statfs stats; - if (statfs(_path.c_str(), &stats) != 0) - handleLastErrorImpl(_path); - - return (FileSizeImpl)stats.f_bavail * (FileSizeImpl)stats.f_bsize; -} - - -FileImpl::FileSizeImpl FileImpl::freeSpaceImpl() const -{ - poco_assert(!_path.empty()); - - struct statfs stats; - if (statfs(_path.c_str(), &stats) != 0) - handleLastErrorImpl(_path); - - return (FileSizeImpl)stats.f_bfree * (FileSizeImpl)stats.f_bsize; -} - - -void FileImpl::handleLastErrorImpl(const std::string& path) -{ - switch (errno) - { - case EIO: - throw IOException(path); - case EPERM: - throw FileAccessDeniedException("insufficient permissions", path); - case EACCES: - throw FileAccessDeniedException(path); - case ENOENT: - throw FileNotFoundException(path); - case ENOTDIR: - throw OpenFileException("not a directory", path); - case EISDIR: - throw OpenFileException("not a file", path); - case EROFS: - throw FileReadOnlyException(path); - case EEXIST: - throw FileExistsException(path); - case ENOSPC: - throw FileException("no space left on device", path); - case ENOTEMPTY: - throw DirectoryNotEmptyException(path); - case ENAMETOOLONG: - throw PathSyntaxException(path); - case ENFILE: - case EMFILE: - throw FileException("too many open files", path); - default: - throw FileException(std::strerror(errno), path); - } -} - - -} // namespace Poco diff --git a/base/poco/Foundation/src/File_WIN32.cpp b/base/poco/Foundation/src/File_WIN32.cpp deleted file mode 100644 index 2c569e36976..00000000000 --- a/base/poco/Foundation/src/File_WIN32.cpp +++ /dev/null @@ -1,465 +0,0 @@ -// -// File_WIN32.cpp -// -// Library: Foundation -// Package: Filesystem -// Module: File -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/File_WIN32.h" -#include "Poco/Exception.h" -#include "Poco/String.h" -#include "Poco/UnWindows.h" - - -namespace Poco { - - -class FileHandle -{ -public: - FileHandle(const std::string& path, DWORD access, DWORD share, DWORD disp) - { - _h = CreateFileA(path.c_str(), access, share, 0, disp, 0, 0); - if (_h == INVALID_HANDLE_VALUE) - { - FileImpl::handleLastErrorImpl(path); - } - } - - ~FileHandle() - { - if (_h != INVALID_HANDLE_VALUE) CloseHandle(_h); - } - - HANDLE get() const - { - return _h; - } - -private: - HANDLE _h; -}; - - -FileImpl::FileImpl() -{ -} - - -FileImpl::FileImpl(const std::string& path): _path(path) -{ - std::string::size_type n = _path.size(); - if (n > 1 && (_path[n - 1] == '\\' || _path[n - 1] == '/') && !((n == 3 && _path[1]==':'))) - { - _path.resize(n - 1); - } -} - - -FileImpl::~FileImpl() -{ -} - - -void FileImpl::swapImpl(FileImpl& file) -{ - std::swap(_path, file._path); -} - - -void FileImpl::setPathImpl(const std::string& path) -{ - _path = path; - std::string::size_type n = _path.size(); - if (n > 1 && (_path[n - 1] == '\\' || _path[n - 1] == '/') && !((n == 3 && _path[1]==':'))) - { - _path.resize(n - 1); - } -} - - -bool FileImpl::existsImpl() const -{ - poco_assert (!_path.empty()); - - DWORD attr = GetFileAttributes(_path.c_str()); - if (attr == INVALID_FILE_ATTRIBUTES) - { - switch (GetLastError()) - { - case ERROR_FILE_NOT_FOUND: - case ERROR_PATH_NOT_FOUND: - case ERROR_NOT_READY: - case ERROR_INVALID_DRIVE: - return false; - default: - handleLastErrorImpl(_path); - } - } - return true; -} - - -bool FileImpl::canReadImpl() const -{ - poco_assert (!_path.empty()); - - DWORD attr = GetFileAttributes(_path.c_str()); - if (attr == INVALID_FILE_ATTRIBUTES) - { - switch (GetLastError()) - { - case ERROR_ACCESS_DENIED: - return false; - default: - handleLastErrorImpl(_path); - } - } - return true; -} - - -bool FileImpl::canWriteImpl() const -{ - poco_assert (!_path.empty()); - - DWORD attr = GetFileAttributes(_path.c_str()); - if (attr == INVALID_FILE_ATTRIBUTES) - handleLastErrorImpl(_path); - return (attr & FILE_ATTRIBUTE_READONLY) == 0; -} - - -bool FileImpl::canExecuteImpl() const -{ - Path p(_path); - return icompare(p.getExtension(), "exe") == 0; -} - - -bool FileImpl::isFileImpl() const -{ - return !isDirectoryImpl() && !isDeviceImpl(); -} - - -bool FileImpl::isDirectoryImpl() const -{ - poco_assert (!_path.empty()); - - DWORD attr = GetFileAttributes(_path.c_str()); - if (attr == INVALID_FILE_ATTRIBUTES) - handleLastErrorImpl(_path); - return (attr & FILE_ATTRIBUTE_DIRECTORY) != 0; -} - - -bool FileImpl::isLinkImpl() const -{ - poco_assert (!_path.empty()); - - DWORD attr = GetFileAttributes(_path.c_str()); - if (attr == INVALID_FILE_ATTRIBUTES) - handleLastErrorImpl(_path); - return (attr & FILE_ATTRIBUTE_DIRECTORY) == 0 && (attr & FILE_ATTRIBUTE_REPARSE_POINT) != 0; -} - - - -bool FileImpl::isDeviceImpl() const -{ - return - _path.compare(0, 4, "\\\\.\\") == 0 || - icompare(_path, "CON") == 0 || - icompare(_path, "PRN") == 0 || - icompare(_path, "AUX") == 0 || - icompare(_path, "NUL") == 0 || - ( (icompare(_path, 0, 3, "LPT") == 0 || icompare(_path, 0, 3, "COM") == 0) && - _path.size() == 4 && - _path[3] > 0x30 && - isdigit(_path[3]) - ); -} - - -bool FileImpl::isHiddenImpl() const -{ - poco_assert (!_path.empty()); - - DWORD attr = GetFileAttributes(_path.c_str()); - if (attr == INVALID_FILE_ATTRIBUTES) - handleLastErrorImpl(_path); - return (attr & FILE_ATTRIBUTE_HIDDEN) != 0; -} - - -Timestamp FileImpl::createdImpl() const -{ - poco_assert (!_path.empty()); - - WIN32_FILE_ATTRIBUTE_DATA fad; - if (GetFileAttributesEx(_path.c_str(), GetFileExInfoStandard, &fad) == 0) - handleLastErrorImpl(_path); - return Timestamp::fromFileTimeNP(fad.ftCreationTime.dwLowDateTime, fad.ftCreationTime.dwHighDateTime); -} - - -Timestamp FileImpl::getLastModifiedImpl() const -{ - poco_assert (!_path.empty()); - - WIN32_FILE_ATTRIBUTE_DATA fad; - if (GetFileAttributesEx(_path.c_str(), GetFileExInfoStandard, &fad) == 0) - handleLastErrorImpl(_path); - return Timestamp::fromFileTimeNP(fad.ftLastWriteTime.dwLowDateTime, fad.ftLastWriteTime.dwHighDateTime); -} - - -void FileImpl::setLastModifiedImpl(const Timestamp& ts) -{ - poco_assert (!_path.empty()); - - UInt32 low; - UInt32 high; - ts.toFileTimeNP(low, high); - FILETIME ft; - ft.dwLowDateTime = low; - ft.dwHighDateTime = high; - FileHandle fh(_path, FILE_WRITE_ATTRIBUTES, FILE_SHARE_READ | FILE_SHARE_WRITE, OPEN_EXISTING); - if (SetFileTime(fh.get(), 0, &ft, &ft) == 0) - handleLastErrorImpl(_path); -} - - -FileImpl::FileSizeImpl FileImpl::getSizeImpl() const -{ - poco_assert (!_path.empty()); - - WIN32_FILE_ATTRIBUTE_DATA fad; - if (GetFileAttributesEx(_path.c_str(), GetFileExInfoStandard, &fad) == 0) - handleLastErrorImpl(_path); - LARGE_INTEGER li; - li.LowPart = fad.nFileSizeLow; - li.HighPart = fad.nFileSizeHigh; - return li.QuadPart; -} - - -void FileImpl::setSizeImpl(FileSizeImpl size) -{ - poco_assert (!_path.empty()); - - FileHandle fh(_path, GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE, OPEN_EXISTING); - LARGE_INTEGER li; - li.QuadPart = size; - if (SetFilePointer(fh.get(), li.LowPart, &li.HighPart, FILE_BEGIN) == INVALID_SET_FILE_POINTER) - handleLastErrorImpl(_path); - if (SetEndOfFile(fh.get()) == 0) - handleLastErrorImpl(_path); -} - - -void FileImpl::setWriteableImpl(bool flag) -{ - poco_assert (!_path.empty()); - - DWORD attr = GetFileAttributes(_path.c_str()); - if (attr == -1) - handleLastErrorImpl(_path); - if (flag) - attr &= ~FILE_ATTRIBUTE_READONLY; - else - attr |= FILE_ATTRIBUTE_READONLY; - if (SetFileAttributes(_path.c_str(), attr) == 0) - handleLastErrorImpl(_path); -} - - -void FileImpl::setExecutableImpl(bool flag) -{ - // not supported -} - - -void FileImpl::copyToImpl(const std::string& path) const -{ - poco_assert (!_path.empty()); - - if (CopyFileA(_path.c_str(), path.c_str(), FALSE) == 0) - handleLastErrorImpl(_path); -} - - -void FileImpl::renameToImpl(const std::string& path) -{ - poco_assert (!_path.empty()); - - if (MoveFileExA(_path.c_str(), path.c_str(), MOVEFILE_REPLACE_EXISTING) == 0) - handleLastErrorImpl(_path); -} - - -void FileImpl::linkToImpl(const std::string& path, int type) const -{ - poco_assert (!_path.empty()); - - if (type == 0) - { - if (CreateHardLinkA(path.c_str(), _path.c_str(), NULL) == 0) - handleLastErrorImpl(_path); - } - else - { -#if _WIN32_WINNT >= 0x0600 && defined(SYMBOLIC_LINK_FLAG_DIRECTORY) - DWORD flags = 0; - if (isDirectoryImpl()) flags |= SYMBOLIC_LINK_FLAG_DIRECTORY; -#ifdef SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE - flags |= SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE; -#endif - if (CreateSymbolicLinkA(path.c_str(), _path.c_str(), flags) == 0) - handleLastErrorImpl(_path); -#else - throw Poco::NotImplementedException("Symbolic link support not available in used version of the Windows SDK") -#endif - - } -} - - -void FileImpl::removeImpl() -{ - poco_assert (!_path.empty()); - - if (isDirectoryImpl()) - { - if (RemoveDirectoryA(_path.c_str()) == 0) - handleLastErrorImpl(_path); - } - else - { - if (DeleteFileA(_path.c_str()) == 0) - handleLastErrorImpl(_path); - } -} - - -bool FileImpl::createFileImpl() -{ - poco_assert (!_path.empty()); - - HANDLE hFile = CreateFileA(_path.c_str(), GENERIC_WRITE, 0, 0, CREATE_NEW, 0, 0); - if (hFile != INVALID_HANDLE_VALUE) - { - CloseHandle(hFile); - return true; - } - else if (GetLastError() == ERROR_FILE_EXISTS) - return false; - else - handleLastErrorImpl(_path); - return false; -} - - -bool FileImpl::createDirectoryImpl() -{ - poco_assert (!_path.empty()); - - if (existsImpl() && isDirectoryImpl()) - return false; - if (CreateDirectoryA(_path.c_str(), 0) == 0) - handleLastErrorImpl(_path); - return true; -} - - -FileImpl::FileSizeImpl FileImpl::totalSpaceImpl() const -{ - poco_assert(!_path.empty()); - - ULARGE_INTEGER space; - if (!GetDiskFreeSpaceExA(_path.c_str(), NULL, &space, NULL)) - handleLastErrorImpl(_path); - return space.QuadPart; -} - - -FileImpl::FileSizeImpl FileImpl::usableSpaceImpl() const -{ - poco_assert(!_path.empty()); - - ULARGE_INTEGER space; - if (!GetDiskFreeSpaceExA(upath.c_str(), &space, NULL, NULL)) - handleLastErrorImpl(_path); - return space.QuadPart; -} - - -FileImpl::FileSizeImpl FileImpl::freeSpaceImpl() const -{ - poco_assert(!_path.empty()); - - ULARGE_INTEGER space; - if (!GetDiskFreeSpaceExA(_path.c_str(), NULL, NULL, &space)) - handleLastErrorImpl(_path); - return space.QuadPart; -} - - -void FileImpl::handleLastErrorImpl(const std::string& path) -{ - DWORD err = GetLastError(); - switch (err) - { - case ERROR_FILE_NOT_FOUND: - throw FileNotFoundException(path, err); - case ERROR_PATH_NOT_FOUND: - case ERROR_BAD_NETPATH: - case ERROR_CANT_RESOLVE_FILENAME: - case ERROR_INVALID_DRIVE: - throw PathNotFoundException(path, err); - case ERROR_ACCESS_DENIED: - throw FileAccessDeniedException(path, err); - case ERROR_ALREADY_EXISTS: - case ERROR_FILE_EXISTS: - throw FileExistsException(path, err); - case ERROR_INVALID_NAME: - case ERROR_DIRECTORY: - case ERROR_FILENAME_EXCED_RANGE: - case ERROR_BAD_PATHNAME: - throw PathSyntaxException(path, err); - case ERROR_FILE_READ_ONLY: - throw FileReadOnlyException(path, err); - case ERROR_CANNOT_MAKE: - throw CreateFileException(path, err); - case ERROR_DIR_NOT_EMPTY: - throw DirectoryNotEmptyException(path, err); - case ERROR_WRITE_FAULT: - throw WriteFileException(path, err); - case ERROR_READ_FAULT: - throw ReadFileException(path, err); - case ERROR_SHARING_VIOLATION: - throw FileException("sharing violation", path, err); - case ERROR_LOCK_VIOLATION: - throw FileException("lock violation", path, err); - case ERROR_HANDLE_EOF: - throw ReadFileException("EOF reached", path, err); - case ERROR_HANDLE_DISK_FULL: - case ERROR_DISK_FULL: - throw WriteFileException("disk is full", path, err); - case ERROR_NEGATIVE_SEEK: - throw FileException("negative seek", path, err); - default: - throw FileException(path, err); - } -} - - -} // namespace Poco diff --git a/base/poco/Foundation/src/File_WIN32U.cpp b/base/poco/Foundation/src/File_WIN32U.cpp deleted file mode 100644 index f8609f17c09..00000000000 --- a/base/poco/Foundation/src/File_WIN32U.cpp +++ /dev/null @@ -1,492 +0,0 @@ -// -// File_WIN32U.cpp -// -// Library: Foundation -// Package: Filesystem -// Module: File -// -// Copyright (c) 2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/File_WIN32U.h" -#include "Poco/Exception.h" -#include "Poco/String.h" -#include "Poco/UnicodeConverter.h" -#include "Poco/UnWindows.h" - - -namespace Poco { - - -class FileHandle -{ -public: - FileHandle(const std::string& path, const std::wstring& upath, DWORD access, DWORD share, DWORD disp) - { - _h = CreateFileW(upath.c_str(), access, share, 0, disp, 0, 0); - if (_h == INVALID_HANDLE_VALUE) - { - FileImpl::handleLastErrorImpl(path); - } - } - - ~FileHandle() - { - if (_h != INVALID_HANDLE_VALUE) CloseHandle(_h); - } - - HANDLE get() const - { - return _h; - } - -private: - HANDLE _h; -}; - - -FileImpl::FileImpl() -{ -} - - -FileImpl::FileImpl(const std::string& path): _path(path) -{ - std::string::size_type n = _path.size(); - if (n > 1 && (_path[n - 1] == '\\' || _path[n - 1] == '/') && !((n == 3 && _path[1]==':'))) - { - _path.resize(n - 1); - } - convertPath(_path, _upath); -} - - -FileImpl::~FileImpl() -{ -} - - -void FileImpl::swapImpl(FileImpl& file) -{ - std::swap(_path, file._path); - std::swap(_upath, file._upath); -} - - -void FileImpl::setPathImpl(const std::string& path) -{ - _path = path; - std::string::size_type n = _path.size(); - if (n > 1 && (_path[n - 1] == '\\' || _path[n - 1] == '/') && !((n == 3 && _path[1]==':'))) - { - _path.resize(n - 1); - } - convertPath(_path, _upath); -} - - -bool FileImpl::existsImpl() const -{ - poco_assert (!_path.empty()); - - DWORD attr = GetFileAttributesW(_upath.c_str()); - if (attr == INVALID_FILE_ATTRIBUTES) - { - switch (GetLastError()) - { - case ERROR_FILE_NOT_FOUND: - case ERROR_PATH_NOT_FOUND: - case ERROR_NOT_READY: - case ERROR_INVALID_DRIVE: - return false; - default: - handleLastErrorImpl(_path); - } - } - return true; -} - - -bool FileImpl::canReadImpl() const -{ - poco_assert (!_path.empty()); - - DWORD attr = GetFileAttributesW(_upath.c_str()); - if (attr == INVALID_FILE_ATTRIBUTES) - { - switch (GetLastError()) - { - case ERROR_ACCESS_DENIED: - return false; - default: - handleLastErrorImpl(_path); - } - } - return true; -} - - -bool FileImpl::canWriteImpl() const -{ - poco_assert (!_path.empty()); - - DWORD attr = GetFileAttributesW(_upath.c_str()); - if (attr == INVALID_FILE_ATTRIBUTES) - handleLastErrorImpl(_path); - return (attr & FILE_ATTRIBUTE_READONLY) == 0; -} - - -bool FileImpl::canExecuteImpl() const -{ - Path p(_path); - return icompare(p.getExtension(), "exe") == 0; -} - - -bool FileImpl::isFileImpl() const -{ - return !isDirectoryImpl() && !isDeviceImpl(); -} - - -bool FileImpl::isDirectoryImpl() const -{ - poco_assert (!_path.empty()); - - DWORD attr = GetFileAttributesW(_upath.c_str()); - if (attr == INVALID_FILE_ATTRIBUTES) - handleLastErrorImpl(_path); - return (attr & FILE_ATTRIBUTE_DIRECTORY) != 0; -} - - -bool FileImpl::isLinkImpl() const -{ - poco_assert (!_path.empty()); - - DWORD attr = GetFileAttributesW(_upath.c_str()); - if (attr == INVALID_FILE_ATTRIBUTES) - handleLastErrorImpl(_path); - return (attr & FILE_ATTRIBUTE_DIRECTORY) == 0 && (attr & FILE_ATTRIBUTE_REPARSE_POINT) != 0; -} - - -bool FileImpl::isDeviceImpl() const -{ - return - _path.compare(0, 4, "\\\\.\\") == 0 || - icompare(_path, "CON") == 0 || - icompare(_path, "PRN") == 0 || - icompare(_path, "AUX") == 0 || - icompare(_path, "NUL") == 0 || - ( (icompare(_path, 0, 3, "LPT") == 0 || icompare(_path, 0, 3, "COM") == 0) && - _path.size() == 4 && - _path[3] > 0x30 && - isdigit(_path[3]) - ); -} - - -bool FileImpl::isHiddenImpl() const -{ - poco_assert (!_path.empty()); - - DWORD attr = GetFileAttributesW(_upath.c_str()); - if (attr == INVALID_FILE_ATTRIBUTES) - handleLastErrorImpl(_path); - return (attr & FILE_ATTRIBUTE_HIDDEN) != 0; -} - - -Timestamp FileImpl::createdImpl() const -{ - poco_assert (!_path.empty()); - - WIN32_FILE_ATTRIBUTE_DATA fad; - if (GetFileAttributesExW(_upath.c_str(), GetFileExInfoStandard, &fad) == 0) - handleLastErrorImpl(_path); - return Timestamp::fromFileTimeNP(fad.ftCreationTime.dwLowDateTime, fad.ftCreationTime.dwHighDateTime); -} - - -Timestamp FileImpl::getLastModifiedImpl() const -{ - poco_assert (!_path.empty()); - - WIN32_FILE_ATTRIBUTE_DATA fad; - if (GetFileAttributesExW(_upath.c_str(), GetFileExInfoStandard, &fad) == 0) - handleLastErrorImpl(_path); - return Timestamp::fromFileTimeNP(fad.ftLastWriteTime.dwLowDateTime, fad.ftLastWriteTime.dwHighDateTime); -} - - -void FileImpl::setLastModifiedImpl(const Timestamp& ts) -{ - poco_assert (!_path.empty()); - - UInt32 low; - UInt32 high; - ts.toFileTimeNP(low, high); - FILETIME ft; - ft.dwLowDateTime = low; - ft.dwHighDateTime = high; - FileHandle fh(_path, _upath, FILE_WRITE_ATTRIBUTES, FILE_SHARE_READ | FILE_SHARE_WRITE, OPEN_EXISTING); - if (SetFileTime(fh.get(), 0, &ft, &ft) == 0) - handleLastErrorImpl(_path); -} - - -FileImpl::FileSizeImpl FileImpl::getSizeImpl() const -{ - poco_assert (!_path.empty()); - - WIN32_FILE_ATTRIBUTE_DATA fad; - if (GetFileAttributesExW(_upath.c_str(), GetFileExInfoStandard, &fad) == 0) - handleLastErrorImpl(_path); - LARGE_INTEGER li; - li.LowPart = fad.nFileSizeLow; - li.HighPart = fad.nFileSizeHigh; - return li.QuadPart; -} - - -void FileImpl::setSizeImpl(FileSizeImpl size) -{ - poco_assert (!_path.empty()); - - FileHandle fh(_path, _upath, GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE, OPEN_EXISTING); - LARGE_INTEGER li; - li.QuadPart = size; - if (SetFilePointer(fh.get(), li.LowPart, &li.HighPart, FILE_BEGIN) == INVALID_SET_FILE_POINTER) - handleLastErrorImpl(_path); - if (SetEndOfFile(fh.get()) == 0) - handleLastErrorImpl(_path); -} - - -void FileImpl::setWriteableImpl(bool flag) -{ - poco_assert (!_path.empty()); - - DWORD attr = GetFileAttributesW(_upath.c_str()); - if (attr == -1) - handleLastErrorImpl(_path); - if (flag) - attr &= ~FILE_ATTRIBUTE_READONLY; - else - attr |= FILE_ATTRIBUTE_READONLY; - if (SetFileAttributesW(_upath.c_str(), attr) == 0) - handleLastErrorImpl(_path); -} - - -void FileImpl::setExecutableImpl(bool flag) -{ - // not supported -} - - -void FileImpl::copyToImpl(const std::string& path) const -{ - poco_assert (!_path.empty()); - - std::wstring upath; - convertPath(path, upath); - if (CopyFileW(_upath.c_str(), upath.c_str(), FALSE) == 0) - handleLastErrorImpl(_path); -} - - -void FileImpl::renameToImpl(const std::string& path) -{ - poco_assert (!_path.empty()); - - std::wstring upath; - convertPath(path, upath); - if (MoveFileExW(_upath.c_str(), upath.c_str(), MOVEFILE_REPLACE_EXISTING) == 0) - handleLastErrorImpl(_path); -} - - -void FileImpl::linkToImpl(const std::string& path, int type) const -{ - poco_assert (!_path.empty()); - - std::wstring upath; - convertPath(path, upath); - - if (type == 0) - { - if (CreateHardLinkW(upath.c_str(), _upath.c_str(), NULL) == 0) - handleLastErrorImpl(_path); - } - else - { -#if _WIN32_WINNT >= 0x0600 && defined(SYMBOLIC_LINK_FLAG_DIRECTORY) - DWORD flags = 0; - if (isDirectoryImpl()) flags |= SYMBOLIC_LINK_FLAG_DIRECTORY; -#ifdef SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE - flags |= SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE; -#endif - if (CreateSymbolicLinkW(upath.c_str(), _upath.c_str(), flags) == 0) - handleLastErrorImpl(_path); -#else - throw Poco::NotImplementedException("Symbolic link support not available in used version of the Windows SDK"); -#endif - } -} - - -void FileImpl::removeImpl() -{ - poco_assert (!_path.empty()); - - if (isDirectoryImpl()) - { - if (RemoveDirectoryW(_upath.c_str()) == 0) - handleLastErrorImpl(_path); - } - else - { - if (DeleteFileW(_upath.c_str()) == 0) - handleLastErrorImpl(_path); - } -} - - -bool FileImpl::createFileImpl() -{ - poco_assert (!_path.empty()); - - HANDLE hFile = CreateFileW(_upath.c_str(), GENERIC_WRITE, 0, 0, CREATE_NEW, 0, 0); - if (hFile != INVALID_HANDLE_VALUE) - { - CloseHandle(hFile); - return true; - } - else if (GetLastError() == ERROR_FILE_EXISTS) - return false; - else - handleLastErrorImpl(_path); - return false; -} - - -bool FileImpl::createDirectoryImpl() -{ - poco_assert (!_path.empty()); - - if (existsImpl() && isDirectoryImpl()) - return false; - if (CreateDirectoryW(_upath.c_str(), 0) == 0) - handleLastErrorImpl(_path); - return true; -} - - -FileImpl::FileSizeImpl FileImpl::totalSpaceImpl() const -{ - poco_assert(!_path.empty()); - - ULARGE_INTEGER space; - if (!GetDiskFreeSpaceExW(_upath.c_str(), NULL, &space, NULL)) - handleLastErrorImpl(_path); - return space.QuadPart; -} - - -FileImpl::FileSizeImpl FileImpl::usableSpaceImpl() const -{ - poco_assert(!_path.empty()); - - ULARGE_INTEGER space; - if (!GetDiskFreeSpaceExW(_upath.c_str(), &space, NULL, NULL)) - handleLastErrorImpl(_path); - return space.QuadPart; -} - - -FileImpl::FileSizeImpl FileImpl::freeSpaceImpl() const -{ - poco_assert(!_path.empty()); - - ULARGE_INTEGER space; - if (!GetDiskFreeSpaceExW(_upath.c_str(), NULL, NULL, &space)) - handleLastErrorImpl(_path); - return space.QuadPart; -} - - -void FileImpl::handleLastErrorImpl(const std::string& path) -{ - DWORD err = GetLastError(); - switch (err) - { - case ERROR_FILE_NOT_FOUND: - throw FileNotFoundException(path, err); - case ERROR_PATH_NOT_FOUND: - case ERROR_BAD_NETPATH: - case ERROR_CANT_RESOLVE_FILENAME: - case ERROR_INVALID_DRIVE: - throw PathNotFoundException(path, err); - case ERROR_ACCESS_DENIED: - throw FileAccessDeniedException(path, err); - case ERROR_ALREADY_EXISTS: - case ERROR_FILE_EXISTS: - throw FileExistsException(path, err); - case ERROR_INVALID_NAME: - case ERROR_DIRECTORY: - case ERROR_FILENAME_EXCED_RANGE: - case ERROR_BAD_PATHNAME: - throw PathSyntaxException(path, err); - case ERROR_FILE_READ_ONLY: - throw FileReadOnlyException(path, err); - case ERROR_CANNOT_MAKE: - throw CreateFileException(path, err); - case ERROR_DIR_NOT_EMPTY: - throw DirectoryNotEmptyException(path, err); - case ERROR_WRITE_FAULT: - throw WriteFileException(path, err); - case ERROR_READ_FAULT: - throw ReadFileException(path, err); - case ERROR_SHARING_VIOLATION: - throw FileException("sharing violation", path, err); - case ERROR_LOCK_VIOLATION: - throw FileException("lock violation", path, err); - case ERROR_HANDLE_EOF: - throw ReadFileException("EOF reached", path, err); - case ERROR_HANDLE_DISK_FULL: - case ERROR_DISK_FULL: - throw WriteFileException("disk is full", path, err); - case ERROR_NEGATIVE_SEEK: - throw FileException("negative seek", path, err); - default: - throw FileException(path, err); - } -} - - -void FileImpl::convertPath(const std::string& utf8Path, std::wstring& utf16Path) -{ - UnicodeConverter::toUTF16(utf8Path, utf16Path); - if (utf16Path.size() > MAX_PATH - 12) // Note: CreateDirectory has a limit of MAX_PATH - 12 (room for 8.3 file name) - { - if (utf16Path[0] == '\\' || utf16Path[1] == ':') - { - if (utf16Path.compare(0, 4, L"\\\\?\\", 4) != 0) - { - if (utf16Path[1] == '\\') - utf16Path.insert(0, L"\\\\?\\UNC\\", 8); - else - utf16Path.insert(0, L"\\\\?\\", 4); - } - } - } -} - -} // namespace Poco diff --git a/base/poco/Foundation/src/File_WINCE.cpp b/base/poco/Foundation/src/File_WINCE.cpp deleted file mode 100644 index ea726c4bcf3..00000000000 --- a/base/poco/Foundation/src/File_WINCE.cpp +++ /dev/null @@ -1,441 +0,0 @@ -// -// File_WIN32U.cpp -// -// Library: Foundation -// Package: Filesystem -// Module: File -// -// Copyright (c) 2006-2010, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/File_WINCE.h" -#include "Poco/Exception.h" -#include "Poco/String.h" -#include "Poco/UnicodeConverter.h" -#include "Poco/Path.h" -#include "Poco/UnWindows.h" - - -namespace Poco { - - -class FileHandle -{ -public: - FileHandle(const std::string& path, const std::wstring& upath, DWORD access, DWORD share, DWORD disp) - { - _h = CreateFileW(upath.c_str(), access, share, 0, disp, 0, 0); - if (_h == INVALID_HANDLE_VALUE) - { - FileImpl::handleLastErrorImpl(path); - } - } - - ~FileHandle() - { - if (_h != INVALID_HANDLE_VALUE) CloseHandle(_h); - } - - HANDLE get() const - { - return _h; - } - -private: - HANDLE _h; -}; - - -FileImpl::FileImpl() -{ -} - - -FileImpl::FileImpl(const std::string& path): _path(path) -{ - std::string::size_type n = _path.size(); - if (n > 1 && (_path[n - 1] == '\\' || _path[n - 1] == '/') && !((n == 3 && _path[1]==':'))) - { - _path.resize(n - 1); - } - convertPath(_path, _upath); -} - - -FileImpl::~FileImpl() -{ -} - - -void FileImpl::swapImpl(FileImpl& file) -{ - std::swap(_path, file._path); - std::swap(_upath, file._upath); -} - - -void FileImpl::setPathImpl(const std::string& path) -{ - _path = path; - std::string::size_type n = _path.size(); - if (n > 1 && (_path[n - 1] == '\\' || _path[n - 1] == '/') && !((n == 3 && _path[1]==':'))) - { - _path.resize(n - 1); - } - convertPath(_path, _upath); -} - - -bool FileImpl::existsImpl() const -{ - poco_assert (!_path.empty()); - - DWORD attr = GetFileAttributesW(_upath.c_str()); - if (attr == INVALID_FILE_ATTRIBUTES) - { - switch (GetLastError()) - { - case ERROR_FILE_NOT_FOUND: - case ERROR_PATH_NOT_FOUND: - case ERROR_NOT_READY: - case ERROR_INVALID_DRIVE: - return false; - default: - handleLastErrorImpl(_path); - } - } - return true; -} - - -bool FileImpl::canReadImpl() const -{ - poco_assert (!_path.empty()); - - DWORD attr = GetFileAttributesW(_upath.c_str()); - if (attr == INVALID_FILE_ATTRIBUTES) - { - switch (GetLastError()) - { - case ERROR_ACCESS_DENIED: - return false; - default: - handleLastErrorImpl(_path); - } - } - return true; -} - - -bool FileImpl::canWriteImpl() const -{ - poco_assert (!_path.empty()); - - DWORD attr = GetFileAttributesW(_upath.c_str()); - if (attr == INVALID_FILE_ATTRIBUTES) - handleLastErrorImpl(_path); - return (attr & FILE_ATTRIBUTE_READONLY) == 0; -} - - -bool FileImpl::canExecuteImpl() const -{ - Path p(_path); - return icompare(p.getExtension(), "exe") == 0; -} - - -bool FileImpl::isFileImpl() const -{ - return !isDirectoryImpl() && !isDeviceImpl(); -} - - -bool FileImpl::isDirectoryImpl() const -{ - poco_assert (!_path.empty()); - - DWORD attr = GetFileAttributesW(_upath.c_str()); - if (attr == INVALID_FILE_ATTRIBUTES) - handleLastErrorImpl(_path); - return (attr & FILE_ATTRIBUTE_DIRECTORY) != 0; -} - - -bool FileImpl::isLinkImpl() const -{ - return false; -} - - -bool FileImpl::isDeviceImpl() const -{ - return false; -} - - -bool FileImpl::isHiddenImpl() const -{ - poco_assert (!_path.empty()); - - DWORD attr = GetFileAttributesW(_upath.c_str()); - if (attr == INVALID_FILE_ATTRIBUTES) - handleLastErrorImpl(_path); - return (attr & FILE_ATTRIBUTE_HIDDEN) != 0; -} - - -Timestamp FileImpl::createdImpl() const -{ - poco_assert (!_path.empty()); - - WIN32_FILE_ATTRIBUTE_DATA fad; - if (GetFileAttributesExW(_upath.c_str(), GetFileExInfoStandard, &fad) == 0) - handleLastErrorImpl(_path); - return Timestamp::fromFileTimeNP(fad.ftCreationTime.dwLowDateTime, fad.ftCreationTime.dwHighDateTime); -} - - -Timestamp FileImpl::getLastModifiedImpl() const -{ - poco_assert (!_path.empty()); - - WIN32_FILE_ATTRIBUTE_DATA fad; - if (GetFileAttributesExW(_upath.c_str(), GetFileExInfoStandard, &fad) == 0) - handleLastErrorImpl(_path); - return Timestamp::fromFileTimeNP(fad.ftLastWriteTime.dwLowDateTime, fad.ftLastWriteTime.dwHighDateTime); -} - - -void FileImpl::setLastModifiedImpl(const Timestamp& ts) -{ - poco_assert (!_path.empty()); - - UInt32 low; - UInt32 high; - ts.toFileTimeNP(low, high); - FILETIME ft; - ft.dwLowDateTime = low; - ft.dwHighDateTime = high; - FileHandle fh(_path, _upath, GENERIC_WRITE, FILE_SHARE_WRITE, OPEN_EXISTING); - if (SetFileTime(fh.get(), 0, &ft, &ft) == 0) - handleLastErrorImpl(_path); -} - - -FileImpl::FileSizeImpl FileImpl::getSizeImpl() const -{ - poco_assert (!_path.empty()); - - WIN32_FILE_ATTRIBUTE_DATA fad; - if (GetFileAttributesExW(_upath.c_str(), GetFileExInfoStandard, &fad) == 0) - handleLastErrorImpl(_path); - LARGE_INTEGER li; - li.LowPart = fad.nFileSizeLow; - li.HighPart = fad.nFileSizeHigh; - return li.QuadPart; -} - - -void FileImpl::setSizeImpl(FileSizeImpl size) -{ - poco_assert (!_path.empty()); - - FileHandle fh(_path, _upath, GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE, OPEN_EXISTING); - LARGE_INTEGER li; - li.QuadPart = size; - if (SetFilePointer(fh.get(), li.LowPart, &li.HighPart, FILE_BEGIN) == INVALID_SET_FILE_POINTER) - handleLastErrorImpl(_path); - if (SetEndOfFile(fh.get()) == 0) - handleLastErrorImpl(_path); -} - - -void FileImpl::setWriteableImpl(bool flag) -{ - poco_assert (!_path.empty()); - - DWORD attr = GetFileAttributesW(_upath.c_str()); - if (attr == -1) - handleLastErrorImpl(_path); - if (flag) - attr &= ~FILE_ATTRIBUTE_READONLY; - else - attr |= FILE_ATTRIBUTE_READONLY; - if (SetFileAttributesW(_upath.c_str(), attr) == 0) - handleLastErrorImpl(_path); -} - - -void FileImpl::setExecutableImpl(bool flag) -{ - // not supported -} - - -void FileImpl::copyToImpl(const std::string& path) const -{ - poco_assert (!_path.empty()); - - std::wstring upath; - convertPath(path, upath); - if (CopyFileW(_upath.c_str(), upath.c_str(), FALSE) == 0) - handleLastErrorImpl(_path); -} - - -void FileImpl::renameToImpl(const std::string& path) -{ - poco_assert (!_path.empty()); - - std::wstring upath; - convertPath(path, upath); - if (MoveFileW(_upath.c_str(), upath.c_str()) == 0) - handleLastErrorImpl(_path); -} - - -void FileImpl::linkToImpl(const std::string& path, int type) const -{ - throw Poco::NotImplementedException("File::linkTo() is not available on this platform"); -} - - -void FileImpl::removeImpl() -{ - poco_assert (!_path.empty()); - - if (isDirectoryImpl()) - { - if (RemoveDirectoryW(_upath.c_str()) == 0) - handleLastErrorImpl(_path); - } - else - { - if (DeleteFileW(_upath.c_str()) == 0) - handleLastErrorImpl(_path); - } -} - - -bool FileImpl::createFileImpl() -{ - poco_assert (!_path.empty()); - - HANDLE hFile = CreateFileW(_upath.c_str(), GENERIC_WRITE, 0, 0, CREATE_NEW, 0, 0); - if (hFile != INVALID_HANDLE_VALUE) - { - CloseHandle(hFile); - return true; - } - else if (GetLastError() == ERROR_FILE_EXISTS) - return false; - else - handleLastErrorImpl(_path); - return false; -} - - -bool FileImpl::createDirectoryImpl() -{ - poco_assert (!_path.empty()); - - if (existsImpl() && isDirectoryImpl()) - return false; - if (CreateDirectoryW(_upath.c_str(), 0) == 0) - handleLastErrorImpl(_path); - return true; -} - - -FileImpl::FileSizeImpl FileImpl::totalSpaceImpl() const -{ - poco_assert(!_path.empty()); - - ULARGE_INTEGER space; - if (!GetDiskFreeSpaceExW(_upath.c_str(), NULL, &space, NULL)) - handleLastErrorImpl(_path); - return space.QuadPart; -} - - -FileImpl::FileSizeImpl FileImpl::usableSpaceImpl() const -{ - poco_assert(!_path.empty()); - - ULARGE_INTEGER space; - if (!GetDiskFreeSpaceExW(_upath.c_str(), &space, NULL, NULL)) - handleLastErrorImpl(_path); - return space.QuadPart; -} - - -FileImpl::FileSizeImpl FileImpl::freeSpaceImpl() const -{ - poco_assert(!_path.empty()); - - ULARGE_INTEGER space; - if (!GetDiskFreeSpaceExW(_upath.c_str(), NULL, NULL, &space)) - handleLastErrorImpl(_path); - return space.QuadPart; -} - - -void FileImpl::handleLastErrorImpl(const std::string& path) -{ - switch (GetLastError()) - { - case ERROR_FILE_NOT_FOUND: - throw FileNotFoundException(path); - case ERROR_PATH_NOT_FOUND: - case ERROR_BAD_NETPATH: - case ERROR_CANT_RESOLVE_FILENAME: - case ERROR_INVALID_DRIVE: - throw PathNotFoundException(path); - case ERROR_ACCESS_DENIED: - throw FileAccessDeniedException(path); - case ERROR_ALREADY_EXISTS: - case ERROR_FILE_EXISTS: - throw FileExistsException(path); - case ERROR_INVALID_NAME: - case ERROR_DIRECTORY: - case ERROR_FILENAME_EXCED_RANGE: - case ERROR_BAD_PATHNAME: - throw PathSyntaxException(path); - case ERROR_FILE_READ_ONLY: - throw FileReadOnlyException(path); - case ERROR_CANNOT_MAKE: - throw CreateFileException(path); - case ERROR_DIR_NOT_EMPTY: - throw DirectoryNotEmptyException(path); - case ERROR_WRITE_FAULT: - throw WriteFileException(path); - case ERROR_READ_FAULT: - throw ReadFileException(path); - case ERROR_SHARING_VIOLATION: - throw FileException("sharing violation", path); - case ERROR_LOCK_VIOLATION: - throw FileException("lock violation", path); - case ERROR_HANDLE_EOF: - throw ReadFileException("EOF reached", path); - case ERROR_HANDLE_DISK_FULL: - case ERROR_DISK_FULL: - throw WriteFileException("disk is full", path); - case ERROR_NEGATIVE_SEEK: - throw FileException("negative seek", path); - default: - throw FileException(path); - } -} - - -void FileImpl::convertPath(const std::string& utf8Path, std::wstring& utf16Path) -{ - UnicodeConverter::toUTF16(utf8Path, utf16Path); -} - -} // namespace Poco diff --git a/base/poco/Foundation/src/LogFile_WIN32.cpp b/base/poco/Foundation/src/LogFile_WIN32.cpp deleted file mode 100644 index df1d403c44f..00000000000 --- a/base/poco/Foundation/src/LogFile_WIN32.cpp +++ /dev/null @@ -1,125 +0,0 @@ -// -// LogFile_WIN32.cpp -// -// Library: Foundation -// Package: Logging -// Module: LogFile -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/LogFile_WIN32.h" -#include "Poco/File.h" -#include "Poco/Exception.h" - - -namespace Poco { - - -LogFileImpl::LogFileImpl(const std::string& path): _path(path), _hFile(INVALID_HANDLE_VALUE) -{ - File file(path); - if (file.exists()) - { - if (0 == sizeImpl()) - _creationDate = file.getLastModified(); - else - _creationDate = file.created(); - } -} - - -LogFileImpl::~LogFileImpl() -{ - CloseHandle(_hFile); -} - - -void LogFileImpl::writeImpl(const std::string& text, bool flush) -{ - if (INVALID_HANDLE_VALUE == _hFile) createFile(); - - DWORD bytesWritten; - BOOL res = WriteFile(_hFile, text.data(), (DWORD) text.size(), &bytesWritten, NULL); - if (!res) throw WriteFileException(_path); - res = WriteFile(_hFile, "\r\n", 2, &bytesWritten, NULL); - if (!res) throw WriteFileException(_path); - if (flush) - { - res = FlushFileBuffers(_hFile); - if (!res) throw WriteFileException(_path); - } -} - -void LogFileImpl::writeBinaryImpl(const char * data, size_t size, bool flush) -{ - if (INVALID_HANDLE_VALUE == _hFile) createFile(); - - DWORD bytesWritten; - BOOL res = WriteFile(_hFile, text.data(), (DWORD) text.size(), &bytesWritten, NULL); - if (!res) throw WriteFileException(_path); - if (flush) - { - res = FlushFileBuffers(_hFile); - if (!res) throw WriteFileException(_path); - } -} - - -UInt64 LogFileImpl::sizeImpl() const -{ - if (INVALID_HANDLE_VALUE == _hFile) - { - File file(_path); - if (file.exists()) return file.getSize(); - else return 0; - } - - LARGE_INTEGER li; - li.HighPart = 0; - li.LowPart = SetFilePointer(_hFile, 0, &li.HighPart, FILE_CURRENT); - return li.QuadPart; -} - - -Timestamp LogFileImpl::creationDateImpl() const -{ - return _creationDate; -} - - -const std::string& LogFileImpl::pathImpl() const -{ - return _path; -} - - -void LogFileImpl::createFile() -{ - _hFile = CreateFileA(_path.c_str(), GENERIC_WRITE, FILE_SHARE_READ, NULL, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); - if (_hFile == INVALID_HANDLE_VALUE) throw OpenFileException(_path); - SetFilePointer(_hFile, 0, 0, FILE_END); - // There seems to be a strange "optimization" in the Windows NTFS - // filesystem that causes it to reuse directory entries of deleted - // files. Example: - // 1. create a file named "test.dat" - // note the file's creation date - // 2. delete the file "test.dat" - // 3. wait a few seconds - // 4. create a file named "test.dat" - // the new file will have the same creation - // date as the old one. - // We work around this bug by taking the file's - // modification date as a reference when the - // file is empty. - if (sizeImpl() == 0) - _creationDate = File(_path).getLastModified(); - else - _creationDate = File(_path).created(); -} - -} // namespace Poco diff --git a/base/poco/Foundation/src/LogFile_WIN32U.cpp b/base/poco/Foundation/src/LogFile_WIN32U.cpp deleted file mode 100644 index c85e6086a4a..00000000000 --- a/base/poco/Foundation/src/LogFile_WIN32U.cpp +++ /dev/null @@ -1,130 +0,0 @@ -// -// LogFile_WIN32U.cpp -// -// Library: Foundation -// Package: Logging -// Module: LogFile -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/LogFile_WIN32U.h" -#include "Poco/File.h" -#include "Poco/Exception.h" -#include "Poco/UnicodeConverter.h" - - -namespace Poco { - - -LogFileImpl::LogFileImpl(const std::string& path): _path(path), _hFile(INVALID_HANDLE_VALUE) -{ - File file(path); - if (file.exists()) - { - if (0 == sizeImpl()) - _creationDate = file.getLastModified(); - else - _creationDate = file.created(); - } -} - - -LogFileImpl::~LogFileImpl() -{ - CloseHandle(_hFile); -} - - -void LogFileImpl::writeImpl(const std::string& text, bool flush) -{ - if (INVALID_HANDLE_VALUE == _hFile) createFile(); - - DWORD bytesWritten; - BOOL res = WriteFile(_hFile, text.data(), (DWORD) text.size(), &bytesWritten, NULL); - if (!res) throw WriteFileException(_path); - res = WriteFile(_hFile, "\r\n", 2, &bytesWritten, NULL); - if (!res) throw WriteFileException(_path); - if (flush) - { - res = FlushFileBuffers(_hFile); - if (!res) throw WriteFileException(_path); - } -} - -void LogFileImpl::writeBinaryImpl(const char * data, size_t size, bool flush) -{ - if (INVALID_HANDLE_VALUE == _hFile) createFile(); - - DWORD bytesWritten; - BOOL res = WriteFile(_hFile, data, (DWORD) size, &bytesWritten, NULL); - if (!res) throw WriteFileException(_path); - if (flush) - { - res = FlushFileBuffers(_hFile); - if (!res) throw WriteFileException(_path); - } -} - - -UInt64 LogFileImpl::sizeImpl() const -{ - if (INVALID_HANDLE_VALUE == _hFile) - { - File file(_path); - if (file.exists()) return file.getSize(); - else return 0; - } - - LARGE_INTEGER li; - li.HighPart = 0; - li.LowPart = SetFilePointer(_hFile, 0, &li.HighPart, FILE_CURRENT); - return li.QuadPart; -} - - -Timestamp LogFileImpl::creationDateImpl() const -{ - return _creationDate; -} - - -const std::string& LogFileImpl::pathImpl() const -{ - return _path; -} - - -void LogFileImpl::createFile() -{ - std::wstring upath; - FileImpl::convertPath(_path, upath); - - _hFile = CreateFileW(upath.c_str(), GENERIC_WRITE, FILE_SHARE_READ, NULL, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); - if (_hFile == INVALID_HANDLE_VALUE) throw OpenFileException(_path); - SetFilePointer(_hFile, 0, 0, FILE_END); - // There seems to be a strange "optimization" in the Windows NTFS - // filesystem that causes it to reuse directory entries of deleted - // files. Example: - // 1. create a file named "test.dat" - // note the file's creation date - // 2. delete the file "test.dat" - // 3. wait a few seconds - // 4. create a file named "test.dat" - // the new file will have the same creation - // date as the old one. - // We work around this bug by taking the file's - // modification date as a reference when the - // file is empty. - if (sizeImpl() == 0) - _creationDate = File(_path).getLastModified(); - else - _creationDate = File(_path).created(); -} - - -} // namespace Poco diff --git a/base/poco/Foundation/src/Mutex_VX.cpp b/base/poco/Foundation/src/Mutex_VX.cpp deleted file mode 100644 index a4b32acbbf6..00000000000 --- a/base/poco/Foundation/src/Mutex_VX.cpp +++ /dev/null @@ -1,68 +0,0 @@ -// -// Mutex_VX.cpp -// -// Library: Foundation -// Package: Threading -// Module: Mutex -// -// Copyright (c) 2004-2011, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Mutex_VX.h" -#include - - -namespace Poco { - - -MutexImpl::MutexImpl() -{ - _sem = semMCreate(SEM_INVERSION_SAFE | SEM_Q_PRIORITY); - if (_sem == 0) - throw Poco::SystemException("cannot create mutex"); -} - - -MutexImpl::MutexImpl(bool fast) -{ - if (fast) - { - _sem = semBCreate(SEM_Q_PRIORITY, SEM_FULL); - } - else - { - _sem = semMCreate(SEM_INVERSION_SAFE | SEM_Q_PRIORITY); - } - if (_sem == 0) - throw Poco::SystemException("cannot create mutex"); -} - - -MutexImpl::~MutexImpl() -{ - semDelete(_sem); -} - - -bool MutexImpl::tryLockImpl(long milliseconds) -{ - int ticks = milliseconds*sysClkRateGet()/1000; - return semTake(_sem, ticks) == OK; -} - - -FastMutexImpl::FastMutexImpl(): MutexImpl(true) -{ -} - - -FastMutexImpl::~FastMutexImpl() -{ -} - - -} // namespace Poco diff --git a/base/poco/Foundation/src/Mutex_WIN32.cpp b/base/poco/Foundation/src/Mutex_WIN32.cpp deleted file mode 100644 index 446f4affcef..00000000000 --- a/base/poco/Foundation/src/Mutex_WIN32.cpp +++ /dev/null @@ -1,59 +0,0 @@ -// -// Mutex_WIN32.cpp -// -// Library: Foundation -// Package: Threading -// Module: Mutex -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Mutex_WIN32.h" -#include "Poco/Timestamp.h" - - -namespace Poco { - - -MutexImpl::MutexImpl() -{ - // the fct has a boolean return value under WInnNt/2000/XP but not on Win98 - // the return only checks if the input address of &_cs was valid, so it is safe to omit it - InitializeCriticalSectionAndSpinCount(&_cs, 4000); -} - - -MutexImpl::~MutexImpl() -{ - DeleteCriticalSection(&_cs); -} - - -bool MutexImpl::tryLockImpl(long milliseconds) -{ - const int sleepMillis = 5; - Timestamp now; - Timestamp::TimeDiff diff(Timestamp::TimeDiff(milliseconds)*1000); - do - { - try - { - if (TryEnterCriticalSection(&_cs) == TRUE) - return true; - } - catch (...) - { - throw SystemException("cannot lock mutex"); - } - Sleep(sleepMillis); - } - while (!now.isElapsed(diff)); - return false; -} - - -} // namespace Poco diff --git a/base/poco/Foundation/src/Mutex_WINCE.cpp b/base/poco/Foundation/src/Mutex_WINCE.cpp deleted file mode 100644 index 90ea6956d88..00000000000 --- a/base/poco/Foundation/src/Mutex_WINCE.cpp +++ /dev/null @@ -1,80 +0,0 @@ -// -// Mutex_WINCE.cpp -// -// Library: Foundation -// Package: Threading -// Module: Mutex -// -// Copyright (c) 2004-2010, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Mutex_WINCE.h" - - -namespace Poco { - - -MutexImpl::MutexImpl() -{ - _mutex = CreateMutexW(NULL, FALSE, NULL); - if (!_mutex) throw SystemException("cannot create mutex"); -} - - -MutexImpl::~MutexImpl() -{ - CloseHandle(_mutex); -} - - -void MutexImpl::lockImpl() -{ - switch (WaitForSingleObject(_mutex, INFINITE)) - { - case WAIT_OBJECT_0: - return; - default: - throw SystemException("cannot lock mutex"); - } -} - - -bool MutexImpl::tryLockImpl() -{ - switch (WaitForSingleObject(_mutex, 0)) - { - case WAIT_TIMEOUT: - return false; - case WAIT_OBJECT_0: - return true; - default: - throw SystemException("cannot lock mutex"); - } -} - - -bool MutexImpl::tryLockImpl(long milliseconds) -{ - switch (WaitForSingleObject(_mutex, milliseconds + 1)) - { - case WAIT_TIMEOUT: - return false; - case WAIT_OBJECT_0: - return true; - default: - throw SystemException("cannot lock mutex"); - } -} - - -void MutexImpl::unlockImpl() -{ - ReleaseMutex(_mutex); -} - - -} // namespace Poco diff --git a/base/poco/Foundation/src/NamedEvent_WIN32.cpp b/base/poco/Foundation/src/NamedEvent_WIN32.cpp deleted file mode 100644 index 0aee72c5fb9..00000000000 --- a/base/poco/Foundation/src/NamedEvent_WIN32.cpp +++ /dev/null @@ -1,61 +0,0 @@ -// -// NamedEvent_WIN32.cpp -// -// Library: Foundation -// Package: Processes -// Module: NamedEvent -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/NamedEvent_WIN32.h" -#include "Poco/Error.h" -#include "Poco/Exception.h" -#include "Poco/Format.h" - - -namespace Poco { - - -NamedEventImpl::NamedEventImpl(const std::string& name): - _name(name) -{ - _event = CreateEventA(NULL, FALSE, FALSE, _name.c_str()); - if (!_event) - { - DWORD dwRetVal = GetLastError(); - throw SystemException(format("cannot create named event %s [Error %d: %s]", _name, (int)dwRetVal, Error::getMessage(dwRetVal))); - } -} - - -NamedEventImpl::~NamedEventImpl() -{ - CloseHandle(_event); -} - - -void NamedEventImpl::setImpl() -{ - if (!SetEvent(_event)) - throw SystemException("cannot signal named event", _name); -} - - -void NamedEventImpl::waitImpl() -{ - switch (WaitForSingleObject(_event, INFINITE)) - { - case WAIT_OBJECT_0: - return; - default: - throw SystemException("wait for named event failed", _name); - } -} - - -} // namespace Poco diff --git a/base/poco/Foundation/src/NamedEvent_WIN32U.cpp b/base/poco/Foundation/src/NamedEvent_WIN32U.cpp deleted file mode 100644 index 8a7ad1c1820..00000000000 --- a/base/poco/Foundation/src/NamedEvent_WIN32U.cpp +++ /dev/null @@ -1,63 +0,0 @@ -// -// NamedEvent_WIN32.cpp -// -// Library: Foundation -// Package: Processes -// Module: NamedEvent -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/NamedEvent_WIN32U.h" -#include "Poco/Error.h" -#include "Poco/Exception.h" -#include "Poco/Format.h" -#include "Poco/UnicodeConverter.h" - - -namespace Poco { - - -NamedEventImpl::NamedEventImpl(const std::string& name): - _name(name) -{ - UnicodeConverter::toUTF16(_name, _uname); - _event = CreateEventW(NULL, FALSE, FALSE, _uname.c_str()); - if (!_event) - { - DWORD dwRetVal = GetLastError(); - throw SystemException(format("cannot create named event %s [Error %d: %s]", _name, (int)dwRetVal, Error::getMessage(dwRetVal))); - } -} - - -NamedEventImpl::~NamedEventImpl() -{ - CloseHandle(_event); -} - - -void NamedEventImpl::setImpl() -{ - if (!SetEvent(_event)) - throw SystemException("cannot signal named event", _name); -} - - -void NamedEventImpl::waitImpl() -{ - switch (WaitForSingleObject(_event, INFINITE)) - { - case WAIT_OBJECT_0: - return; - default: - throw SystemException("wait for named event failed", _name); - } -} - - -} // namespace Poco diff --git a/base/poco/Foundation/src/NamedMutex_WIN32.cpp b/base/poco/Foundation/src/NamedMutex_WIN32.cpp deleted file mode 100644 index 6d2a5c383f6..00000000000 --- a/base/poco/Foundation/src/NamedMutex_WIN32.cpp +++ /dev/null @@ -1,73 +0,0 @@ -// -// NamedMutex_WIN32.cpp -// -// Library: Foundation -// Package: Processes -// Module: NamedMutex -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/NamedMutex_WIN32.h" -#include "Poco/Exception.h" - - -namespace Poco { - - -NamedMutexImpl::NamedMutexImpl(const std::string& name): - _name(name) -{ - _mutex = CreateMutexA(NULL, FALSE, _name.c_str()); - if (!_mutex) - throw SystemException("cannot create named mutex", _name); -} - - -NamedMutexImpl::~NamedMutexImpl() -{ - CloseHandle(_mutex); -} - - -void NamedMutexImpl::lockImpl() -{ - switch (WaitForSingleObject(_mutex, INFINITE)) - { - case WAIT_OBJECT_0: - return; - case WAIT_ABANDONED: - throw SystemException("cannot lock named mutex (abadoned)", _name); - default: - throw SystemException("cannot lock named mutex", _name); - } -} - - -bool NamedMutexImpl::tryLockImpl() -{ - switch (WaitForSingleObject(_mutex, 0)) - { - case WAIT_OBJECT_0: - return true; - case WAIT_TIMEOUT: - return false; - case WAIT_ABANDONED: - throw SystemException("cannot lock named mutex (abadoned)", _name); - default: - throw SystemException("cannot lock named mutex", _name); - } -} - - -void NamedMutexImpl::unlockImpl() -{ - ReleaseMutex(_mutex); -} - - -} // namespace Poco diff --git a/base/poco/Foundation/src/NamedMutex_WIN32U.cpp b/base/poco/Foundation/src/NamedMutex_WIN32U.cpp deleted file mode 100644 index 1358658b961..00000000000 --- a/base/poco/Foundation/src/NamedMutex_WIN32U.cpp +++ /dev/null @@ -1,75 +0,0 @@ -// -// NamedMutex_WIN32.cpp -// -// Library: Foundation -// Package: Processes -// Module: NamedMutex -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/NamedMutex_WIN32U.h" -#include "Poco/Exception.h" -#include "Poco/UnicodeConverter.h" - - -namespace Poco { - - -NamedMutexImpl::NamedMutexImpl(const std::string& name): - _name(name) -{ - UnicodeConverter::toUTF16(_name, _uname); - _mutex = CreateMutexW(NULL, FALSE, _uname.c_str()); - if (!_mutex) - throw SystemException("cannot create named mutex", _name); -} - - -NamedMutexImpl::~NamedMutexImpl() -{ - CloseHandle(_mutex); -} - - -void NamedMutexImpl::lockImpl() -{ - switch (WaitForSingleObject(_mutex, INFINITE)) - { - case WAIT_OBJECT_0: - return; - case WAIT_ABANDONED: - throw SystemException("cannot lock named mutex (abadoned)", _name); - default: - throw SystemException("cannot lock named mutex", _name); - } -} - - -bool NamedMutexImpl::tryLockImpl() -{ - switch (WaitForSingleObject(_mutex, 0)) - { - case WAIT_OBJECT_0: - return true; - case WAIT_TIMEOUT: - return false; - case WAIT_ABANDONED: - throw SystemException("cannot lock named mutex (abadoned)", _name); - default: - throw SystemException("cannot lock named mutex", _name); - } -} - - -void NamedMutexImpl::unlockImpl() -{ - ReleaseMutex(_mutex); -} - - -} // namespace Poco diff --git a/base/poco/Foundation/src/Path_WIN32.cpp b/base/poco/Foundation/src/Path_WIN32.cpp deleted file mode 100644 index 9d8b357a319..00000000000 --- a/base/poco/Foundation/src/Path_WIN32.cpp +++ /dev/null @@ -1,203 +0,0 @@ -// -// Path_WIN32.cpp -// -// Library: Foundation -// Package: Filesystem -// Module: Path -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Path_WIN32.h" -#include "Poco/Environment_WIN32.h" -#include "Poco/UnWindows.h" - - -namespace Poco { - - -std::string PathImpl::currentImpl() -{ - char buffer[MAX_PATH]; - DWORD n = GetCurrentDirectoryA(sizeof(buffer), buffer); - if (n > 0 && n < sizeof(buffer)) - { - std::string result(buffer, n); - if (result[n - 1] != '\\') - result.append("\\"); - return result; - } - else throw SystemException("Cannot get current directory"); -} - - -std::string PathImpl::systemImpl() -{ - char buffer[MAX_PATH]; - DWORD n = GetSystemDirectoryA(buffer, sizeof(buffer)); - if (n > 0 && n < sizeof(buffer)) - { - std::string result(buffer, n); - if (result[n - 1] != '\\') - result.append("\\"); - return result; - } - else throw SystemException("Cannot get system directory"); -} - - -std::string PathImpl::homeImpl() -{ - std::string result; - if (EnvironmentImpl::hasImpl("USERPROFILE")) - { - result = EnvironmentImpl::getImpl("USERPROFILE"); - } - else if (EnvironmentImpl::hasImpl("HOMEDRIVE") && EnvironmentImpl::hasImpl("HOMEPATH")) - { - result = EnvironmentImpl::getImpl("HOMEDRIVE"); - result.append(EnvironmentImpl::getImpl("HOMEPATH")); - } - else - { - result = systemImpl(); - } - - std::string::size_type n = result.size(); - if (n > 0 && result[n - 1] != '\\') - result.append("\\"); - return result; -} - - -std::string PathImpl::configHomeImpl() -{ - std::string result; - - // if APPDATA environment variable not exist, return home directory instead - try - { - result = EnvironmentImpl::getImpl("APPDATA"); - } - catch (NotFoundException&) - { - result = homeImpl(); - } - - std::string::size_type n = result.size(); - if (n > 0 && result[n - 1] != '\\') - result.append("\\"); - return result; -} - - -std::string PathImpl::dataHomeImpl() -{ - std::string result; - - // if LOCALAPPDATA environment variable not exist, return config home instead - try - { - result = EnvironmentImpl::getImpl("LOCALAPPDATA"); - } - catch (NotFoundException&) - { - result = configHomeImpl(); - } - - std::string::size_type n = result.size(); - if (n > 0 && result[n - 1] != '\\') - result.append("\\"); - return result; -} - - -std::string PathImpl::cacheHomeImpl() -{ - return tempImpl(); -} - - -std::string PathImpl::tempHomeImpl() -{ - return tempImpl(); -} - - -std::string PathImpl::tempImpl() -{ - char buffer[MAX_PATH]; - DWORD n = GetTempPathA(sizeof(buffer), buffer); - if (n > 0 && n < sizeof(buffer)) - { - n = GetLongPathNameA(buffer, buffer, static_cast(sizeof buffer)); - if (n <= 0) throw SystemException("Cannot get temporary directory long path name"); - std::string result(buffer, n); - if (result[n - 1] != '\\') - result.append("\\"); - return result; - } - else throw SystemException("Cannot get temporary directory"); -} - - -std::string PathImpl::configImpl() -{ - std::string result; - - // if PROGRAMDATA environment variable not exist, return system directory instead - try - { - result = EnvironmentImpl::getImpl("PROGRAMDATA"); - } - catch (NotFoundException&) - { - result = systemImpl(); - } - - std::string::size_type n = result.size(); - if (n > 0 && result[n - 1] != '\\') - result.append("\\"); - return result; -} - - -std::string PathImpl::nullImpl() -{ - return "NUL:"; -} - - -std::string PathImpl::expandImpl(const std::string& path) -{ - char buffer[MAX_PATH]; - DWORD n = ExpandEnvironmentStringsA(path.c_str(), buffer, sizeof(buffer)); - if (n > 0 && n < sizeof(buffer)) - return std::string(buffer, n - 1); - else - return path; -} - - -void PathImpl::listRootsImpl(std::vector& roots) -{ - roots.clear(); - char buffer[128]; - DWORD n = GetLogicalDriveStrings(sizeof(buffer) - 1, buffer); - char* it = buffer; - char* end = buffer + (n > sizeof(buffer) ? sizeof(buffer) : n); - while (it < end) - { - std::string dev; - while (it < end && *it) dev += *it++; - roots.push_back(dev); - ++it; - } -} - - -} // namespace Poco diff --git a/base/poco/Foundation/src/Path_WIN32U.cpp b/base/poco/Foundation/src/Path_WIN32U.cpp deleted file mode 100644 index ac757d6eab8..00000000000 --- a/base/poco/Foundation/src/Path_WIN32U.cpp +++ /dev/null @@ -1,222 +0,0 @@ -// -// Path_WIN32U.cpp -// -// Library: Foundation -// Package: Filesystem -// Module: Path -// -// Copyright (c) 2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Path_WIN32U.h" -#include "Poco/Environment_WIN32.h" -#include "Poco/UnicodeConverter.h" -#include "Poco/Buffer.h" -#include "Poco/Exception.h" -#include "Poco/UnWindows.h" - - -namespace Poco { - - -std::string PathImpl::currentImpl() -{ - std::string result; - DWORD len = GetCurrentDirectoryW(0, NULL); - if (len > 0) - { - Buffer buffer(len); - DWORD n = GetCurrentDirectoryW(len, buffer.begin()); - if (n > 0 && n <= len) - { - UnicodeConverter::toUTF8(buffer.begin(), result); - if (result[result.size() - 1] != '\\') - result.append("\\"); - return result; - } - } - throw SystemException("Cannot get current directory"); -} - - -std::string PathImpl::systemImpl() -{ - Buffer buffer(MAX_PATH_LEN); - DWORD n = GetSystemDirectoryW(buffer.begin(), static_cast(buffer.size())); - if (n > 0) - { - n = GetLongPathNameW(buffer.begin(), buffer.begin(), static_cast(buffer.size())); - if (n <= 0) throw SystemException("Cannot get system directory long path name"); - std::string result; - UnicodeConverter::toUTF8(buffer.begin(), result); - if (result[result.size() - 1] != '\\') result.append("\\"); - return result; - } - throw SystemException("Cannot get temporary directory path"); -} - - -std::string PathImpl::homeImpl() -{ - std::string result; - if (EnvironmentImpl::hasImpl("USERPROFILE")) - { - result = EnvironmentImpl::getImpl("USERPROFILE"); - } - else if (EnvironmentImpl::hasImpl("HOMEDRIVE") && EnvironmentImpl::hasImpl("HOMEPATH")) - { - result = EnvironmentImpl::getImpl("HOMEDRIVE"); - result.append(EnvironmentImpl::getImpl("HOMEPATH")); - } - else - { - result = systemImpl(); - } - - std::string::size_type n = result.size(); - if (n > 0 && result[n - 1] != '\\') - result.append("\\"); - return result; -} - - -std::string PathImpl::configHomeImpl() -{ - std::string result; - - // if APPDATA environment variable no exist, return home directory instead - try - { - result = EnvironmentImpl::getImpl("APPDATA"); - } - catch (NotFoundException&) - { - result = homeImpl(); - } - - std::string::size_type n = result.size(); - if (n > 0 && result[n - 1] != '\\') - result.append("\\"); - return result; -} - - -std::string PathImpl::dataHomeImpl() -{ - std::string result; - - // if LOCALAPPDATA environment variable no exist, return config home instead - try - { - result = EnvironmentImpl::getImpl("LOCALAPPDATA"); - } - catch (NotFoundException&) - { - result = configHomeImpl(); - } - - std::string::size_type n = result.size(); - if (n > 0 && result[n - 1] != '\\') - result.append("\\"); - return result; -} - - -std::string PathImpl::cacheHomeImpl() -{ - return tempImpl(); -} - - -std::string PathImpl::tempHomeImpl() -{ - return tempImpl(); -} - - -std::string PathImpl::tempImpl() -{ - Buffer buffer(MAX_PATH_LEN); - DWORD n = GetTempPathW(static_cast(buffer.size()), buffer.begin()); - if (n > 0) - { - n = GetLongPathNameW(buffer.begin(), buffer.begin(), static_cast(buffer.size())); - if (n <= 0) throw SystemException("Cannot get temporary directory long path name"); - std::string result; - UnicodeConverter::toUTF8(buffer.begin(), result); - if (result[result.size() - 1] != '\\') - result.append("\\"); - return result; - } - throw SystemException("Cannot get temporary directory path"); -} - - -std::string PathImpl::configImpl() -{ - std::string result; - - // if PROGRAMDATA environment variable not exist, return system directory instead - try - { - result = EnvironmentImpl::getImpl("PROGRAMDATA"); - } - catch (NotFoundException&) - { - result = systemImpl(); - } - - std::string::size_type n = result.size(); - if (n > 0 && result[n - 1] != '\\') - result.append("\\"); - return result; -} - - -std::string PathImpl::nullImpl() -{ - return "NUL:"; -} - - -std::string PathImpl::expandImpl(const std::string& path) -{ - std::wstring upath; - UnicodeConverter::toUTF16(path, upath); - Buffer buffer(MAX_PATH_LEN); - DWORD n = ExpandEnvironmentStringsW(upath.c_str(), buffer.begin(), static_cast(buffer.size())); - if (n > 0 && n < buffer.size() - 1) - { - buffer[n + 1] = 0; - std::string result; - UnicodeConverter::toUTF8(buffer.begin(), result); - return result; - } - else return path; -} - - -void PathImpl::listRootsImpl(std::vector& roots) -{ - roots.clear(); - wchar_t buffer[128]; - DWORD n = GetLogicalDriveStringsW(sizeof(buffer)/sizeof(wchar_t) - 1, buffer); - wchar_t* it = buffer; - wchar_t* end = buffer + (n > sizeof(buffer) ? sizeof(buffer) : n); - while (it < end) - { - std::wstring udev; - while (it < end && *it) udev += *it++; - std::string dev; - UnicodeConverter::toUTF8(udev, dev); - roots.push_back(dev); - ++it; - } -} - - -} // namespace Poco diff --git a/base/poco/Foundation/src/Path_WINCE.cpp b/base/poco/Foundation/src/Path_WINCE.cpp deleted file mode 100644 index 750f18dda6f..00000000000 --- a/base/poco/Foundation/src/Path_WINCE.cpp +++ /dev/null @@ -1,144 +0,0 @@ -// -// Path_WIN32U.cpp -// -// Library: Foundation -// Package: Filesystem -// Module: Path -// -// Copyright (c) 2006-2010, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Path_WINCE.h" -#include "Poco/Environment_WINCE.h" -#include "Poco/UnicodeConverter.h" -#include "Poco/Buffer.h" -#include "Poco/Environment.h" -#include "Poco/Exception.h" -#include "Poco/UnWindows.h" - - -namespace Poco { - - -std::string PathImpl::currentImpl() -{ - return("\\"); -} - -std::string PathImpl::homeImpl() -{ - return("\\"); -} - -std::string PathImpl::configHomeImpl() -{ - return homeImpl(); -} - -std::string PathImpl::dataHomeImpl() -{ - return homeImpl(); -} - -std::string PathImpl::cacheHomeImpl() -{ - return homeImpl(); -} - - -std::string PathImpl::tempHomeImpl() -{ - return tempImpl(); -} - - -std::string PathImpl::configImpl() -{ - return("\\"); -} - -std::string PathImpl::systemImpl() -{ - return("\\"); -} - - -std::string PathImpl::nullImpl() -{ - return "NUL:"; -} - - -std::string PathImpl::tempImpl() -{ - return "\\Temp\\"; -} - - -std::string PathImpl::expandImpl(const std::string& path) -{ - std::string result; - std::string::const_iterator it = path.begin(); - std::string::const_iterator end = path.end(); - while (it != end) - { - if (*it == '%') - { - ++it; - if (it != end && *it == '%') - { - result += '%'; - } - else - { - std::string var; - while (it != end && *it != '%') var += *it++; - if (it != end) ++it; - result += Environment::get(var, ""); - } - } - else result += *it++; - } - return result; -} - - -void PathImpl::listRootsImpl(std::vector& roots) -{ - roots.clear(); - roots.push_back("\\"); - - WIN32_FIND_DATAW fd; - HANDLE hFind = FindFirstFileW(L"\\*.*", &fd); - if (hFind != INVALID_HANDLE_VALUE) - { - do - { - if ((fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) && - (fd.dwFileAttributes & FILE_ATTRIBUTE_TEMPORARY)) - { - std::wstring name(fd.cFileName); - name += L"\\Vol:"; - HANDLE h = CreateFileW(name.c_str(), GENERIC_READ|GENERIC_WRITE, 0, NULL, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); - if (h != INVALID_HANDLE_VALUE) - { - // its a device volume - CloseHandle(h); - std::string name; - UnicodeConverter::toUTF8(fd.cFileName, name); - std::string root = "\\" + name; - roots.push_back(root); - } - } - } - while (FindNextFileW(hFind, &fd)); - FindClose(hFind); - } -} - - -} // namespace Poco diff --git a/base/poco/Foundation/src/PipeImpl_WIN32.cpp b/base/poco/Foundation/src/PipeImpl_WIN32.cpp deleted file mode 100644 index 87dbbbf3e51..00000000000 --- a/base/poco/Foundation/src/PipeImpl_WIN32.cpp +++ /dev/null @@ -1,97 +0,0 @@ -// -// PipeImpl_WIN32.cpp -// -// Library: Foundation -// Package: Processes -// Module: PipeImpl -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/PipeImpl_WIN32.h" -#include "Poco/Exception.h" - - -namespace Poco { - - -PipeImpl::PipeImpl() -{ - SECURITY_ATTRIBUTES attr; - attr.nLength = sizeof(attr); - attr.lpSecurityDescriptor = NULL; - attr.bInheritHandle = FALSE; - - if (!CreatePipe(&_readHandle, &_writeHandle, &attr, 0)) - throw CreateFileException("anonymous pipe"); -} - - -PipeImpl::~PipeImpl() -{ - closeRead(); - closeWrite(); -} - - -int PipeImpl::writeBytes(const void* buffer, int length) -{ - poco_assert (_writeHandle != INVALID_HANDLE_VALUE); - - DWORD bytesWritten = 0; - if (!WriteFile(_writeHandle, buffer, length, &bytesWritten, NULL)) - throw WriteFileException("anonymous pipe"); - return bytesWritten; -} - - -int PipeImpl::readBytes(void* buffer, int length) -{ - poco_assert (_readHandle != INVALID_HANDLE_VALUE); - - DWORD bytesRead = 0; - BOOL ok = ReadFile(_readHandle, buffer, length, &bytesRead, NULL); - if (ok || GetLastError() == ERROR_BROKEN_PIPE) - return bytesRead; - else - throw ReadFileException("anonymous pipe"); -} - - -PipeImpl::Handle PipeImpl::readHandle() const -{ - return _readHandle; -} - - -PipeImpl::Handle PipeImpl::writeHandle() const -{ - return _writeHandle; -} - - -void PipeImpl::closeRead() -{ - if (_readHandle != INVALID_HANDLE_VALUE) - { - CloseHandle(_readHandle); - _readHandle = INVALID_HANDLE_VALUE; - } -} - - -void PipeImpl::closeWrite() -{ - if (_writeHandle != INVALID_HANDLE_VALUE) - { - CloseHandle(_writeHandle); - _writeHandle = INVALID_HANDLE_VALUE; - } -} - - -} // namespace Poco diff --git a/base/poco/Foundation/src/Process_VX.cpp b/base/poco/Foundation/src/Process_VX.cpp deleted file mode 100644 index 699feb9be71..00000000000 --- a/base/poco/Foundation/src/Process_VX.cpp +++ /dev/null @@ -1,100 +0,0 @@ -// -// Process_VX.cpp -// -// Library: Foundation -// Package: Processes -// Module: Process -// -// Copyright (c) 2004-2011, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Process_VX.h" -#include "Poco/Exception.h" - - -namespace Poco { - - -// -// ProcessHandleImpl -// -ProcessHandleImpl::ProcessHandleImpl(int pid): - _pid(pid) -{ -} - - -ProcessHandleImpl::~ProcessHandleImpl() -{ -} - - -int ProcessHandleImpl::id() const -{ - return _pid; -} - - -int ProcessHandleImpl::wait() const -{ - throw Poco::NotImplementedException("Process::wait()"); -} - - -// -// ProcessImpl -// -ProcessImpl::PIDImpl ProcessImpl::idImpl() -{ - return 0; -} - - -void ProcessImpl::timesImpl(long& userTime, long& kernelTime) -{ - userTime = 0; - kernelTime = 0; -} - - -ProcessHandleImpl* ProcessImpl::launchImpl(const std::string& command, const ArgsImpl& args, const std::string& initialDirectory,Pipe* inPipe, Pipe* outPipe, Pipe* errPipe, const EnvImpl& env) -{ - throw Poco::NotImplementedException("Process::launch()"); -} - - -void ProcessImpl::killImpl(ProcessHandleImpl& handle) -{ - throw Poco::NotImplementedException("Process::kill()"); -} - - -void ProcessImpl::killImpl(PIDImpl pid) -{ - throw Poco::NotImplementedException("Process::kill()"); -} - - -bool ProcessImpl::isRunningImpl(const ProcessHandleImpl& handle) -{ - throw Poco::NotImplementedException("Process::is_running()"); -} - - -bool ProcessImpl::isRunningImpl(PIDImpl pid) -{ - throw Poco::NotImplementedException("Process::is_running()"); -} - - -void ProcessImpl::requestTerminationImpl(PIDImpl pid) -{ - throw Poco::NotImplementedException("Process::requestTermination()"); -} - - -} // namespace Poco diff --git a/base/poco/Foundation/src/Process_WIN32.cpp b/base/poco/Foundation/src/Process_WIN32.cpp deleted file mode 100644 index d698a9726e8..00000000000 --- a/base/poco/Foundation/src/Process_WIN32.cpp +++ /dev/null @@ -1,349 +0,0 @@ -// -// Process_WIN32.cpp -// -// Library: Foundation -// Package: Processes -// Module: Process -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Process_WIN32.h" -#include "Poco/Exception.h" -#include "Poco/NumberFormatter.h" -#include "Poco/NamedEvent.h" -#include "Poco/Pipe.h" - - -namespace Poco { - - -// -// ProcessHandleImpl -// -ProcessHandleImpl::ProcessHandleImpl(HANDLE hProcess, UInt32 pid) : - _hProcess(hProcess), - _pid(pid) -{ -} - - -ProcessHandleImpl::~ProcessHandleImpl() -{ - closeHandle(); -} - - -void ProcessHandleImpl::closeHandle() -{ - if (_hProcess) - { - CloseHandle(_hProcess); - _hProcess = NULL; - } -} - - -UInt32 ProcessHandleImpl::id() const -{ - return _pid; -} - - -HANDLE ProcessHandleImpl::process() const -{ - return _hProcess; -} - - -int ProcessHandleImpl::wait() const -{ - DWORD rc = WaitForSingleObject(_hProcess, INFINITE); - if (rc != WAIT_OBJECT_0) - throw SystemException("Wait failed for process", NumberFormatter::format(_pid)); - - DWORD exitCode; - if (GetExitCodeProcess(_hProcess, &exitCode) == 0) - throw SystemException("Cannot get exit code for process", NumberFormatter::format(_pid)); - - return exitCode; -} - - -// -// ProcessImpl -// -ProcessImpl::PIDImpl ProcessImpl::idImpl() -{ - return GetCurrentProcessId(); -} - - -void ProcessImpl::timesImpl(long& userTime, long& kernelTime) -{ - FILETIME ftCreation; - FILETIME ftExit; - FILETIME ftKernel; - FILETIME ftUser; - - if (GetProcessTimes(GetCurrentProcess(), &ftCreation, &ftExit, &ftKernel, &ftUser) != 0) - { - ULARGE_INTEGER time; - time.LowPart = ftKernel.dwLowDateTime; - time.HighPart = ftKernel.dwHighDateTime; - kernelTime = long(time.QuadPart / 10000000L); - time.LowPart = ftUser.dwLowDateTime; - time.HighPart = ftUser.dwHighDateTime; - userTime = long(time.QuadPart / 10000000L); - } - else - { - userTime = kernelTime = -1; - } -} - - -static bool argNeedsEscaping(const std::string& arg) -{ - bool containsQuotableChar = std::string::npos != arg.find_first_of(" \t\n\v\""); - // Assume args that start and end with quotes are already quoted and do not require further quoting. - // There is probably code out there written before launch() escaped the arguments that does its own - // escaping of arguments. This ensures we do not interfere with those arguments. - bool isAlreadyQuoted = arg.size() > 1 && '\"' == arg[0] && '\"' == arg[arg.size() - 1]; - return containsQuotableChar && !isAlreadyQuoted; -} - - -// Based on code from https://blogs.msdn.microsoft.com/twistylittlepassagesallalike/2011/04/23/everyone-quotes-command-line-arguments-the-wrong-way/ -static std::string escapeArg(const std::string& arg) -{ - if (argNeedsEscaping(arg)) - { - std::string quotedArg("\""); - for (std::string::const_iterator it = arg.begin(); ; ++it) - { - unsigned backslashCount = 0; - while (it != arg.end() && '\\' == *it) - { - ++it; - ++backslashCount; - } - - if (it == arg.end()) - { - quotedArg.append(2 * backslashCount, '\\'); - break; - } - else if ('"' == *it) - { - quotedArg.append(2 * backslashCount + 1, '\\'); - quotedArg.push_back('"'); - } - else - { - quotedArg.append(backslashCount, '\\'); - quotedArg.push_back(*it); - } - } - quotedArg.push_back('"'); - return quotedArg; - } - else - { - return arg; - } -} - - -ProcessHandleImpl* ProcessImpl::launchImpl(const std::string& command, const ArgsImpl& args, const std::string& initialDirectory, Pipe* inPipe, Pipe* outPipe, Pipe* errPipe, const EnvImpl& env) -{ - std::string commandLine = command; - for (ArgsImpl::const_iterator it = args.begin(); it != args.end(); ++it) - { - commandLine.append(" "); - commandLine.append(escapeArg(*it)); - } - - STARTUPINFOA startupInfo; - GetStartupInfoA(&startupInfo); // take defaults from current process - startupInfo.cb = sizeof(STARTUPINFOA); - startupInfo.lpReserved = NULL; - startupInfo.lpDesktop = NULL; - startupInfo.lpTitle = NULL; - startupInfo.dwFlags = STARTF_FORCEOFFFEEDBACK; - startupInfo.cbReserved2 = 0; - startupInfo.lpReserved2 = NULL; - - HANDLE hProc = GetCurrentProcess(); - bool mustInheritHandles = false; - if (inPipe) - { - DuplicateHandle(hProc, inPipe->readHandle(), hProc, &startupInfo.hStdInput, 0, TRUE, DUPLICATE_SAME_ACCESS); - mustInheritHandles = true; - inPipe->close(Pipe::CLOSE_READ); - } - else if (GetStdHandle(STD_INPUT_HANDLE)) - { - DuplicateHandle(hProc, GetStdHandle(STD_INPUT_HANDLE), hProc, &startupInfo.hStdInput, 0, TRUE, DUPLICATE_SAME_ACCESS); - mustInheritHandles = true; - } - else - { - startupInfo.hStdInput = 0; - } - // outPipe may be the same as errPipe, so we duplicate first and close later. - if (outPipe) - { - DuplicateHandle(hProc, outPipe->writeHandle(), hProc, &startupInfo.hStdOutput, 0, TRUE, DUPLICATE_SAME_ACCESS); - mustInheritHandles = true; - } - else if (GetStdHandle(STD_OUTPUT_HANDLE)) - { - DuplicateHandle(hProc, GetStdHandle(STD_OUTPUT_HANDLE), hProc, &startupInfo.hStdOutput, 0, TRUE, DUPLICATE_SAME_ACCESS); - mustInheritHandles = true; - } - else - { - startupInfo.hStdOutput = 0; - } - if (errPipe) - { - DuplicateHandle(hProc, errPipe->writeHandle(), hProc, &startupInfo.hStdError, 0, TRUE, DUPLICATE_SAME_ACCESS); - mustInheritHandles = true; - } - else if (GetStdHandle(STD_ERROR_HANDLE)) - { - DuplicateHandle(hProc, GetStdHandle(STD_ERROR_HANDLE), hProc, &startupInfo.hStdError, 0, TRUE, DUPLICATE_SAME_ACCESS); - mustInheritHandles = true; - } - else - { - startupInfo.hStdError = 0; - } - if (outPipe) outPipe->close(Pipe::CLOSE_WRITE); - if (errPipe) errPipe->close(Pipe::CLOSE_WRITE); - - if (mustInheritHandles) - { - startupInfo.dwFlags |= STARTF_USESTDHANDLES; - } - - const char* workingDirectory = initialDirectory.empty() ? 0 : initialDirectory.c_str(); - - const char* pEnv = 0; - std::vector envChars; - if (!env.empty()) - { - envChars = getEnvironmentVariablesBuffer(env); - pEnv = &envChars[0]; - } - - PROCESS_INFORMATION processInfo; - DWORD creationFlags = GetConsoleWindow() ? 0 : CREATE_NO_WINDOW; - BOOL rc = CreateProcessA( - NULL, - const_cast(commandLine.c_str()), - NULL, // processAttributes - NULL, // threadAttributes - mustInheritHandles, - creationFlags, - (LPVOID)pEnv, - workingDirectory, - &startupInfo, - &processInfo - ); - if (startupInfo.hStdInput) CloseHandle(startupInfo.hStdInput); - if (startupInfo.hStdOutput) CloseHandle(startupInfo.hStdOutput); - if (startupInfo.hStdError) CloseHandle(startupInfo.hStdError); - if (rc) - { - CloseHandle(processInfo.hThread); - return new ProcessHandleImpl(processInfo.hProcess, processInfo.dwProcessId); - } - else throw SystemException("Cannot launch process", command); -} - - -void ProcessImpl::killImpl(ProcessHandleImpl& handle) -{ - if (handle.process()) - { - if (TerminateProcess(handle.process(), 0) == 0) - { - handle.closeHandle(); - throw SystemException("cannot kill process"); - } - handle.closeHandle(); - } -} - -void ProcessImpl::killImpl(PIDImpl pid) -{ - HANDLE hProc = OpenProcess(PROCESS_TERMINATE, FALSE, pid); - if (hProc) - { - if (TerminateProcess(hProc, 0) == 0) - { - CloseHandle(hProc); - throw SystemException("cannot kill process"); - } - CloseHandle(hProc); - } - else - { - switch (GetLastError()) - { - case ERROR_ACCESS_DENIED: - throw NoPermissionException("cannot kill process"); - case ERROR_NOT_FOUND: - throw NotFoundException("cannot kill process"); - case ERROR_INVALID_PARAMETER: - throw NotFoundException("cannot kill process"); - default: - throw SystemException("cannot kill process"); - } - } -} - - -bool ProcessImpl::isRunningImpl(const ProcessHandleImpl& handle) -{ - bool result = true; - DWORD exitCode; - BOOL rc = GetExitCodeProcess(handle.process(), &exitCode); - if (!rc || exitCode != STILL_ACTIVE) result = false; - return result; -} - - -bool ProcessImpl::isRunningImpl(PIDImpl pid) -{ - HANDLE hProc = OpenProcess(PROCESS_QUERY_INFORMATION, FALSE, pid); - bool result = true; - DWORD exitCode; - BOOL rc = GetExitCodeProcess(hProc, &exitCode); - if (!rc || exitCode != STILL_ACTIVE) result = false; - return result; -} - - -void ProcessImpl::requestTerminationImpl(PIDImpl pid) -{ - NamedEvent ev(terminationEventName(pid)); - ev.set(); -} - - -std::string ProcessImpl::terminationEventName(PIDImpl pid) -{ - std::string evName("POCOTRM"); - NumberFormatter::appendHex(evName, pid, 8); - return evName; -} - - -} // namespace Poco diff --git a/base/poco/Foundation/src/Process_WIN32U.cpp b/base/poco/Foundation/src/Process_WIN32U.cpp deleted file mode 100644 index 2a81a8dd468..00000000000 --- a/base/poco/Foundation/src/Process_WIN32U.cpp +++ /dev/null @@ -1,371 +0,0 @@ -// -// Process_WIN32U.cpp -// -// Library: Foundation -// Package: Processes -// Module: Process -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Process_WIN32U.h" -#include "Poco/Exception.h" -#include "Poco/NumberFormatter.h" -#include "Poco/NamedEvent.h" -#include "Poco/UnicodeConverter.h" -#include "Poco/Pipe.h" -#include "Poco/File.h" -#include "Poco/Path.h" -#include "Poco/String.h" - - -namespace Poco { - - -// -// ProcessHandleImpl -// -ProcessHandleImpl::ProcessHandleImpl(HANDLE hProcess, UInt32 pid) : - _hProcess(hProcess), - _pid(pid) -{ -} - - -ProcessHandleImpl::~ProcessHandleImpl() -{ - closeHandle(); -} - - -void ProcessHandleImpl::closeHandle() -{ - if (_hProcess) - { - CloseHandle(_hProcess); - _hProcess = NULL; - } -} - - -UInt32 ProcessHandleImpl::id() const -{ - return _pid; -} - - -HANDLE ProcessHandleImpl::process() const -{ - return _hProcess; -} - - -int ProcessHandleImpl::wait() const -{ - DWORD rc = WaitForSingleObject(_hProcess, INFINITE); - if (rc != WAIT_OBJECT_0) - throw SystemException("Wait failed for process", NumberFormatter::format(_pid)); - - DWORD exitCode; - if (GetExitCodeProcess(_hProcess, &exitCode) == 0) - throw SystemException("Cannot get exit code for process", NumberFormatter::format(_pid)); - - return exitCode; -} - - -// -// ProcessImpl -// -ProcessImpl::PIDImpl ProcessImpl::idImpl() -{ - return GetCurrentProcessId(); -} - - -void ProcessImpl::timesImpl(long& userTime, long& kernelTime) -{ - FILETIME ftCreation; - FILETIME ftExit; - FILETIME ftKernel; - FILETIME ftUser; - - if (GetProcessTimes(GetCurrentProcess(), &ftCreation, &ftExit, &ftKernel, &ftUser) != 0) - { - ULARGE_INTEGER time; - time.LowPart = ftKernel.dwLowDateTime; - time.HighPart = ftKernel.dwHighDateTime; - kernelTime = long(time.QuadPart / 10000000L); - time.LowPart = ftUser.dwLowDateTime; - time.HighPart = ftUser.dwHighDateTime; - userTime = long(time.QuadPart / 10000000L); - } - else - { - userTime = kernelTime = -1; - } -} - - -static bool argNeedsEscaping(const std::string& arg) -{ - bool containsQuotableChar = std::string::npos != arg.find_first_of(" \t\n\v\""); - // Assume args that start and end with quotes are already quoted and do not require further quoting. - // There is probably code out there written before launch() escaped the arguments that does its own - // escaping of arguments. This ensures we do not interfere with those arguments. - bool isAlreadyQuoted = arg.size() > 1 && '\"' == arg[0] && '\"' == arg[arg.size() - 1]; - return containsQuotableChar && !isAlreadyQuoted; -} - - -// Based on code from https://blogs.msdn.microsoft.com/twistylittlepassagesallalike/2011/04/23/everyone-quotes-command-line-arguments-the-wrong-way/ -static std::string escapeArg(const std::string& arg) -{ - if (argNeedsEscaping(arg)) - { - std::string quotedArg("\""); - for (std::string::const_iterator it = arg.begin(); ; ++it) - { - unsigned backslashCount = 0; - while (it != arg.end() && '\\' == *it) - { - ++it; - ++backslashCount; - } - - if (it == arg.end()) - { - quotedArg.append(2 * backslashCount, '\\'); - break; - } - else if ('"' == *it) - { - quotedArg.append(2 * backslashCount + 1, '\\'); - quotedArg.push_back('"'); - } - else - { - quotedArg.append(backslashCount, '\\'); - quotedArg.push_back(*it); - } - } - quotedArg.push_back('"'); - return quotedArg; - } - else - { - return arg; - } -} - - -ProcessHandleImpl* ProcessImpl::launchImpl(const std::string& command, const ArgsImpl& args, const std::string& initialDirectory, Pipe* inPipe, Pipe* outPipe, Pipe* errPipe, const EnvImpl& env) -{ - std::string commandLine = escapeArg(command); - for (ArgsImpl::const_iterator it = args.begin(); it != args.end(); ++it) - { - commandLine.append(" "); - commandLine.append(escapeArg(*it)); - } - - std::wstring ucommandLine; - UnicodeConverter::toUTF16(commandLine, ucommandLine); - - const wchar_t* applicationName = 0; - std::wstring uapplicationName; - if (command.size() > MAX_PATH) - { - Poco::Path p(command); - if (p.isAbsolute()) - { - UnicodeConverter::toUTF16(command, uapplicationName); - if (p.getExtension().empty()) uapplicationName += L".EXE"; - applicationName = uapplicationName.c_str(); - } - } - - STARTUPINFOW startupInfo; - GetStartupInfoW(&startupInfo); // take defaults from current process - startupInfo.cb = sizeof(STARTUPINFOW); - startupInfo.lpReserved = NULL; - startupInfo.lpDesktop = NULL; - startupInfo.lpTitle = NULL; - startupInfo.dwFlags = STARTF_FORCEOFFFEEDBACK; - startupInfo.cbReserved2 = 0; - startupInfo.lpReserved2 = NULL; - - HANDLE hProc = GetCurrentProcess(); - bool mustInheritHandles = false; - if (inPipe) - { - DuplicateHandle(hProc, inPipe->readHandle(), hProc, &startupInfo.hStdInput, 0, TRUE, DUPLICATE_SAME_ACCESS); - mustInheritHandles = true; - inPipe->close(Pipe::CLOSE_READ); - } - else if (GetStdHandle(STD_INPUT_HANDLE)) - { - DuplicateHandle(hProc, GetStdHandle(STD_INPUT_HANDLE), hProc, &startupInfo.hStdInput, 0, TRUE, DUPLICATE_SAME_ACCESS); - mustInheritHandles = true; - } - else - { - startupInfo.hStdInput = 0; - } - // outPipe may be the same as errPipe, so we duplicate first and close later. - if (outPipe) - { - DuplicateHandle(hProc, outPipe->writeHandle(), hProc, &startupInfo.hStdOutput, 0, TRUE, DUPLICATE_SAME_ACCESS); - mustInheritHandles = true; - } - else if (GetStdHandle(STD_OUTPUT_HANDLE)) - { - DuplicateHandle(hProc, GetStdHandle(STD_OUTPUT_HANDLE), hProc, &startupInfo.hStdOutput, 0, TRUE, DUPLICATE_SAME_ACCESS); - mustInheritHandles = true; - } - else - { - startupInfo.hStdOutput = 0; - } - if (errPipe) - { - DuplicateHandle(hProc, errPipe->writeHandle(), hProc, &startupInfo.hStdError, 0, TRUE, DUPLICATE_SAME_ACCESS); - mustInheritHandles = true; - } - else if (GetStdHandle(STD_ERROR_HANDLE)) - { - DuplicateHandle(hProc, GetStdHandle(STD_ERROR_HANDLE), hProc, &startupInfo.hStdError, 0, TRUE, DUPLICATE_SAME_ACCESS); - mustInheritHandles = true; - } - else - { - startupInfo.hStdError = 0; - } - if (outPipe) outPipe->close(Pipe::CLOSE_WRITE); - if (errPipe) errPipe->close(Pipe::CLOSE_WRITE); - - if (mustInheritHandles) - { - startupInfo.dwFlags |= STARTF_USESTDHANDLES; - } - - std::wstring uinitialDirectory; - UnicodeConverter::toUTF16(initialDirectory, uinitialDirectory); - const wchar_t* workingDirectory = uinitialDirectory.empty() ? 0 : uinitialDirectory.c_str(); - - const char* pEnv = 0; - std::vector envChars; - if (!env.empty()) - { - envChars = getEnvironmentVariablesBuffer(env); - pEnv = &envChars[0]; - } - - PROCESS_INFORMATION processInfo; - DWORD creationFlags = GetConsoleWindow() ? 0 : CREATE_NO_WINDOW; - BOOL rc = CreateProcessW( - applicationName, - const_cast(ucommandLine.c_str()), - NULL, // processAttributes - NULL, // threadAttributes - mustInheritHandles, - creationFlags, - (LPVOID)pEnv, - workingDirectory, - &startupInfo, - &processInfo - ); - if (startupInfo.hStdInput) CloseHandle(startupInfo.hStdInput); - if (startupInfo.hStdOutput) CloseHandle(startupInfo.hStdOutput); - if (startupInfo.hStdError) CloseHandle(startupInfo.hStdError); - if (rc) - { - CloseHandle(processInfo.hThread); - return new ProcessHandleImpl(processInfo.hProcess, processInfo.dwProcessId); - } - else throw SystemException("Cannot launch process", command); -} - - -void ProcessImpl::killImpl(ProcessHandleImpl& handle) -{ - if (handle.process()) - { - if (TerminateProcess(handle.process(), 0) == 0) - { - handle.closeHandle(); - throw SystemException("cannot kill process"); - } - handle.closeHandle(); - } -} - -void ProcessImpl::killImpl(PIDImpl pid) -{ - HANDLE hProc = OpenProcess(PROCESS_TERMINATE, FALSE, pid); - if (hProc) - { - if (TerminateProcess(hProc, 0) == 0) - { - CloseHandle(hProc); - throw SystemException("cannot kill process"); - } - CloseHandle(hProc); - } - else - { - switch (GetLastError()) - { - case ERROR_ACCESS_DENIED: - throw NoPermissionException("cannot kill process"); - case ERROR_NOT_FOUND: - throw NotFoundException("cannot kill process"); - case ERROR_INVALID_PARAMETER: - throw NotFoundException("cannot kill process"); - default: - throw SystemException("cannot kill process"); - } - } -} - - -bool ProcessImpl::isRunningImpl(const ProcessHandleImpl& handle) -{ - bool result = true; - DWORD exitCode; - BOOL rc = GetExitCodeProcess(handle.process(), &exitCode); - if (!rc || exitCode != STILL_ACTIVE) result = false; - return result; -} - - -bool ProcessImpl::isRunningImpl(PIDImpl pid) -{ - HANDLE hProc = OpenProcess(PROCESS_QUERY_INFORMATION, FALSE, pid); - bool result = true; - DWORD exitCode; - BOOL rc = GetExitCodeProcess(hProc, &exitCode); - if (!rc || exitCode != STILL_ACTIVE) result = false; - return result; -} - - -void ProcessImpl::requestTerminationImpl(PIDImpl pid) -{ - NamedEvent ev(terminationEventName(pid)); - ev.set(); -} - - -std::string ProcessImpl::terminationEventName(PIDImpl pid) -{ - std::string evName("POCOTRM"); - NumberFormatter::appendHex(evName, pid, 8); - return evName; -} - - -} // namespace Poco diff --git a/base/poco/Foundation/src/Process_WINCE.cpp b/base/poco/Foundation/src/Process_WINCE.cpp deleted file mode 100644 index e640f3be3d5..00000000000 --- a/base/poco/Foundation/src/Process_WINCE.cpp +++ /dev/null @@ -1,223 +0,0 @@ -// -// Process_WINCE.cpp -// -// Library: Foundation -// Package: Processes -// Module: Process -// -// Copyright (c) 2004-2010, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Process_WINCE.h" -#include "Poco/Exception.h" -#include "Poco/NumberFormatter.h" -#include "Poco/NamedEvent.h" -#include "Poco/UnicodeConverter.h" -#include "Poco/Pipe.h" - - -namespace Poco { - - -// -// ProcessHandleImpl -// -ProcessHandleImpl::ProcessHandleImpl(HANDLE hProcess, UInt32 pid): - _hProcess(hProcess), - _pid(pid) -{ -} - - -ProcessHandleImpl::~ProcessHandleImpl() -{ - closeHandle(); -} - -void ProcessHandleImpl::closeHandle() -{ - if (_hProcess) - { - CloseHandle(_hProcess); - _hProcess = NULL; - } -} - -UInt32 ProcessHandleImpl::id() const -{ - return _pid; -} - - -HANDLE ProcessHandleImpl::process() const -{ - return _hProcess; -} - - -int ProcessHandleImpl::wait() const -{ - DWORD rc = WaitForSingleObject(_hProcess, INFINITE); - if (rc != WAIT_OBJECT_0) - throw SystemException("Wait failed for process", NumberFormatter::format(_pid)); - - DWORD exitCode; - if (GetExitCodeProcess(_hProcess, &exitCode) == 0) - throw SystemException("Cannot get exit code for process", NumberFormatter::format(_pid)); - - return exitCode; -} - - -// -// ProcessImpl -// -ProcessImpl::PIDImpl ProcessImpl::idImpl() -{ - return GetCurrentProcessId(); -} - - -void ProcessImpl::timesImpl(long& userTime, long& kernelTime) -{ - FILETIME ftCreation; - FILETIME ftExit; - FILETIME ftKernel; - FILETIME ftUser; - - if (GetThreadTimes(GetCurrentThread(), &ftCreation, &ftExit, &ftKernel, &ftUser) != 0) - { - ULARGE_INTEGER time; - time.LowPart = ftKernel.dwLowDateTime; - time.HighPart = ftKernel.dwHighDateTime; - kernelTime = long(time.QuadPart/10000000L); - time.LowPart = ftUser.dwLowDateTime; - time.HighPart = ftUser.dwHighDateTime; - userTime = long(time.QuadPart/10000000L); - } - else - { - userTime = kernelTime = -1; - } -} - - -ProcessHandleImpl* ProcessImpl::launchImpl(const std::string& command, const ArgsImpl& args, const std::string& initialDirectory, Pipe* inPipe, Pipe* outPipe, Pipe* errPipe, const EnvImpl& env) -{ - std::wstring ucommand; - UnicodeConverter::toUTF16(command, ucommand); - - std::string commandLine; - for (ArgsImpl::const_iterator it = args.begin(); it != args.end(); ++it) - { - if (it != args.begin()) commandLine.append(" "); - commandLine.append(*it); - } - - std::wstring ucommandLine; - UnicodeConverter::toUTF16(commandLine, ucommandLine); - - PROCESS_INFORMATION processInfo; - BOOL rc = CreateProcessW( - ucommand.c_str(), - const_cast(ucommandLine.c_str()), - NULL, - NULL, - FALSE, - 0, - NULL, - NULL, - NULL/*&startupInfo*/, - &processInfo - ); - - if (rc) - { - CloseHandle(processInfo.hThread); - return new ProcessHandleImpl(processInfo.hProcess, processInfo.dwProcessId); - } - else throw SystemException("Cannot launch process", command); -} - - -void ProcessImpl::killImpl(ProcessHandleImpl& handle) -{ - if (handle.process()) - { - if (TerminateProcess(handle.process(), 0) == 0) - { - handle.closeHandle(); - throw SystemException("cannot kill process"); - } - handle.closeHandle(); - } -} - - -void ProcessImpl::killImpl(PIDImpl pid) -{ - HANDLE hProc = OpenProcess(PROCESS_TERMINATE, FALSE, pid); - if (hProc) - { - if (TerminateProcess(hProc, 0) == 0) - { - CloseHandle(hProc); - throw SystemException("cannot kill process"); - } - CloseHandle(hProc); - } - else - { - switch (GetLastError()) - { - case ERROR_ACCESS_DENIED: - throw NoPermissionException("cannot kill process"); - case ERROR_NOT_FOUND: - throw NotFoundException("cannot kill process"); - default: - throw SystemException("cannot kill process"); - } - } -} - - -bool ProcessImpl::isRunningImpl(const ProcessHandleImpl& handle) -{ - bool result = true; - DWORD exitCode; - BOOL rc = GetExitCodeProcess(handle.process(), &exitCode); - if (!rc || exitCode != STILL_ACTIVE) result = false; - return result; -} - - -bool ProcessImpl::isRunningImpl(PIDImpl pid) -{ - HANDLE hProc = OpenProcess(PROCESS_QUERY_INFORMATION, FALSE, pid); - bool result = true; - DWORD exitCode; - BOOL rc = GetExitCodeProcess(hProc, &exitCode); - if (!rc || exitCode != STILL_ACTIVE) result = false; - return result;} - - -void ProcessImpl::requestTerminationImpl(PIDImpl pid) -{ - NamedEvent ev(terminationEventName(pid)); - ev.set(); -} - - -std::string ProcessImpl::terminationEventName(PIDImpl pid) -{ - std::string evName("POCOTRM"); - NumberFormatter::appendHex(evName, pid, 8); - return evName; -} - - -} // namespace Poco diff --git a/base/poco/Foundation/src/RWLock_VX.cpp b/base/poco/Foundation/src/RWLock_VX.cpp deleted file mode 100644 index 6b7eb166b79..00000000000 --- a/base/poco/Foundation/src/RWLock_VX.cpp +++ /dev/null @@ -1,41 +0,0 @@ -// -// RWLock_VX.cpp -// -// Library: Foundation -// Package: Threading -// Module: RWLock -// -// Copyright (c) 2004-2011, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/RWLock_VX.h" -#include - - -namespace Poco { - - -RWLockImpl::RWLockImpl() -{ - pthread_mutexattr_t attr; - pthread_mutexattr_init(&attr); - if (pthread_mutex_init(&_mutex, &attr)) - { - pthread_mutexattr_destroy(&attr); - throw SystemException("cannot create mutex"); - } - pthread_mutexattr_destroy(&attr); -} - - -RWLockImpl::~RWLockImpl() -{ - pthread_mutex_destroy(&_mutex); -} - - -} // namespace Poco diff --git a/base/poco/Foundation/src/RWLock_WIN32.cpp b/base/poco/Foundation/src/RWLock_WIN32.cpp deleted file mode 100644 index 973b0ae3c5a..00000000000 --- a/base/poco/Foundation/src/RWLock_WIN32.cpp +++ /dev/null @@ -1,207 +0,0 @@ -// -// RWLock_WIN32.cpp -// -// Library: Foundation -// Package: Threading -// Module: RWLock -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/RWLock_WIN32.h" - - -namespace Poco { - - -RWLockImpl::RWLockImpl(): _readers(0), _writersWaiting(0), _writers(0) -{ - _mutex = CreateMutexW(NULL, FALSE, NULL); - if (_mutex == NULL) - throw SystemException("cannot create reader/writer lock"); - - _readEvent = CreateEventW(NULL, TRUE, TRUE, NULL); - if (_readEvent == NULL) - throw SystemException("cannot create reader/writer lock"); - - _writeEvent = CreateEventW(NULL, TRUE, TRUE, NULL); - if (_writeEvent == NULL) - throw SystemException("cannot create reader/writer lock"); -} - - -RWLockImpl::~RWLockImpl() -{ - CloseHandle(_mutex); - CloseHandle(_readEvent); - CloseHandle(_writeEvent); -} - - -inline void RWLockImpl::addWriter() -{ - switch (WaitForSingleObject(_mutex, INFINITE)) - { - case WAIT_OBJECT_0: - if (++_writersWaiting == 1) ResetEvent(_readEvent); - ReleaseMutex(_mutex); - break; - default: - throw SystemException("cannot lock reader/writer lock"); - } -} - - -inline void RWLockImpl::removeWriter() -{ - switch (WaitForSingleObject(_mutex, INFINITE)) - { - case WAIT_OBJECT_0: - if (--_writersWaiting == 0 && _writers == 0) SetEvent(_readEvent); - ReleaseMutex(_mutex); - break; - default: - throw SystemException("cannot lock reader/writer lock"); - } -} - - -void RWLockImpl::readLockImpl() -{ - HANDLE h[2]; - h[0] = _mutex; - h[1] = _readEvent; - switch (WaitForMultipleObjects(2, h, TRUE, INFINITE)) - { - case WAIT_OBJECT_0: - case WAIT_OBJECT_0 + 1: - ++_readers; - ResetEvent(_writeEvent); - ReleaseMutex(_mutex); - poco_assert_dbg(_writers == 0); - break; - default: - throw SystemException("cannot lock reader/writer lock"); - } -} - - -bool RWLockImpl::tryReadLockImpl() -{ - for (;;) - { - if (_writers != 0 || _writersWaiting != 0) - return false; - - DWORD result = tryReadLockOnce(); - switch (result) - { - case WAIT_OBJECT_0: - case WAIT_OBJECT_0 + 1: - return true; - case WAIT_TIMEOUT: - continue; // try again - default: - throw SystemException("cannot lock reader/writer lock"); - } - } -} - - -void RWLockImpl::writeLockImpl() -{ - addWriter(); - HANDLE h[2]; - h[0] = _mutex; - h[1] = _writeEvent; - switch (WaitForMultipleObjects(2, h, TRUE, INFINITE)) - { - case WAIT_OBJECT_0: - case WAIT_OBJECT_0 + 1: - --_writersWaiting; - ++_readers; - ++_writers; - ResetEvent(_readEvent); - ResetEvent(_writeEvent); - ReleaseMutex(_mutex); - poco_assert_dbg(_writers == 1); - break; - default: - removeWriter(); - throw SystemException("cannot lock reader/writer lock"); - } -} - - -bool RWLockImpl::tryWriteLockImpl() -{ - addWriter(); - HANDLE h[2]; - h[0] = _mutex; - h[1] = _writeEvent; - switch (WaitForMultipleObjects(2, h, TRUE, 1)) - { - case WAIT_OBJECT_0: - case WAIT_OBJECT_0 + 1: - --_writersWaiting; - ++_readers; - ++_writers; - ResetEvent(_readEvent); - ResetEvent(_writeEvent); - ReleaseMutex(_mutex); - poco_assert_dbg(_writers == 1); - return true; - case WAIT_TIMEOUT: - removeWriter(); - return false; - default: - removeWriter(); - throw SystemException("cannot lock reader/writer lock"); - } -} - - -void RWLockImpl::unlockImpl() -{ - switch (WaitForSingleObject(_mutex, INFINITE)) - { - case WAIT_OBJECT_0: - _writers = 0; - if (_writersWaiting == 0) SetEvent(_readEvent); - if (--_readers == 0) SetEvent(_writeEvent); - ReleaseMutex(_mutex); - break; - default: - throw SystemException("cannot unlock reader/writer lock"); - } -} - - -DWORD RWLockImpl::tryReadLockOnce() -{ - HANDLE h[2]; - h[0] = _mutex; - h[1] = _readEvent; - DWORD result = WaitForMultipleObjects(2, h, TRUE, 1); - switch (result) - { - case WAIT_OBJECT_0: - case WAIT_OBJECT_0 + 1: - ++_readers; - ResetEvent(_writeEvent); - ReleaseMutex(_mutex); - poco_assert_dbg(_writers == 0); - return result; - case WAIT_TIMEOUT: - return result; - default: - throw SystemException("cannot lock reader/writer lock"); - } -} - - -} // namespace Poco diff --git a/base/poco/Foundation/src/RWLock_WINCE.cpp b/base/poco/Foundation/src/RWLock_WINCE.cpp deleted file mode 100644 index f383a9da5d1..00000000000 --- a/base/poco/Foundation/src/RWLock_WINCE.cpp +++ /dev/null @@ -1,174 +0,0 @@ -// -// RWLock_WINCE.cpp -// -// Library: Foundation -// Package: Threading -// Module: RWLock -// -// Copyright (c) 2009-2010, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/RWLock_WINCE.h" -#include "Poco/Thread.h" - - -namespace Poco { - - -RWLockImpl::RWLockImpl(): - _readerCount(0), - _readerWaiting(0), - _writerCount(0), - _writerWaiting(0), - _writeLock(false) - -{ - InitializeCriticalSection(&_cs); - _readerGreen = CreateEventW(NULL, FALSE, TRUE, NULL); - if (!_readerGreen) throw SystemException("Cannot create RWLock"); - _writerGreen = CreateEventW(NULL, FALSE, TRUE, NULL); - if (!_writerGreen) - { - CloseHandle(_readerGreen); - throw SystemException("Cannot create RWLock"); - } -} - - -RWLockImpl::~RWLockImpl() -{ - CloseHandle(_readerGreen); - CloseHandle(_writerGreen); - DeleteCriticalSection(&_cs); -} - - -void RWLockImpl::readLockImpl() -{ - tryReadLockImpl(INFINITE); -} - - -bool RWLockImpl::tryReadLockImpl(DWORD timeout) -{ - bool wait = false; - do - { - EnterCriticalSection(&_cs); - if (!_writerCount && !_writerWaiting) - { - if (wait) - { - _readerWaiting--; - wait = false; - } - _readerCount++; - } - else - { - if (!wait) - { - _readerWaiting++; - wait = true; - } - ResetEvent(_readerGreen); - } - LeaveCriticalSection(&_cs); - if (wait) - { - if (WaitForSingleObject(_readerGreen, timeout) != WAIT_OBJECT_0) - { - EnterCriticalSection(&_cs); - _readerWaiting--; - SetEvent(_readerGreen); - SetEvent(_writerGreen); - LeaveCriticalSection(&_cs); - return false; - } - } - } - while (wait); - - return true; -} - - -void RWLockImpl::writeLockImpl() -{ - tryWriteLockImpl(INFINITE); -} - - -bool RWLockImpl::tryWriteLockImpl(DWORD timeout) -{ - bool wait = false; - - do - { - EnterCriticalSection(&_cs); - if (!_readerCount && !_writerCount) - { - if (wait) - { - _writerWaiting--; - wait = false; - } - _writerCount++; - } - else - { - if (!wait) - { - _writerWaiting++; - wait = true; - } - ResetEvent(_writerGreen); - } - LeaveCriticalSection(&_cs); - if (wait) - { - if (WaitForSingleObject(_writerGreen, timeout) != WAIT_OBJECT_0) - { - EnterCriticalSection(&_cs); - _writerWaiting--; - SetEvent(_readerGreen); - SetEvent(_writerGreen); - LeaveCriticalSection(&_cs); - return false; - } - } - } - while (wait); - - _writeLock = true; - return true; -} - - -void RWLockImpl::unlockImpl() -{ - EnterCriticalSection(&_cs); - - if (_writeLock) - { - _writeLock = false; - _writerCount--; - } - else - { - _readerCount--; - } - if (_writerWaiting) - SetEvent(_writerGreen); - else if (_readerWaiting) - SetEvent(_readerGreen); - - LeaveCriticalSection(&_cs); -} - - -} // namespace Poco diff --git a/base/poco/Foundation/src/SharedLibrary_HPUX.cpp b/base/poco/Foundation/src/SharedLibrary_HPUX.cpp deleted file mode 100644 index 668bb5f64dd..00000000000 --- a/base/poco/Foundation/src/SharedLibrary_HPUX.cpp +++ /dev/null @@ -1,100 +0,0 @@ -// -// SharedLibrary_HPUX.cpp -// -// Library: Foundation -// Package: SharedLibrary -// Module: SharedLibrary -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/SharedLibrary_HPUX.h" -#include - - -namespace Poco { - - -FastMutex SharedLibraryImpl::_mutex; - - -SharedLibraryImpl::SharedLibraryImpl() -{ - _handle = 0; -} - - -SharedLibraryImpl::~SharedLibraryImpl() -{ -} - - -void SharedLibraryImpl::loadImpl(const std::string& path, int /*flags*/) -{ - FastMutex::ScopedLock lock(_mutex); - - if (_handle) throw LibraryAlreadyLoadedException(path); - _handle = shl_load(path.c_str(), BIND_DEFERRED, 0); - if (!_handle) throw LibraryLoadException(path); - _path = path; -} - - -void SharedLibraryImpl::unloadImpl() -{ - FastMutex::ScopedLock lock(_mutex); - - if (_handle) - { - shl_unload(_handle); - _handle = 0; - _path.clear(); - } -} - - -bool SharedLibraryImpl::isLoadedImpl() const -{ - return _handle != 0; -} - - -void* SharedLibraryImpl::findSymbolImpl(const std::string& name) -{ - FastMutex::ScopedLock lock(_mutex); - - void* result = 0; - if (_handle && shl_findsym(&_handle, name.c_str(), TYPE_UNDEFINED, &result) != -1) - return result; - else - return 0; -} - - -const std::string& SharedLibraryImpl::getPathImpl() const -{ - return _path; -} - - -std::string SharedLibraryImpl::suffixImpl() -{ -#if defined(_DEBUG) && !defined(POCO_NO_SHARED_LIBRARY_DEBUG_SUFFIX) - return "d.sl"; -#else - return ".sl"; -#endif -} - - -bool SharedLibraryImpl::setSearchPathImpl(const std::string&) -{ - return false; -} - - -} // namespace Poco diff --git a/base/poco/Foundation/src/SharedLibrary_VX.cpp b/base/poco/Foundation/src/SharedLibrary_VX.cpp deleted file mode 100644 index bb2b14681ed..00000000000 --- a/base/poco/Foundation/src/SharedLibrary_VX.cpp +++ /dev/null @@ -1,141 +0,0 @@ -// -// SharedLibrary_VX.cpp -// -// Library: Foundation -// Package: SharedLibrary -// Module: SharedLibrary -// -// Copyright (c) 2004-2011, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/SharedLibrary_VX.h" -#include "Poco/Exception.h" -#include "Poco/Format.h" -#include -#include -#include -#include -#include -#include - - -struct SymLookup -{ - const char* name; - int group; - void* addr; -}; - - -extern "C" bool lookupFunc(char* name, int val, SYM_TYPE type, int arg, UINT16 group) -{ - SymLookup* symLookup = reinterpret_cast(arg); - if (group == symLookup->group && std::strcmp(name, symLookup->name) == 0) - { - symLookup->addr = reinterpret_cast(val); - return TRUE; - } - else return FALSE; -} - - -namespace Poco { - - -FastMutex SharedLibraryImpl::_mutex; - - -SharedLibraryImpl::SharedLibraryImpl(): - _moduleId(0) -{ -} - - -SharedLibraryImpl::~SharedLibraryImpl() -{ -} - - -void SharedLibraryImpl::loadImpl(const std::string& path, int /*flags*/) -{ - FastMutex::ScopedLock lock(_mutex); - - if (_moduleId) throw LibraryAlreadyLoadedException(path); - int fd = open(const_cast(path.c_str()), O_RDONLY, 0); - if (fd) - { - _moduleId = loadModule(fd, LOAD_GLOBAL_SYMBOLS); - if (!_moduleId) - { - int err = errno; - close(fd); - throw LibraryLoadException(Poco::format("error %d", err)); - } - } - else - { - int err = errno; - throw LibraryLoadException(Poco::format("cannot open library (error %d)", err)); - } - _path = path; -} - - -void SharedLibraryImpl::unloadImpl() -{ - FastMutex::ScopedLock lock(_mutex); - - if (_moduleId) - { - unldByModuleId(_moduleId, 0); - _moduleId = 0; - } -} - - -bool SharedLibraryImpl::isLoadedImpl() const -{ - return _moduleId != 0; -} - - -void* SharedLibraryImpl::findSymbolImpl(const std::string& name) -{ - poco_assert (_moduleId != 0); - - FastMutex::ScopedLock lock(_mutex); - - MODULE_INFO mi; - if (!moduleInfoGet(_moduleId, &mi)) return 0; - SymLookup symLookup; - symLookup.name = name.c_str(); - symLookup.group = mi.group; - symLookup.addr = 0; - symEach(sysSymTbl, reinterpret_cast(lookupFunc), reinterpret_cast(&symLookup)); - return symLookup.addr; -} - - -const std::string& SharedLibraryImpl::getPathImpl() const -{ - return _path; -} - - -std::string SharedLibraryImpl::suffixImpl() -{ - return ".out"; -} - - -bool SharedLibraryImpl::setSearchPathImpl(const std::string&) -{ - return false; -} - - -} // namespace Poco diff --git a/base/poco/Foundation/src/SharedLibrary_WIN32.cpp b/base/poco/Foundation/src/SharedLibrary_WIN32.cpp deleted file mode 100644 index 340a742e8f5..00000000000 --- a/base/poco/Foundation/src/SharedLibrary_WIN32.cpp +++ /dev/null @@ -1,108 +0,0 @@ -// -// SharedLibrary_WIN32.cpp -// -// Library: Foundation -// Package: SharedLibrary -// Module: SharedLibrary -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/SharedLibrary_WIN32.h" -#include "Poco/Path.h" -#include "Poco/UnWindows.h" - - -namespace Poco { - - -FastMutex SharedLibraryImpl::_mutex; - - -SharedLibraryImpl::SharedLibraryImpl() -{ - _handle = 0; -} - - -SharedLibraryImpl::~SharedLibraryImpl() -{ -} - - -void SharedLibraryImpl::loadImpl(const std::string& path, int /*flags*/) -{ - FastMutex::ScopedLock lock(_mutex); - - if (_handle) throw LibraryAlreadyLoadedException(_path); - DWORD flags(0); - Path p(path); - if (p.isAbsolute()) flags |= LOAD_WITH_ALTERED_SEARCH_PATH; - _handle = LoadLibraryExA(path.c_str(), 0, flags); - if (!_handle) throw LibraryLoadException(path); - _path = path; -} - - -void SharedLibraryImpl::unloadImpl() -{ - FastMutex::ScopedLock lock(_mutex); - - if (_handle) - { - FreeLibrary((HMODULE) _handle); - _handle = 0; - } - _path.clear(); -} - - -bool SharedLibraryImpl::isLoadedImpl() const -{ - return _handle != 0; -} - - -void* SharedLibraryImpl::findSymbolImpl(const std::string& name) -{ - FastMutex::ScopedLock lock(_mutex); - - if (_handle) - { - return (void*) GetProcAddress((HMODULE) _handle, name.c_str()); - } - else return 0; -} - - -const std::string& SharedLibraryImpl::getPathImpl() const -{ - return _path; -} - - -std::string SharedLibraryImpl::suffixImpl() -{ -#if defined(_DEBUG) && !defined(POCO_NO_SHARED_LIBRARY_DEBUG_SUFFIX) - return "d.dll"; -#else - return ".dll"; -#endif -} - - -bool SharedLibraryImpl::setSearchPathImpl(const std::string& path) -{ -#if _WIN32_WINNT >= 0x0502 - return SetDllDirectoryA(path.c_str()) != 0; -#else - return false; -#endif -} - - -} // namespace Poco diff --git a/base/poco/Foundation/src/SharedLibrary_WIN32U.cpp b/base/poco/Foundation/src/SharedLibrary_WIN32U.cpp deleted file mode 100644 index 0a52be7966c..00000000000 --- a/base/poco/Foundation/src/SharedLibrary_WIN32U.cpp +++ /dev/null @@ -1,113 +0,0 @@ -// -// SharedLibrary_WIN32U.cpp -// -// Library: Foundation -// Package: SharedLibrary -// Module: SharedLibrary -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/SharedLibrary_WIN32U.h" -#include "Poco/UnicodeConverter.h" -#include "Poco/Path.h" -#include "Poco/UnWindows.h" - - -namespace Poco { - - -FastMutex SharedLibraryImpl::_mutex; - - -SharedLibraryImpl::SharedLibraryImpl() -{ - _handle = 0; -} - - -SharedLibraryImpl::~SharedLibraryImpl() -{ -} - - -void SharedLibraryImpl::loadImpl(const std::string& path, int /*flags*/) -{ - FastMutex::ScopedLock lock(_mutex); - - if (_handle) throw LibraryAlreadyLoadedException(_path); - DWORD flags(0); - Path p(path); - if (p.isAbsolute()) flags |= LOAD_WITH_ALTERED_SEARCH_PATH; - std::wstring upath; - UnicodeConverter::toUTF16(path, upath); - _handle = LoadLibraryExW(upath.c_str(), 0, flags); - if (!_handle) throw LibraryLoadException(path); - _path = path; -} - - -void SharedLibraryImpl::unloadImpl() -{ - FastMutex::ScopedLock lock(_mutex); - - if (_handle) - { - FreeLibrary((HMODULE) _handle); - _handle = 0; - } - _path.clear(); -} - - -bool SharedLibraryImpl::isLoadedImpl() const -{ - return _handle != 0; -} - - -void* SharedLibraryImpl::findSymbolImpl(const std::string& name) -{ - FastMutex::ScopedLock lock(_mutex); - - if (_handle) - { - return (void*) GetProcAddress((HMODULE) _handle, name.c_str()); - } - else return 0; -} - - -const std::string& SharedLibraryImpl::getPathImpl() const -{ - return _path; -} - - -std::string SharedLibraryImpl::suffixImpl() -{ -#if defined(_DEBUG) && !defined(POCO_NO_SHARED_LIBRARY_DEBUG_SUFFIX) - return "d.dll"; -#else - return ".dll"; -#endif -} - - -bool SharedLibraryImpl::setSearchPathImpl(const std::string& path) -{ -#if _WIN32_WINNT >= 0x0502 - std::wstring wpath; - Poco::UnicodeConverter::toUTF16(path, wpath); - return SetDllDirectoryW(wpath.c_str()) != 0; -#else - return false; -#endif -} - - -} // namespace Poco diff --git a/base/poco/Foundation/src/SharedMemory_WIN32.cpp b/base/poco/Foundation/src/SharedMemory_WIN32.cpp deleted file mode 100644 index a05a32eca4d..00000000000 --- a/base/poco/Foundation/src/SharedMemory_WIN32.cpp +++ /dev/null @@ -1,145 +0,0 @@ -// -// SharedMemoryImpl.cpp -// -// Library: Foundation -// Package: Processes -// Module: SharedMemoryImpl -// -// Copyright (c) 2007, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/SharedMemory_WIN32.h" -#include "Poco/Error.h" -#include "Poco/Exception.h" -#include "Poco/File.h" -#include "Poco/Format.h" -#include "Poco/UnWindows.h" - - -namespace Poco { - - -SharedMemoryImpl::SharedMemoryImpl(const std::string& name, std::size_t size, SharedMemory::AccessMode mode, const void*, bool): - _name(name), - _memHandle(INVALID_HANDLE_VALUE), - _fileHandle(INVALID_HANDLE_VALUE), - _size(static_cast(size)), - _mode(PAGE_READONLY), - _address(0) -{ - if (mode == SharedMemory::AM_WRITE) - _mode = PAGE_READWRITE; - - _memHandle = CreateFileMappingA(INVALID_HANDLE_VALUE, NULL, _mode, 0, _size, _name.c_str()); - - if (!_memHandle) - { - DWORD dwRetVal = GetLastError(); - if (_mode != PAGE_READONLY || dwRetVal != 5) - throw SystemException(format("Cannot create shared memory object %s [Error %d: %s]", _name, static_cast(dwRetVal), Error::getMessage(dwRetVal))); - - _memHandle = OpenFileMappingA(PAGE_READONLY, FALSE, _name.c_str()); - if (!_memHandle) - { - dwRetVal = GetLastError(); - throw SystemException(format("Cannot open shared memory object %s [Error %d: %s]", _name, static_cast(dwRetVal), Error::getMessage(dwRetVal))); - } - } - map(); -} - - -SharedMemoryImpl::SharedMemoryImpl(const Poco::File& file, SharedMemory::AccessMode mode, const void*): - _name(file.path()), - _memHandle(INVALID_HANDLE_VALUE), - _fileHandle(INVALID_HANDLE_VALUE), - _size(0), - _mode(PAGE_READONLY), - _address(0) -{ - if (!file.exists() || !file.isFile()) - throw FileNotFoundException(_name); - - _size = static_cast(file.getSize()); - - DWORD shareMode = FILE_SHARE_READ | FILE_SHARE_WRITE; - DWORD fileMode = GENERIC_READ; - - if (mode == SharedMemory::AM_WRITE) - { - _mode = PAGE_READWRITE; - fileMode |= GENERIC_WRITE; - } - - _fileHandle = CreateFileA(_name.c_str(), fileMode, shareMode, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); - - if (_fileHandle == INVALID_HANDLE_VALUE) - throw OpenFileException("Cannot open memory mapped file", _name); - - _memHandle = CreateFileMapping(_fileHandle, NULL, _mode, 0, 0, NULL); - if (!_memHandle) - { - DWORD dwRetVal = GetLastError(); - CloseHandle(_fileHandle); - _fileHandle = INVALID_HANDLE_VALUE; - throw SystemException(format("Cannot map file into shared memory %s [Error %d: %s]", _name, (int)dwRetVal, Error::getMessage(dwRetVal))); - } - map(); -} - - -SharedMemoryImpl::~SharedMemoryImpl() -{ - unmap(); - close(); -} - - -void SharedMemoryImpl::map() -{ - DWORD access = FILE_MAP_READ; - if (_mode == PAGE_READWRITE) - access = FILE_MAP_WRITE; - LPVOID addr = MapViewOfFile(_memHandle, access, 0, 0, _size); - if (!addr) - { - DWORD dwRetVal = GetLastError(); - throw SystemException(format("Cannot map shared memory object %s [Error %d: %s]", _name, (int)dwRetVal, Error::getMessage(dwRetVal))); - } - - _address = static_cast(addr); -} - - -void SharedMemoryImpl::unmap() -{ - if (_address) - { - UnmapViewOfFile(_address); - _address = 0; - return; - } -} - - -void SharedMemoryImpl::close() -{ - if (_memHandle != INVALID_HANDLE_VALUE) - { - CloseHandle(_memHandle); - _memHandle = INVALID_HANDLE_VALUE; - } - - if (_fileHandle != INVALID_HANDLE_VALUE) - { - CloseHandle(_fileHandle); - _fileHandle = INVALID_HANDLE_VALUE; - } -} - - -} // namespace Poco diff --git a/base/poco/Foundation/src/Thread_VX.cpp b/base/poco/Foundation/src/Thread_VX.cpp deleted file mode 100644 index d1883bf9156..00000000000 --- a/base/poco/Foundation/src/Thread_VX.cpp +++ /dev/null @@ -1,273 +0,0 @@ -// -// Thread_VX.cpp -// -// Library: Foundation -// Package: Threading -// Module: Thread -// -// Copyright (c) 2004-2011, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Thread_VX.h" -#include "Poco/ErrorHandler.h" -#include "Poco/Exception.h" -#include "Poco/Timestamp.h" -#include "Poco/Timespan.h" -#include - - -namespace Poco { - - -ThreadImpl* ThreadImpl::_pCurrent(0); - - -ThreadImpl::ThreadImpl(): - _pData(new ThreadData) -{ -} - - -ThreadImpl::~ThreadImpl() -{ -} - - -void ThreadImpl::setPriorityImpl(int prio) -{ - if (prio != _pData->prio) - { - _pData->prio = prio; - _pData->osPrio = mapPrio(_pData->prio); - if (isRunningImpl()) - { - if (taskPrioritySet(_pData->task, _pData->osPrio) != OK) - throw SystemException("cannot set task priority"); - } - } -} - - -void ThreadImpl::setOSPriorityImpl(int prio, int /* policy */) -{ - if (prio != _pData->osPrio) - { - _pData->prio = reverseMapPrio(prio); - _pData->osPrio = prio; - if (_pData->pRunnableTarget || _pData->pCallbackTarget) - { - if (taskPrioritySet(_pData->task, prio) != OK) - throw SystemException("cannot set task priority"); - } - } -} - - -int ThreadImpl::getMinOSPriorityImpl(int /* policy */) -{ - return 255; -} - - -int ThreadImpl::getMaxOSPriorityImpl(int /* policy */) -{ - return 0; -} - - -void ThreadImpl::setStackSizeImpl(int size) -{ - _pData->stackSize = size; -} - - -void ThreadImpl::startImpl(Runnable& target) -{ - if (_pData->pRunnableTarget) - throw SystemException("thread already running"); - - _pData->pRunnableTarget = ⌖ - - int stackSize = _pData->stackSize == 0 ? DEFAULT_THREAD_STACK_SIZE : _pData->stackSize; - int id = taskSpawn(NULL, _pData->osPrio, VX_FP_TASK, stackSize, reinterpret_cast(runnableEntry), reinterpret_cast(this), 0, 0, 0, 0, 0, 0, 0, 0, 0); - if (id == ERROR) - throw SystemException("cannot spawn task"); - - _pData->task = id; -} - - -void ThreadImpl::startImpl(Callable target, void* pData) -{ - if (_pData->pCallbackTarget && _pData->pCallbackTarget->callback) - throw SystemException("thread already running"); - - if (0 == _pData->pCallbackTarget.get()) - _pData->pCallbackTarget = new CallbackData; - - _pData->pCallbackTarget->callback = target; - _pData->pCallbackTarget->pData = pData; - - int stackSize = _pData->stackSize == 0 ? DEFAULT_THREAD_STACK_SIZE : _pData->stackSize; - int id = taskSpawn(NULL, _pData->osPrio, VX_FP_TASK, stackSize, reinterpret_cast(callableEntry), reinterpret_cast(this), 0, 0, 0, 0, 0, 0, 0, 0, 0); - if (id == ERROR) - throw SystemException("cannot spawn task"); - - _pData->task = id; -} - - -void ThreadImpl::joinImpl() -{ - _pData->done.wait(); -} - - -bool ThreadImpl::joinImpl(long milliseconds) -{ - return _pData->done.tryWait(milliseconds); -} - - -ThreadImpl* ThreadImpl::currentImpl() -{ - return _pCurrent; -} - - -ThreadImpl::TIDImpl ThreadImpl::currentTidImpl() -{ - return taskIdSelf(); -} - - -void ThreadImpl::sleepImpl(long milliseconds) -{ - Poco::Timespan remainingTime(1000*Poco::Timespan::TimeDiff(milliseconds)); - int rc; - do - { - struct timespec ts; - ts.tv_sec = (long) remainingTime.totalSeconds(); - ts.tv_nsec = (long) remainingTime.useconds()*1000; - Poco::Timestamp start; - rc = ::nanosleep(&ts, 0); - if (rc < 0 && errno == EINTR) - { - Poco::Timestamp end; - Poco::Timespan waited = start.elapsed(); - if (waited < remainingTime) - remainingTime -= waited; - else - remainingTime = 0; - } - } - while (remainingTime > 0 && rc < 0 && errno == EINTR); - if (rc < 0 && remainingTime > 0) throw Poco::SystemException("Thread::sleep(): nanosleep() failed"); -} - - -void ThreadImpl::runnableEntry(void* pThread, int, int, int, int, int, int, int, int, int) -{ - taskVarAdd(0, reinterpret_cast(&_pCurrent)); - _pCurrent = reinterpret_cast(pThread); - - AutoPtr pData = _pCurrent->_pData; - try - { - pData->pRunnableTarget->run(); - } - catch (Exception& exc) - { - ErrorHandler::handle(exc); - } - catch (std::exception& exc) - { - ErrorHandler::handle(exc); - } - catch (...) - { - ErrorHandler::handle(); - } - - pData->pRunnableTarget = 0; - pData->done.set(); -} - - -void ThreadImpl::callableEntry(void* pThread, int, int, int, int, int, int, int, int, int) -{ - taskVarAdd(0, reinterpret_cast(&_pCurrent)); - _pCurrent = reinterpret_cast(pThread); - - AutoPtr pData = _pCurrent->_pData; - try - { - pData->pCallbackTarget->callback(pData->pCallbackTarget->pData); - } - catch (Exception& exc) - { - ErrorHandler::handle(exc); - } - catch (std::exception& exc) - { - ErrorHandler::handle(exc); - } - catch (...) - { - ErrorHandler::handle(); - } - - pData->pCallbackTarget->callback = 0; - pData->pCallbackTarget->pData = 0; - pData->done.set(); -} - - -int ThreadImpl::mapPrio(int prio) -{ - int pmin = getMinOSPriorityImpl(); - int pmax = getMaxOSPriorityImpl(); - - switch (prio) - { - case PRIO_LOWEST_IMPL: - return pmin; - case PRIO_LOW_IMPL: - return pmin + (pmax - pmin)/4; - case PRIO_NORMAL_IMPL: - return pmin + (pmax - pmin)/2; - case PRIO_HIGH_IMPL: - return pmin + 3*(pmax - pmin)/4; - case PRIO_HIGHEST_IMPL: - return pmax; - default: - poco_bugcheck_msg("invalid thread priority"); - } - return -1; // just to satisfy compiler - we'll never get here anyway -} - - -int ThreadImpl::reverseMapPrio(int prio) -{ - int pmin = getMinOSPriorityImpl(); - int pmax = getMaxOSPriorityImpl(); - int normal = pmin + (pmax - pmin)/2; - if (prio == pmax) - return PRIO_HIGHEST_IMPL; - if (prio > normal) - return PRIO_HIGH_IMPL; - else if (prio == normal) - return PRIO_NORMAL_IMPL; - else if (prio > pmin) - return PRIO_LOW_IMPL; - else - return PRIO_LOWEST_IMPL; -} - - -} // namespace Poco diff --git a/base/poco/Foundation/src/Thread_WIN32.cpp b/base/poco/Foundation/src/Thread_WIN32.cpp deleted file mode 100644 index 9bd6d25b976..00000000000 --- a/base/poco/Foundation/src/Thread_WIN32.cpp +++ /dev/null @@ -1,223 +0,0 @@ -// -// Thread_WIN32.h -// -// Library: Foundation -// Package: Threading -// Module: Thread -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Thread_WIN32.h" -#include "Poco/Exception.h" -#include "Poco/ErrorHandler.h" -#include - - -#if defined(POCO_WIN32_DEBUGGER_THREAD_NAMES) - - -namespace -{ - /// See - /// and for - /// more information on the code below. - - const DWORD MS_VC_EXCEPTION = 0x406D1388; - - #pragma pack(push,8) - typedef struct tagTHREADNAME_INFO - { - DWORD dwType; // Must be 0x1000. - LPCSTR szName; // Pointer to name (in user addr space). - DWORD dwThreadID; // Thread ID (-1=caller thread). - DWORD dwFlags; // Reserved for future use, must be zero. - } THREADNAME_INFO; - #pragma pack(pop) - - void setThreadName(DWORD dwThreadID, const char* threadName) - { - THREADNAME_INFO info; - info.dwType = 0x1000; - info.szName = threadName; - info.dwThreadID = dwThreadID; - info.dwFlags = 0; - - __try - { - RaiseException(MS_VC_EXCEPTION, 0, sizeof(info)/sizeof(ULONG_PTR), (ULONG_PTR*)&info); - } - __except (EXCEPTION_CONTINUE_EXECUTION) - { - } - } -} - - -#endif - - -namespace Poco { - - -ThreadImpl::CurrentThreadHolder ThreadImpl::_currentThreadHolder; - - -ThreadImpl::ThreadImpl(): - _thread(0), - _threadId(0), - _prio(PRIO_NORMAL_IMPL), - _stackSize(POCO_THREAD_STACK_SIZE) -{ -} - - -ThreadImpl::~ThreadImpl() -{ - if (_thread) CloseHandle(_thread); -} - - -void ThreadImpl::setPriorityImpl(int prio) -{ - if (prio != _prio) - { - _prio = prio; - if (_thread) - { - if (SetThreadPriority(_thread, _prio) == 0) - throw SystemException("cannot set thread priority"); - } - } -} - - -void ThreadImpl::setOSPriorityImpl(int prio, int /* policy */) -{ - setPriorityImpl(prio); -} - - -void ThreadImpl::startImpl(SharedPtr pTarget) -{ - if (isRunningImpl()) - throw SystemException("thread already running"); - - _pRunnableTarget = pTarget; - createImpl(runnableEntry, this); -} - - -void ThreadImpl::createImpl(Entry ent, void* pData) -{ -#if defined(_DLL) - _thread = CreateThread(NULL, _stackSize, ent, pData, 0, &_threadId); -#else - unsigned threadId; - _thread = (HANDLE) _beginthreadex(NULL, _stackSize, ent, this, 0, &threadId); - _threadId = static_cast(threadId); -#endif - if (!_thread) - throw SystemException("cannot create thread"); - if (_prio != PRIO_NORMAL_IMPL && !SetThreadPriority(_thread, _prio)) - throw SystemException("cannot set thread priority"); -} - - -void ThreadImpl::joinImpl() -{ - if (!_thread) return; - - switch (WaitForSingleObject(_thread, INFINITE)) - { - case WAIT_OBJECT_0: - threadCleanup(); - return; - default: - throw SystemException("cannot join thread"); - } -} - - -bool ThreadImpl::joinImpl(long milliseconds) -{ - if (!_thread) return true; - - switch (WaitForSingleObject(_thread, milliseconds + 1)) - { - case WAIT_TIMEOUT: - return false; - case WAIT_OBJECT_0: - threadCleanup(); - return true; - default: - throw SystemException("cannot join thread"); - } -} - - -bool ThreadImpl::isRunningImpl() const -{ - if (_thread) - { - DWORD ec = 0; - return GetExitCodeThread(_thread, &ec) && ec == STILL_ACTIVE; - } - return false; -} - - -void ThreadImpl::threadCleanup() -{ - if (!_thread) return; - if (CloseHandle(_thread)) _thread = 0; -} - - -ThreadImpl* ThreadImpl::currentImpl() -{ - return _currentThreadHolder.get(); -} - - -ThreadImpl::TIDImpl ThreadImpl::currentTidImpl() -{ - return GetCurrentThreadId(); -} - - -#if defined(_DLL) -DWORD WINAPI ThreadImpl::runnableEntry(LPVOID pThread) -#else -unsigned __stdcall ThreadImpl::runnableEntry(void* pThread) -#endif -{ - _currentThreadHolder.set(reinterpret_cast(pThread)); -#if defined(POCO_WIN32_DEBUGGER_THREAD_NAMES) - setThreadName(-1, reinterpret_cast(pThread)->getName().c_str()); -#endif - try - { - reinterpret_cast(pThread)->_pRunnableTarget->run(); - } - catch (Exception& exc) - { - ErrorHandler::handle(exc); - } - catch (std::exception& exc) - { - ErrorHandler::handle(exc); - } - catch (...) - { - ErrorHandler::handle(); - } - return 0; -} - - -} // namespace Poco diff --git a/base/poco/Foundation/src/Thread_WINCE.cpp b/base/poco/Foundation/src/Thread_WINCE.cpp deleted file mode 100644 index d769f88354c..00000000000 --- a/base/poco/Foundation/src/Thread_WINCE.cpp +++ /dev/null @@ -1,169 +0,0 @@ -// -// Thread_WINCE.h -// -// Library: Foundation -// Package: Threading -// Module: Thread -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Thread_WINCE.h" -#include "Poco/Exception.h" -#include "Poco/ErrorHandler.h" - - -namespace Poco { - - -ThreadImpl::CurrentThreadHolder ThreadImpl::_currentThreadHolder; - - -ThreadImpl::ThreadImpl(): - _pRunnableTarget(0), - _thread(0), - _threadId(0), - _prio(PRIO_NORMAL_IMPL), - _stackSize(POCO_THREAD_STACK_SIZE) -{ -} - - -ThreadImpl::~ThreadImpl() -{ - if (_thread) CloseHandle(_thread); -} - - -void ThreadImpl::setPriorityImpl(int prio) -{ - if (prio != _prio) - { - _prio = prio; - if (_thread) - { - if (SetThreadPriority(_thread, _prio) == 0) - throw SystemException("cannot set thread priority"); - } - } -} - - -void ThreadImpl::setOSPriorityImpl(int prio, int /* policy */) -{ - setPriorityImpl(prio); -} - - -void ThreadImpl::startImpl(SharedPtr pTarget) -{ - if (isRunningImpl()) - throw SystemException("thread already running"); - - _pRunnableTarget = pTarget; - - createImpl(runnableEntry, this); -} - - -void ThreadImpl::createImpl(Entry ent, void* pData) -{ - _thread = CreateThread(NULL, _stackSize, ent, pData, 0, &_threadId); - - if (!_thread) - throw SystemException("cannot create thread"); - if (_prio != PRIO_NORMAL_IMPL && !SetThreadPriority(_thread, _prio)) - throw SystemException("cannot set thread priority"); -} - - -void ThreadImpl::joinImpl() -{ - if (!_thread) return; - - switch (WaitForSingleObject(_thread, INFINITE)) - { - case WAIT_OBJECT_0: - threadCleanup(); - return; - default: - throw SystemException("cannot join thread"); - } -} - - -bool ThreadImpl::joinImpl(long milliseconds) -{ - if (!_thread) return true; - - switch (WaitForSingleObject(_thread, milliseconds + 1)) - { - case WAIT_TIMEOUT: - return false; - case WAIT_OBJECT_0: - threadCleanup(); - return true; - default: - throw SystemException("cannot join thread"); - } -} - - -bool ThreadImpl::isRunningImpl() const -{ - if (_thread) - { - DWORD ec = 0; - return GetExitCodeThread(_thread, &ec) && ec == STILL_ACTIVE; - } - return false; -} - - -void ThreadImpl::threadCleanup() -{ - if (!_thread) return; - if (CloseHandle(_thread)) _thread = 0; -} - - -ThreadImpl* ThreadImpl::currentImpl() -{ - return _currentThreadHolder.get(); -} - - -ThreadImpl::TIDImpl ThreadImpl::currentTidImpl() -{ - return GetCurrentThreadId(); -} - - -DWORD WINAPI ThreadImpl::runnableEntry(LPVOID pThread) -{ - _currentThreadHolder.set(reinterpret_cast(pThread)); - try - { - reinterpret_cast(pThread)->_pRunnableTarget->run(); - } - catch (Exception& exc) - { - ErrorHandler::handle(exc); - } - catch (std::exception& exc) - { - ErrorHandler::handle(exc); - } - catch (...) - { - ErrorHandler::handle(); - } - return 0; -} - - -} // namespace Poco diff --git a/base/poco/Foundation/src/Timezone_WIN32.cpp b/base/poco/Foundation/src/Timezone_WIN32.cpp deleted file mode 100644 index a64f6cfc53c..00000000000 --- a/base/poco/Foundation/src/Timezone_WIN32.cpp +++ /dev/null @@ -1,89 +0,0 @@ -// -// Timezone_WIN32.cpp -// -// Library: Foundation -// Package: DateTime -// Module: Timezone -// -// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Timezone.h" -#include "Poco/UnicodeConverter.h" -#include "Poco/Exception.h" -#include "Poco/UnWindows.h" -#include - - -namespace Poco { - - -int Timezone::utcOffset() -{ - TIME_ZONE_INFORMATION tzInfo; - DWORD dstFlag = GetTimeZoneInformation(&tzInfo); - return -tzInfo.Bias*60; -} - - -int Timezone::dst() -{ - TIME_ZONE_INFORMATION tzInfo; - DWORD dstFlag = GetTimeZoneInformation(&tzInfo); - return dstFlag == TIME_ZONE_ID_DAYLIGHT ? -tzInfo.DaylightBias*60 : 0; -} - - -bool Timezone::isDst(const Timestamp& timestamp) -{ - std::time_t time = timestamp.epochTime(); - struct std::tm* tms = std::localtime(&time); - if (!tms) throw Poco::SystemException("cannot get local time DST flag"); - return tms->tm_isdst > 0; -} - - -std::string Timezone::name() -{ - std::string result; - TIME_ZONE_INFORMATION tzInfo; - DWORD dstFlag = GetTimeZoneInformation(&tzInfo); - WCHAR* ptr = dstFlag == TIME_ZONE_ID_DAYLIGHT ? tzInfo.DaylightName : tzInfo.StandardName; - char buffer[256]; - DWORD rc = WideCharToMultiByte(CP_ACP, 0, ptr, -1, buffer, sizeof(buffer), NULL, NULL); - if (rc) result = buffer; - return result; -} - - -std::string Timezone::standardName() -{ - std::string result; - TIME_ZONE_INFORMATION tzInfo; - DWORD dstFlag = GetTimeZoneInformation(&tzInfo); - WCHAR* ptr = tzInfo.StandardName; - char buffer[256]; - DWORD rc = WideCharToMultiByte(CP_ACP, 0, ptr, -1, buffer, sizeof(buffer), NULL, NULL); - if (rc) result = buffer; - return result; -} - - -std::string Timezone::dstName() -{ - std::string result; - TIME_ZONE_INFORMATION tzInfo; - DWORD dstFlag = GetTimeZoneInformation(&tzInfo); - WCHAR* ptr = tzInfo.DaylightName; - char buffer[256]; - DWORD rc = WideCharToMultiByte(CP_ACP, 0, ptr, -1, buffer, sizeof(buffer), NULL, NULL); - if (rc) result = buffer; - return result; -} - - -} // namespace Poco diff --git a/base/poco/Foundation/src/Timezone_WINCE.cpp b/base/poco/Foundation/src/Timezone_WINCE.cpp deleted file mode 100644 index c0cd7c28684..00000000000 --- a/base/poco/Foundation/src/Timezone_WINCE.cpp +++ /dev/null @@ -1,84 +0,0 @@ -// -// Timezone_WINCE.cpp -// -// Library: Foundation -// Package: DateTime -// Module: Timezone -// -// Copyright (c) 2004-2010, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Timezone.h" -#include "Poco/UnicodeConverter.h" -#include "Poco/Exception.h" -#include "Poco/UnWindows.h" -#include -#include "wce_time.h" - - -namespace Poco { - - -int Timezone::utcOffset() -{ - TIME_ZONE_INFORMATION tzInfo; - DWORD dstFlag = GetTimeZoneInformation(&tzInfo); - return -tzInfo.Bias*60; -} - - -int Timezone::dst() -{ - TIME_ZONE_INFORMATION tzInfo; - DWORD dstFlag = GetTimeZoneInformation(&tzInfo); - return dstFlag == TIME_ZONE_ID_DAYLIGHT ? -tzInfo.DaylightBias*60 : 0; -} - - -bool Timezone::isDst(const Timestamp& timestamp) -{ - std::time_t time = timestamp.epochTime(); - struct std::tm* tms = wceex_localtime(&time); - if (!tms) throw SystemException("cannot get local time DST flag"); - return tms->tm_isdst > 0; -} - - -std::string Timezone::name() -{ - std::string result; - TIME_ZONE_INFORMATION tzInfo; - DWORD dstFlag = GetTimeZoneInformation(&tzInfo); - WCHAR* ptr = dstFlag == TIME_ZONE_ID_DAYLIGHT ? tzInfo.DaylightName : tzInfo.StandardName; - UnicodeConverter::toUTF8(ptr, result); - return result; -} - - -std::string Timezone::standardName() -{ - std::string result; - TIME_ZONE_INFORMATION tzInfo; - DWORD dstFlag = GetTimeZoneInformation(&tzInfo); - WCHAR* ptr = tzInfo.StandardName; - UnicodeConverter::toUTF8(ptr, result); - return result; -} - - -std::string Timezone::dstName() -{ - std::string result; - TIME_ZONE_INFORMATION tzInfo; - DWORD dstFlag = GetTimeZoneInformation(&tzInfo); - WCHAR* ptr = tzInfo.DaylightName; - UnicodeConverter::toUTF8(ptr, result); - return result; -} - - -} // namespace Poco From db6e9743144cfdb07c1d9e15f736493f60c99e94 Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Wed, 15 Feb 2023 11:50:04 +0100 Subject: [PATCH 236/717] Update test --- tests/queries/0_stateless/02560_null_as_default.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02560_null_as_default.sql b/tests/queries/0_stateless/02560_null_as_default.sql index 0a7da55a210..1aaa2c41a3f 100644 --- a/tests/queries/0_stateless/02560_null_as_default.sql +++ b/tests/queries/0_stateless/02560_null_as_default.sql @@ -2,11 +2,11 @@ drop table if exists test; create table test (x UInt64) engine=Memory(); set insert_null_as_default=1; insert into test select number % 2 ? NULL : 42 as x from numbers(2); -select * from test; +select * from test order by x; drop table test; create table test (x LowCardinality(String) default 'Hello') engine=Memory(); insert into test select (number % 2 ? NULL : 'World')::LowCardinality(Nullable(String)) from numbers(2); -select * from test; +select * from test order by x; drop table test; From e31fbf32fccb790eedc00963ee715f3d9dd0c25c Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Wed, 15 Feb 2023 11:50:21 +0100 Subject: [PATCH 237/717] Update test reference --- tests/queries/0_stateless/02560_null_as_default.reference | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02560_null_as_default.reference b/tests/queries/0_stateless/02560_null_as_default.reference index 517cbfb1e27..a500da3cbc5 100644 --- a/tests/queries/0_stateless/02560_null_as_default.reference +++ b/tests/queries/0_stateless/02560_null_as_default.reference @@ -1,4 +1,4 @@ -42 0 +42 World Hello From 289c5c60d350f0b55eadabaf5031d92ebf67ad81 Mon Sep 17 00:00:00 2001 From: flynn Date: Wed, 15 Feb 2023 11:24:06 +0000 Subject: [PATCH 238/717] fix --- src/Storages/IStorageDataLake.h | 73 +-------------------------------- src/Storages/StorageS3.cpp | 4 +- src/Storages/StorageS3.h | 2 +- 3 files changed, 5 insertions(+), 74 deletions(-) diff --git a/src/Storages/IStorageDataLake.h b/src/Storages/IStorageDataLake.h index f5f00a0b8ed..90e8e5cf1b0 100644 --- a/src/Storages/IStorageDataLake.h +++ b/src/Storages/IStorageDataLake.h @@ -23,23 +23,6 @@ namespace DB { -namespace ErrorCodes -{ - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; -} - -static const std::unordered_set required_configuration_keys = { - "url", -}; -static const std::unordered_set optional_configuration_keys = { - "format", - "compression", - "compression_method", - "structure", - "access_key_id", - "secret_access_key", -}; - template class IStorageDataLake : public IStorage { @@ -58,7 +41,6 @@ public: : IStorage(table_id_) , base_configuration{configuration_} , log(&Poco::Logger::get("Storage" + String(name) + "(" + table_id_.table_name + ")")) - , table_path(base_configuration.url.key) { StorageInMemoryMetadata storage_metadata; StorageS3::updateS3Configuration(context_, base_configuration); @@ -113,6 +95,7 @@ public: { StorageS3::updateS3Configuration(ctx, configuration); auto new_configuration = getAdjustedS3Configuration(ctx, configuration, &Poco::Logger::get("Storage" + String(name))); + return StorageS3::getTableStructureFromData( new_configuration, /*distributed processing*/ false, format_settings, ctx, /*object_infos*/ nullptr); } @@ -134,60 +117,10 @@ public: return new_configuration; } - static void processNamedCollectionResult(StorageS3::Configuration & configuration, const NamedCollection & collection) - { - validateNamedCollection(collection, required_configuration_keys, optional_configuration_keys); - - configuration.url = S3::URI{collection.get("url")}; - - configuration.auth_settings.access_key_id = collection.getOrDefault("access_key_id", ""); - configuration.auth_settings.secret_access_key = collection.getOrDefault("secret_access_key", ""); - - configuration.format = collection.getOrDefault("format", "Parquet"); - - configuration.compression_method - = collection.getOrDefault("compression_method", collection.getOrDefault("compression", "auto")); - - configuration.structure = collection.getOrDefault("structure", "auto"); - - configuration.request_settings = S3Settings::RequestSettings(collection); - } static StorageS3::Configuration getConfiguration(ASTs & engine_args, ContextPtr local_context) { - StorageS3::Configuration configuration; - - if (auto named_collection = tryGetNamedCollectionWithOverrides(engine_args)) - { - processNamedCollectionResult(configuration, *named_collection); - } - else - { - /// Supported signatures: - /// - /// xx('url', 'aws_access_key_id', 'aws_secret_access_key') - /// xx('url', 'aws_access_key_id', 'aws_secret_access_key', 'format') - - if (engine_args.empty() || engine_args.size() < 3) - throw Exception( - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, - "Storage {} requires 3 or 4 arguments: url, access_key_id, secret_access_key, [format]", - name); - - auto * header_it = StorageURL::collectHeaders(engine_args, configuration.headers_from_ast, local_context); - if (header_it != engine_args.end()) - engine_args.erase(header_it); - - for (auto & engine_arg : engine_args) - engine_arg = evaluateConstantExpressionOrIdentifierAsLiteral(engine_arg, local_context); - - configuration.url = S3::URI{checkAndGetLiteralArgument(engine_args[0], "url")}; - configuration.auth_settings.access_key_id = checkAndGetLiteralArgument(engine_args[1], "access_key_id"); - configuration.auth_settings.secret_access_key = checkAndGetLiteralArgument(engine_args[2], "secret_access_key"); - - if (engine_args.size() == 4) - configuration.format = checkAndGetLiteralArgument(engine_args[3], "format"); - } + auto configuration = StorageS3::getConfiguration(engine_args, local_context, false /* get_format_from_file */); if (configuration.format == "auto") configuration.format = "Parquet"; @@ -196,11 +129,9 @@ public: } private: - StorageS3::Configuration base_configuration; std::shared_ptr s3engine; Poco::Logger * log; - String table_path; }; } diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 59f49b56262..157a9171147 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -1284,7 +1284,7 @@ void StorageS3::processNamedCollectionResult(StorageS3::Configuration & configur configuration.request_settings = S3Settings::RequestSettings(collection); } -StorageS3::Configuration StorageS3::getConfiguration(ASTs & engine_args, ContextPtr local_context) +StorageS3::Configuration StorageS3::getConfiguration(ASTs & engine_args, ContextPtr local_context, bool get_format_from_file) { StorageS3::Configuration configuration; @@ -1361,7 +1361,7 @@ StorageS3::Configuration StorageS3::getConfiguration(ASTs & engine_args, Context configuration.static_configuration = !configuration.auth_settings.access_key_id.empty(); - if (configuration.format == "auto") + if (configuration.format == "auto" && get_format_from_file) configuration.format = FormatFactory::instance().getFormatFromFileName(configuration.url.key, true); return configuration; diff --git a/src/Storages/StorageS3.h b/src/Storages/StorageS3.h index 3bacfdca1b2..af3eecb33c7 100644 --- a/src/Storages/StorageS3.h +++ b/src/Storages/StorageS3.h @@ -301,7 +301,7 @@ public: bool supportsPartitionBy() const override; - static StorageS3::Configuration getConfiguration(ASTs & engine_args, ContextPtr local_context); + static StorageS3::Configuration getConfiguration(ASTs & engine_args, ContextPtr local_context, bool get_format_from_file = false); using ObjectInfos = StorageS3Source::ObjectInfos; From 0d6c6a94de01771ffe237f8e42feb503dacb568e Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 15 Feb 2023 12:18:17 +0000 Subject: [PATCH 239/717] finally fix attachProfileCountersScope --- src/Interpreters/ThreadStatusExt.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/ThreadStatusExt.cpp b/src/Interpreters/ThreadStatusExt.cpp index a770c84a3e7..8b84d4cae9b 100644 --- a/src/Interpreters/ThreadStatusExt.cpp +++ b/src/Interpreters/ThreadStatusExt.cpp @@ -169,7 +169,8 @@ ProfileEvents::Counters * ThreadStatus::attachProfileCountersScope(ProfileEvents /// Allow to attach the same scope multiple times return prev_counters; - performance_counters_scope->setParent(&performance_counters); + if (!performance_counters_scope->getParent()) + performance_counters_scope->setParent(&performance_counters); current_performance_counters = performance_counters_scope; return prev_counters; From de11a05d891e7ecb1e4afe29ea54e22fc08c4751 Mon Sep 17 00:00:00 2001 From: flynn Date: Wed, 15 Feb 2023 13:27:25 +0000 Subject: [PATCH 240/717] remove redundant header --- src/Storages/IStorageDataLake.h | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/Storages/IStorageDataLake.h b/src/Storages/IStorageDataLake.h index 90e8e5cf1b0..01e4dc835e9 100644 --- a/src/Storages/IStorageDataLake.h +++ b/src/Storages/IStorageDataLake.h @@ -6,13 +6,6 @@ # include -# include -# include - -# include -# include -# include - # include # include From f1d68cbeaf7002cab6d4d4bc68799e25bbfcf3d0 Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Wed, 15 Feb 2023 15:10:00 +0100 Subject: [PATCH 241/717] Fix test reference --- tests/queries/0_stateless/02560_null_as_default.reference | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02560_null_as_default.reference b/tests/queries/0_stateless/02560_null_as_default.reference index a500da3cbc5..d622b14dff7 100644 --- a/tests/queries/0_stateless/02560_null_as_default.reference +++ b/tests/queries/0_stateless/02560_null_as_default.reference @@ -1,4 +1,4 @@ 0 42 -World Hello +World From 9fd2226c4c5395baf8f9fdea0eced26959ba7a62 Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Wed, 15 Feb 2023 15:13:04 +0100 Subject: [PATCH 242/717] Update NativeReader.h --- src/Formats/NativeReader.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Formats/NativeReader.h b/src/Formats/NativeReader.h index 64d3e4d6df0..2d8b16e06eb 100644 --- a/src/Formats/NativeReader.h +++ b/src/Formats/NativeReader.h @@ -49,9 +49,9 @@ private: ReadBuffer & istr; Block header; UInt64 server_revision; - bool skip_unknown_columns; - bool null_as_default; - BlockMissingValues * block_missing_values; + bool skip_unknown_columns = false; + bool null_as_default = false; + BlockMissingValues * block_missing_values = nullptr; bool use_index = false; IndexForNativeFormat::Blocks::const_iterator index_block_it; From 885f14311c2ba31bb40e714a47e1f088bb204429 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 15 Feb 2023 16:37:44 +0100 Subject: [PATCH 243/717] fix --- src/Storages/StorageReplicatedMergeTree.cpp | 4 ++++ .../configs/config.d/storage_conf.xml | 7 +++++++ .../test_replicated_merge_tree_s3_zero_copy/test.py | 2 +- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index c92d4efab0d..4a8f38da349 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -928,7 +928,11 @@ void StorageReplicatedMergeTree::drop() dropAllData(); if (maybe_has_metadata_in_zookeeper) + { + /// Session could expire, get it again + zookeeper = getZooKeeperIfTableShutDown(); dropReplica(zookeeper, zookeeper_path, replica_name, log, getSettings(), &has_metadata_in_zookeeper); + } } void StorageReplicatedMergeTree::dropReplica(zkutil::ZooKeeperPtr zookeeper, const String & zookeeper_path, const String & replica, diff --git a/tests/integration/test_replicated_merge_tree_s3_zero_copy/configs/config.d/storage_conf.xml b/tests/integration/test_replicated_merge_tree_s3_zero_copy/configs/config.d/storage_conf.xml index 15239041478..801e2072f17 100644 --- a/tests/integration/test_replicated_merge_tree_s3_zero_copy/configs/config.d/storage_conf.xml +++ b/tests/integration/test_replicated_merge_tree_s3_zero_copy/configs/config.d/storage_conf.xml @@ -55,4 +55,11 @@ 3 0 + + + system + zookeeper_log
+ 5000 + 5000 + diff --git a/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py b/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py index 158d559d4b9..f955b0cb754 100644 --- a/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py +++ b/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py @@ -178,7 +178,7 @@ def test_drop_table(cluster): "create table if not exists test_drop_table (n int) engine=ReplicatedMergeTree('/test/drop_table', '1') " "order by n partition by n % 99 settings storage_policy='s3'" ) - node.query("system sync replica test_drop_table", settings={"receive_timeout": 60}) + node.query_with_retry("system sync replica test_drop_table", settings={"receive_timeout": 10}, retry_count=5) node2.query("drop table test_drop_table") assert "1000\t499500\n" == node.query( "select count(n), sum(n) from test_drop_table" From 69463f852e6b770d69357ca7ac57f2f5b6ec3a66 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 15 Feb 2023 15:44:19 +0000 Subject: [PATCH 244/717] Automatic style fix --- .../test_replicated_merge_tree_s3_zero_copy/test.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py b/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py index f955b0cb754..fb93dc8aa7c 100644 --- a/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py +++ b/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py @@ -178,7 +178,11 @@ def test_drop_table(cluster): "create table if not exists test_drop_table (n int) engine=ReplicatedMergeTree('/test/drop_table', '1') " "order by n partition by n % 99 settings storage_policy='s3'" ) - node.query_with_retry("system sync replica test_drop_table", settings={"receive_timeout": 10}, retry_count=5) + node.query_with_retry( + "system sync replica test_drop_table", + settings={"receive_timeout": 10}, + retry_count=5, + ) node2.query("drop table test_drop_table") assert "1000\t499500\n" == node.query( "select count(n), sum(n) from test_drop_table" From cfb832b20528080e20cd3fa557ab2fb596936a7a Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Wed, 15 Feb 2023 15:50:06 +0000 Subject: [PATCH 245/717] Wait for tasks finish when scheduler failed --- src/IO/S3/copyS3File.cpp | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/IO/S3/copyS3File.cpp b/src/IO/S3/copyS3File.cpp index 7f48c338d06..f687fcca8ae 100644 --- a/src/IO/S3/copyS3File.cpp +++ b/src/IO/S3/copyS3File.cpp @@ -66,17 +66,7 @@ namespace { } - virtual ~UploadHelper() - { - try - { - waitForAllBackGroundTasks(); - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__); - } - } + virtual ~UploadHelper() = default; protected: std::shared_ptr client_ptr; @@ -219,17 +209,27 @@ namespace size_t position = start_offset; size_t end_position = start_offset + size; - for (size_t part_number = 1; position < end_position; ++part_number) + try { - if (multipart_upload_aborted) - break; /// No more part uploads. + for (size_t part_number = 1; position < end_position; ++part_number) + { + if (multipart_upload_aborted) + break; /// No more part uploads. - size_t next_position = std::min(position + normal_part_size, end_position); - size_t part_size = next_position - position; /// `part_size` is either `normal_part_size` or smaller if it's the final part. + size_t next_position = std::min(position + normal_part_size, end_position); + size_t part_size = next_position - position; /// `part_size` is either `normal_part_size` or smaller if it's the final part. - uploadPart(part_number, position, part_size); + uploadPart(part_number, position, part_size); - position = next_position; + position = next_position; + } + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + abortMultipartUpload(); + waitForAllBackGroundTasks(); + throw; } waitForAllBackGroundTasks(); From cb89fccac6823569799551853e0e7b42465acd88 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Wed, 15 Feb 2023 21:26:00 +0000 Subject: [PATCH 246/717] Disable debug output --- .../QueryPlan/Optimizations/removeRedundantDistinct.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp index c0d80309db4..57684cc23a3 100644 --- a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp +++ b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp @@ -25,7 +25,7 @@ namespace DB::QueryPlanOptimizations namespace { - constexpr bool debug_logging_enabled = true; + constexpr bool debug_logging_enabled = false; template void logDebug(String key, const T & value, const char * separator = " : ") From 3ff0c7edcc3a86f5bc7baa7d5f27d41e6e445843 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Wed, 15 Feb 2023 22:37:04 +0100 Subject: [PATCH 247/717] fix typo Co-authored-by: Nikita Taranov --- src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp b/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp index d41572c9db7..c93c3ff6b53 100644 --- a/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp +++ b/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp @@ -44,7 +44,7 @@ void fillRequiredColumns(const ActionsDAG::Node * node, std::unordered_map Date: Wed, 15 Feb 2023 22:37:18 +0100 Subject: [PATCH 248/717] fix typo Co-authored-by: Nikita Taranov --- src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp b/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp index c93c3ff6b53..268ae5509d1 100644 --- a/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp +++ b/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp @@ -15,7 +15,7 @@ namespace ErrorCodes namespace { -/// Stores the ist of columns required to compute a node in the DAG. +/// Stores the list of columns required to compute a node in the DAG. struct NodeInfo { NameSet required_columns; From 69364711500f603cccb00fa04357c28d29962cfd Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Wed, 15 Feb 2023 22:37:40 +0100 Subject: [PATCH 249/717] fix include Co-authored-by: Nikita Taranov --- src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp index 95c76c81665..f811f71e23b 100644 --- a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp +++ b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp @@ -5,7 +5,7 @@ #include #include #include -#include "Storages/MergeTree/MergeTreeBaseSelectProcessor.h" +#include #include #include #include From cdf382857558eeb3091c0fc900f05f215d376733 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Wed, 15 Feb 2023 22:41:55 +0100 Subject: [PATCH 250/717] Removed unneeded default value for enable_multiple_prewhere_read_steps --- src/Storages/MergeTree/MergeTreeBaseSelectProcessor.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.h b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.h index b27a7114122..ae307a6a35c 100644 --- a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.h +++ b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.h @@ -70,6 +70,8 @@ public: const MergeTreeReaderSettings & getSettings() const { return reader_settings; } virtual std::string getName() const = 0; + + static std::unique_ptr getPrewhereActions(PrewhereInfoPtr prewhere_info, const ExpressionActionsSettings & actions_settings, bool enable_multiple_prewhere_read_steps); protected: /// This struct allow to return block with no columns but with non-zero number of rows similar to Chunk @@ -101,9 +103,6 @@ protected: static void injectVirtualColumns(Block & block, size_t row_count, MergeTreeReadTask * task, const DataTypePtr & partition_value_type, const Names & virtual_columns); -public: - static std::unique_ptr getPrewhereActions(PrewhereInfoPtr prewhere_info, const ExpressionActionsSettings & actions_settings, bool enable_multiple_prewhere_read_steps = true); - protected: static void initializeRangeReadersImpl( MergeTreeRangeReader & range_reader, From 659e680c76d458c42c667cb86cbaa6645eea5188 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 16 Feb 2023 01:24:31 +0000 Subject: [PATCH 251/717] Add a comment --- src/IO/S3/copyS3File.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/IO/S3/copyS3File.cpp b/src/IO/S3/copyS3File.cpp index f687fcca8ae..6972e9f1c82 100644 --- a/src/IO/S3/copyS3File.cpp +++ b/src/IO/S3/copyS3File.cpp @@ -227,6 +227,8 @@ namespace catch (...) { tryLogCurrentException(__PRETTY_FUNCTION__); + // Multipart upload failed because it wasn't possible to schedule all the tasks. + // To avoid execution of already scheduled tasks we abort MultipartUpload. abortMultipartUpload(); waitForAllBackGroundTasks(); throw; From 0506d9289cfae0cb240b01f5aabf6200c424ba12 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Thu, 16 Feb 2023 09:30:27 +0100 Subject: [PATCH 252/717] Updated Backup/Restore Coordination construction and removed coordination_path and added uuid in settings - Use cluster state data to check concurrent backup/restore --- src/Backups/BackupCoordinationLocal.cpp | 2 +- src/Backups/BackupCoordinationLocal.h | 2 +- src/Backups/BackupCoordinationRemote.cpp | 29 +++++++++------- src/Backups/BackupCoordinationRemote.h | 6 ++-- src/Backups/BackupSettings.cpp | 1 - src/Backups/BackupSettings.h | 4 --- src/Backups/BackupsWorker.cpp | 42 +++++++++++------------ src/Backups/IBackupCoordination.h | 2 +- src/Backups/IRestoreCoordination.h | 2 +- src/Backups/RestoreCoordinationLocal.cpp | 2 +- src/Backups/RestoreCoordinationLocal.h | 2 +- src/Backups/RestoreCoordinationRemote.cpp | 28 ++++++++------- src/Backups/RestoreCoordinationRemote.h | 6 ++-- src/Backups/RestoreSettings.cpp | 2 +- src/Backups/RestoreSettings.h | 5 +-- 15 files changed, 69 insertions(+), 66 deletions(-) diff --git a/src/Backups/BackupCoordinationLocal.cpp b/src/Backups/BackupCoordinationLocal.cpp index dd5afbed8e8..91da16097cc 100644 --- a/src/Backups/BackupCoordinationLocal.cpp +++ b/src/Backups/BackupCoordinationLocal.cpp @@ -202,7 +202,7 @@ Strings BackupCoordinationLocal::getAllArchiveSuffixes() const return archive_suffixes; } -bool BackupCoordinationLocal::hasConcurrentBackups(const String &, const String &, const std::atomic & num_active_backups) const +bool BackupCoordinationLocal::hasConcurrentBackups(const std::atomic & num_active_backups) const { return (num_active_backups > 1); } diff --git a/src/Backups/BackupCoordinationLocal.h b/src/Backups/BackupCoordinationLocal.h index 6e8a793ccd4..8e54eb6fb27 100644 --- a/src/Backups/BackupCoordinationLocal.h +++ b/src/Backups/BackupCoordinationLocal.h @@ -52,7 +52,7 @@ public: String getNextArchiveSuffix() override; Strings getAllArchiveSuffixes() const override; - bool hasConcurrentBackups(const String & backup_id, const String & common_backups_path, const std::atomic & num_active_backups) const override; + bool hasConcurrentBackups(const std::atomic & num_active_backups) const override; private: mutable std::mutex mutex; diff --git a/src/Backups/BackupCoordinationRemote.cpp b/src/Backups/BackupCoordinationRemote.cpp index 29514b8fe1f..8c696057755 100644 --- a/src/Backups/BackupCoordinationRemote.cpp +++ b/src/Backups/BackupCoordinationRemote.cpp @@ -166,14 +166,16 @@ namespace } BackupCoordinationRemote::BackupCoordinationRemote( - const String & zookeeper_path_, zkutil::GetZooKeeper get_zookeeper_, bool is_internal_) - : zookeeper_path(zookeeper_path_) + const String & root_zookeeper_path_, const String & backup_uuid_, zkutil::GetZooKeeper get_zookeeper_, bool is_internal_) + : root_zookeeper_path(root_zookeeper_path_) + , zookeeper_path(root_zookeeper_path_ + "/backup-" + backup_uuid_) + , backup_uuid(backup_uuid_) , get_zookeeper(get_zookeeper_) , is_internal(is_internal_) { createRootNodes(); stage_sync.emplace( - zookeeper_path_ + "/stage", [this] { return getZooKeeper(); }, &Poco::Logger::get("BackupCoordination")); + zookeeper_path + "/stage", [this] { return getZooKeeper(); }, &Poco::Logger::get("BackupCoordination")); } BackupCoordinationRemote::~BackupCoordinationRemote() @@ -595,36 +597,36 @@ Strings BackupCoordinationRemote::getAllArchiveSuffixes() const return node_names; } -bool BackupCoordinationRemote::hasConcurrentBackups(const String & backup_id, const String & common_backups_path, const std::atomic &) const +bool BackupCoordinationRemote::hasConcurrentBackups(const std::atomic &) const { /// If its internal concurrency will be checked for the base backup if (is_internal) return false; auto zk = getZooKeeper(); - std::string backup_stage_path = common_backups_path + "/backup-" + toString(backup_id) +"/stage"; + std::string backup_stage_path = zookeeper_path +"/stage"; - if (!zk->exists(common_backups_path)) - zk->createAncestors(common_backups_path); + if (!zk->exists(root_zookeeper_path)) + zk->createAncestors(root_zookeeper_path); for (size_t attempt = 0; attempt < MAX_ZOOKEEPER_ATTEMPTS; ++attempt) { Coordination::Stat stat; - zk->get(common_backups_path, &stat); - Strings existing_backup_paths = zk->getChildren(common_backups_path); + zk->get(root_zookeeper_path, &stat); + Strings existing_backup_paths = zk->getChildren(root_zookeeper_path); for (const auto & existing_backup_path : existing_backup_paths) { if (startsWith(existing_backup_path, "restore-")) continue; - String existing_backup_id = existing_backup_path; - existing_backup_id.erase(0, String("backup-").size()); + String existing_backup_uuid = existing_backup_path; + existing_backup_uuid.erase(0, String("backup-").size()); - if (existing_backup_id == toString(backup_id)) + if (existing_backup_uuid == toString(backup_uuid)) continue; - const auto status = zk->get(common_backups_path + "/" + existing_backup_path + "/stage"); + const auto status = zk->get(root_zookeeper_path + "/" + existing_backup_path + "/stage"); if (status != Stage::COMPLETED) return true; } @@ -637,6 +639,7 @@ bool BackupCoordinationRemote::hasConcurrentBackups(const String & backup_id, co if ((code != Coordination::Error::ZBADVERSION) || is_last_attempt) throw zkutil::KeeperException(code, backup_stage_path); } + return false; } diff --git a/src/Backups/BackupCoordinationRemote.h b/src/Backups/BackupCoordinationRemote.h index 50b98fcd77d..c7260bcd237 100644 --- a/src/Backups/BackupCoordinationRemote.h +++ b/src/Backups/BackupCoordinationRemote.h @@ -16,7 +16,7 @@ constexpr size_t MAX_ZOOKEEPER_ATTEMPTS = 10; class BackupCoordinationRemote : public IBackupCoordination { public: - BackupCoordinationRemote(const String & zookeeper_path_, zkutil::GetZooKeeper get_zookeeper_, bool is_internal_); + BackupCoordinationRemote(const String & root_zookeeper_path_, const String & backup_uuid_, zkutil::GetZooKeeper get_zookeeper_, bool is_internal_); ~BackupCoordinationRemote() override; void setStage(const String & current_host, const String & new_stage, const String & message) override; @@ -58,7 +58,7 @@ public: String getNextArchiveSuffix() override; Strings getAllArchiveSuffixes() const override; - bool hasConcurrentBackups(const String & backup_id, const String & common_backups_path, const std::atomic & num_active_backups) const override; + bool hasConcurrentBackups(const std::atomic & num_active_backups) const override; private: zkutil::ZooKeeperPtr getZooKeeper() const; @@ -68,7 +68,9 @@ private: void prepareReplicatedTables() const; void prepareReplicatedAccess() const; + const String root_zookeeper_path; const String zookeeper_path; + const String backup_uuid; const zkutil::GetZooKeeper get_zookeeper; const bool is_internal; diff --git a/src/Backups/BackupSettings.cpp b/src/Backups/BackupSettings.cpp index c801430cad7..57d85305e25 100644 --- a/src/Backups/BackupSettings.cpp +++ b/src/Backups/BackupSettings.cpp @@ -28,7 +28,6 @@ namespace ErrorCodes M(UInt64, replica_num) \ M(Bool, internal) \ M(String, host_id) \ - M(String, coordination_zk_path) \ M(OptionalUUID, backup_uuid) /// M(Int64, compression_level) diff --git a/src/Backups/BackupSettings.h b/src/Backups/BackupSettings.h index 2e7717c3afe..1b97256c75b 100644 --- a/src/Backups/BackupSettings.h +++ b/src/Backups/BackupSettings.h @@ -55,10 +55,6 @@ struct BackupSettings /// Cluster's hosts' IDs in the format 'escaped_host_name:port' for all shards and replicas in a cluster specified in BACKUP ON CLUSTER. std::vector cluster_host_ids; - /// Internal, should not be specified by user. - /// Path in Zookeeper used to coordinate a distributed backup created by BACKUP ON CLUSTER. - String coordination_zk_path; - /// Internal, should not be specified by user. /// UUID of the backup. If it's not set it will be generated randomly. std::optional backup_uuid; diff --git a/src/Backups/BackupsWorker.cpp b/src/Backups/BackupsWorker.cpp index 2bb176efce4..865151cc9ec 100644 --- a/src/Backups/BackupsWorker.cpp +++ b/src/Backups/BackupsWorker.cpp @@ -38,12 +38,12 @@ namespace Stage = BackupCoordinationStage; namespace { - std::shared_ptr makeBackupCoordination(const String & coordination_zk_path, const ContextPtr & context, bool is_internal_backup) + std::shared_ptr makeBackupCoordination(const String & root_zk_path, const String & backup_uuid, const ContextPtr & context, bool is_internal_backup) { - if (!coordination_zk_path.empty()) + if (!root_zk_path.empty()) { auto get_zookeeper = [global_context = context->getGlobalContext()] { return global_context->getZooKeeper(); }; - return std::make_shared(coordination_zk_path, get_zookeeper, is_internal_backup); + return std::make_shared(root_zk_path, backup_uuid, get_zookeeper, is_internal_backup); } else { @@ -51,12 +51,12 @@ namespace } } - std::shared_ptr makeRestoreCoordination(const String & coordination_zk_path, const ContextPtr & context, bool is_internal_backup) + std::shared_ptr makeRestoreCoordination(const String & root_zk_path, const String & restore_uuid, const ContextPtr & context, bool is_internal_backup) { - if (!coordination_zk_path.empty()) + if (!root_zk_path.empty()) { auto get_zookeeper = [global_context = context->getGlobalContext()] { return global_context->getZooKeeper(); }; - return std::make_shared(coordination_zk_path, get_zookeeper, is_internal_backup); + return std::make_shared(root_zk_path, restore_uuid, get_zookeeper, is_internal_backup); } else { @@ -160,13 +160,16 @@ OperationID BackupsWorker::startMakingBackup(const ASTPtr & query, const Context else backup_id = toString(*backup_settings.backup_uuid); + String root_zk_path; + std::shared_ptr backup_coordination; if (backup_settings.internal) { /// The following call of makeBackupCoordination() is not essential because doBackup() will later create a backup coordination /// if it's not created here. However to handle errors better it's better to make a coordination here because this way /// if an exception will be thrown in startMakingBackup() other hosts will know about that. - backup_coordination = makeBackupCoordination(backup_settings.coordination_zk_path, context, backup_settings.internal); + root_zk_path = context->getConfigRef().getString("backups.zookeeper_path", "/clickhouse/backups"); + backup_coordination = makeBackupCoordination(root_zk_path, toString(*backup_settings.backup_uuid), context, backup_settings.internal); } auto backup_info = BackupInfo::fromAST(*backup_query->backup_name); @@ -270,17 +273,13 @@ void BackupsWorker::doBackup( backup_query->cluster = context->getMacros()->expand(backup_query->cluster); cluster = context->getCluster(backup_query->cluster); backup_settings.cluster_host_ids = cluster->getHostIDs(); - if (backup_settings.coordination_zk_path.empty()) - { - backup_settings.coordination_zk_path = root_zk_path + "/backup-" + toString(*backup_settings.backup_uuid); - } } /// Make a backup coordination. if (!backup_coordination) - backup_coordination = makeBackupCoordination(backup_settings.coordination_zk_path, context, backup_settings.internal); + backup_coordination = makeBackupCoordination(root_zk_path, toString(*backup_settings.backup_uuid), context, backup_settings.internal); - if (!allow_concurrent_backups && backup_coordination->hasConcurrentBackups(backup_id, root_zk_path, std::ref(num_active_backups))) + if (!allow_concurrent_backups && backup_coordination->hasConcurrentBackups(std::ref(num_active_backups))) throw Exception(ErrorCodes::CONCURRENT_ACCESS_NOT_SUPPORTED, "Concurrent backups not supported, turn on setting 'allow_concurrent_backups'"); /// Opens a backup for writing. @@ -383,6 +382,9 @@ OperationID BackupsWorker::startRestoring(const ASTPtr & query, ContextMutablePt auto restore_query = std::static_pointer_cast(query->clone()); auto restore_settings = RestoreSettings::fromRestoreQuery(*restore_query); + if (!restore_settings.restore_uuid) + restore_settings.restore_uuid = UUIDHelpers::generateV4(); + /// `restore_id` will be used as a key to the `infos` map, so it should be unique. OperationID restore_id; if (restore_settings.internal) @@ -390,7 +392,7 @@ OperationID BackupsWorker::startRestoring(const ASTPtr & query, ContextMutablePt else if (!restore_settings.id.empty()) restore_id = restore_settings.id; else - restore_id = toString(UUIDHelpers::generateV4()); + restore_id = toString(*restore_settings.restore_uuid); std::shared_ptr restore_coordination; if (restore_settings.internal) @@ -398,7 +400,8 @@ OperationID BackupsWorker::startRestoring(const ASTPtr & query, ContextMutablePt /// The following call of makeRestoreCoordination() is not essential because doRestore() will later create a restore coordination /// if it's not created here. However to handle errors better it's better to make a coordination here because this way /// if an exception will be thrown in startRestoring() other hosts will know about that. - restore_coordination = makeRestoreCoordination(restore_settings.coordination_zk_path, context, restore_settings.internal); + auto root_zk_path = context->getConfigRef().getString("backups.zookeeper_path", "/clickhouse/backups"); + restore_coordination = makeRestoreCoordination(root_zk_path, toString(*restore_settings.restore_uuid), context, restore_settings.internal); } try @@ -518,15 +521,10 @@ void BackupsWorker::doRestore( } /// Make a restore coordination. - if (on_cluster && restore_settings.coordination_zk_path.empty()) - { - restore_settings.coordination_zk_path = root_zk_path + "/restore-" + toString(restore_id); - } - if (!restore_coordination) - restore_coordination = makeRestoreCoordination(restore_settings.coordination_zk_path, context, restore_settings.internal); + restore_coordination = makeRestoreCoordination(root_zk_path, toString(*restore_settings.restore_uuid), context, restore_settings.internal); - if (!allow_concurrent_restores && restore_coordination->hasConcurrentRestores(restore_id, root_zk_path, std::ref(num_active_restores))) + if (!allow_concurrent_restores && restore_coordination->hasConcurrentRestores(std::ref(num_active_restores))) throw Exception(ErrorCodes::CONCURRENT_ACCESS_NOT_SUPPORTED, "Concurrent restores not supported, turn on setting 'allow_concurrent_restores'"); /// Do RESTORE. diff --git a/src/Backups/IBackupCoordination.h b/src/Backups/IBackupCoordination.h index 787b86f8cbd..f5fa01a1530 100644 --- a/src/Backups/IBackupCoordination.h +++ b/src/Backups/IBackupCoordination.h @@ -117,7 +117,7 @@ public: /// This function is used to check if concurrent backups are running /// other than the backup passed to the function - virtual bool hasConcurrentBackups(const String & backup_id, const String & common_backups_path, const std::atomic & num_active_backups) const = 0; + virtual bool hasConcurrentBackups(const std::atomic & num_active_backups) const = 0; }; } diff --git a/src/Backups/IRestoreCoordination.h b/src/Backups/IRestoreCoordination.h index a2506e5c157..098d048f6a3 100644 --- a/src/Backups/IRestoreCoordination.h +++ b/src/Backups/IRestoreCoordination.h @@ -37,7 +37,7 @@ public: /// This function is used to check if concurrent restores are running /// other than the restore passed to the function - virtual bool hasConcurrentRestores(const String & restore_id, const String & common_restores_path, const std::atomic & num_active_restores) const = 0; + virtual bool hasConcurrentRestores(const std::atomic & num_active_restores) const = 0; }; } diff --git a/src/Backups/RestoreCoordinationLocal.cpp b/src/Backups/RestoreCoordinationLocal.cpp index 56951c56baa..4e908d5d67d 100644 --- a/src/Backups/RestoreCoordinationLocal.cpp +++ b/src/Backups/RestoreCoordinationLocal.cpp @@ -42,7 +42,7 @@ bool RestoreCoordinationLocal::acquireReplicatedAccessStorage(const String &) return true; } -bool RestoreCoordinationLocal::hasConcurrentRestores(const String &, const String &, const std::atomic & num_active_restores) const +bool RestoreCoordinationLocal::hasConcurrentRestores(const std::atomic & num_active_restores) const { return (num_active_restores > 1); } diff --git a/src/Backups/RestoreCoordinationLocal.h b/src/Backups/RestoreCoordinationLocal.h index fa6941a7577..ab9d1ce0a59 100644 --- a/src/Backups/RestoreCoordinationLocal.h +++ b/src/Backups/RestoreCoordinationLocal.h @@ -35,7 +35,7 @@ public: /// The function returns false if this access storage is being already restored by another replica. bool acquireReplicatedAccessStorage(const String & access_storage_zk_path) override; - bool hasConcurrentRestores(const String & restore_id, const String & common_restores_path, const std::atomic & num_active_restores) const override; + bool hasConcurrentRestores(const std::atomic & num_active_restores) const override; private: std::set> acquired_tables_in_replicated_databases; diff --git a/src/Backups/RestoreCoordinationRemote.cpp b/src/Backups/RestoreCoordinationRemote.cpp index 95766bfcae3..f829cd20c0d 100644 --- a/src/Backups/RestoreCoordinationRemote.cpp +++ b/src/Backups/RestoreCoordinationRemote.cpp @@ -10,15 +10,17 @@ namespace DB namespace Stage = BackupCoordinationStage; RestoreCoordinationRemote::RestoreCoordinationRemote( - const String & zookeeper_path_, zkutil::GetZooKeeper get_zookeeper_, bool is_internal_) - : zookeeper_path(zookeeper_path_) + const String & root_zookeeper_path_, const String & restore_uuid_, zkutil::GetZooKeeper get_zookeeper_, bool is_internal_) + : root_zookeeper_path(root_zookeeper_path_) + , zookeeper_path(root_zookeeper_path_ + "/restore-" + restore_uuid_) + , restore_uuid(restore_uuid_) , get_zookeeper(get_zookeeper_) , is_internal(is_internal_) { createRootNodes(); stage_sync.emplace( - zookeeper_path_ + "/stage", [this] { return getZooKeeper(); }, &Poco::Logger::get("RestoreCoordination")); + zookeeper_path + "/stage", [this] { return getZooKeeper(); }, &Poco::Logger::get("RestoreCoordination")); } RestoreCoordinationRemote::~RestoreCoordinationRemote() @@ -132,36 +134,36 @@ void RestoreCoordinationRemote::removeAllNodes() zk->removeRecursive(zookeeper_path); } -bool RestoreCoordinationRemote::hasConcurrentRestores(const String & restore_id, const String & common_restores_path, const std::atomic &) const +bool RestoreCoordinationRemote::hasConcurrentRestores(const std::atomic &) const { /// If its internal concurrency will be checked for the base restore if (is_internal) return false; auto zk = getZooKeeper(); - std::string path = common_restores_path + "/restore-" + toString(restore_id) +"/stage"; + std::string path = zookeeper_path +"/stage"; - if (! zk->exists(common_restores_path)) - zk->createAncestors(common_restores_path); + if (! zk->exists(root_zookeeper_path)) + zk->createAncestors(root_zookeeper_path); for (size_t attempt = 0; attempt < MAX_ZOOKEEPER_ATTEMPTS; ++attempt) { Coordination::Stat stat; - zk->get(common_restores_path, &stat); - Strings existing_restore_paths = zk->getChildren(common_restores_path); + zk->get(root_zookeeper_path, &stat); + Strings existing_restore_paths = zk->getChildren(root_zookeeper_path); for (const auto & existing_restore_path : existing_restore_paths) { if (startsWith(existing_restore_path, "backup-")) continue; - String existing_restore_id = existing_restore_path; - existing_restore_id.erase(0, String("restore-").size()); + String existing_restore_uuid = existing_restore_path; + existing_restore_uuid.erase(0, String("restore-").size()); - if (existing_restore_id == toString(restore_id)) + if (existing_restore_uuid == toString(restore_uuid)) continue; - const auto status = zk->get(common_restores_path + "/" + existing_restore_path + "/stage"); + const auto status = zk->get(root_zookeeper_path + "/" + existing_restore_path + "/stage"); if (status != Stage::COMPLETED) return true; } diff --git a/src/Backups/RestoreCoordinationRemote.h b/src/Backups/RestoreCoordinationRemote.h index 67d78192e1e..d72781bac2f 100644 --- a/src/Backups/RestoreCoordinationRemote.h +++ b/src/Backups/RestoreCoordinationRemote.h @@ -11,7 +11,7 @@ namespace DB class RestoreCoordinationRemote : public IRestoreCoordination { public: - RestoreCoordinationRemote(const String & zookeeper_path_, zkutil::GetZooKeeper get_zookeeper_, bool is_internal_); + RestoreCoordinationRemote(const String & root_zookeeper_path_, const String & restore_uuid_, zkutil::GetZooKeeper get_zookeeper_, bool is_internal_); ~RestoreCoordinationRemote() override; /// Sets the current stage and waits for other hosts to come to this stage too. @@ -31,7 +31,7 @@ public: /// The function returns false if this access storage is being already restored by another replica. bool acquireReplicatedAccessStorage(const String & access_storage_zk_path) override; - bool hasConcurrentRestores(const String & restore_id, const String & common_restores_path, const std::atomic & num_active_restores) const override; + bool hasConcurrentRestores(const std::atomic & num_active_restores) const override; private: zkutil::ZooKeeperPtr getZooKeeper() const; @@ -40,7 +40,9 @@ private: class ReplicatedDatabasesMetadataSync; + const String root_zookeeper_path; const String zookeeper_path; + const String restore_uuid; const zkutil::GetZooKeeper get_zookeeper; const bool is_internal; diff --git a/src/Backups/RestoreSettings.cpp b/src/Backups/RestoreSettings.cpp index bbcefb819cf..d12da704b2d 100644 --- a/src/Backups/RestoreSettings.cpp +++ b/src/Backups/RestoreSettings.cpp @@ -163,7 +163,7 @@ namespace M(RestoreUDFCreationMode, create_function) \ M(Bool, internal) \ M(String, host_id) \ - M(String, coordination_zk_path) + M(OptionalUUID, restore_uuid) RestoreSettings RestoreSettings::fromRestoreQuery(const ASTBackupQuery & query) diff --git a/src/Backups/RestoreSettings.h b/src/Backups/RestoreSettings.h index 713adbe8029..3bce8698620 100644 --- a/src/Backups/RestoreSettings.h +++ b/src/Backups/RestoreSettings.h @@ -119,8 +119,9 @@ struct RestoreSettings std::vector cluster_host_ids; /// Internal, should not be specified by user. - /// Path in Zookeeper used to coordinate restoring process while executing by RESTORE ON CLUSTER. - String coordination_zk_path; + /// UUID of the restore. If it's not set it will be generated randomly. + /// This is used to generate coordination path and for concurrency check + std::optional restore_uuid; static RestoreSettings fromRestoreQuery(const ASTBackupQuery & query); void copySettingsToQuery(ASTBackupQuery & query) const; From 4a7dd63f41995c88abdce6ccd5a59074de60ea77 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Thu, 16 Feb 2023 12:04:48 +0800 Subject: [PATCH 253/717] add new function regexp_extract, like in spark --- src/Functions/regexpExtract.cpp | 275 ++++++++++++++++++ .../02665_regexp_extract.reference | 30 ++ .../0_stateless/02665_regexp_extract.sql | 40 +++ 3 files changed, 345 insertions(+) create mode 100644 src/Functions/regexpExtract.cpp create mode 100644 tests/queries/0_stateless/02665_regexp_extract.reference create mode 100644 tests/queries/0_stateless/02665_regexp_extract.sql diff --git a/src/Functions/regexpExtract.cpp b/src/Functions/regexpExtract.cpp new file mode 100644 index 00000000000..bcf7cd98be3 --- /dev/null +++ b/src/Functions/regexpExtract.cpp @@ -0,0 +1,275 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int ILLEGAL_COLUMN; + extern const int INDEX_OF_POSITIONAL_ARGUMENT_IS_OUT_OF_RANGE; +} + +namespace +{ + class FunctionRegexpExtract : public IFunction + { + public: + static constexpr auto name = "regexpExtract"; + static FunctionPtr create(ContextPtr) { return std::make_shared(); } + + String getName() const override { return name; } + + bool isVariadic() const override { return true; } + size_t getNumberOfArguments() const override { return 0; } + + bool useDefaultImplementationForConstants() const override { return true; } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; } + + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + if (arguments.size() != 2 && arguments.size() != 3) + throw Exception( + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Number of arguments for function {} doesn't match: passed {}", + getName(), + arguments.size()); + + if (!isString(arguments[0])) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of first argument of function {}", + arguments[0]->getName(), + getName()); + + if (!isString(arguments[1])) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of second argument of function {}", + arguments[1]->getName(), + getName()); + + if (arguments.size() > 2 && !isInteger(arguments[2])) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of third argument of function {}", + arguments[2]->getName(), + getName()); + + return std::make_shared(); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override + { + const ColumnPtr column = arguments[0].column; + const ColumnPtr column_pattern = arguments[1].column; + const ColumnPtr column_index = arguments.size() > 2 ? arguments[2].column : nullptr; + + /// Check if the second argument is const column + const ColumnConst * col_pattern = typeid_cast(&*column_pattern); + if (!col_pattern) + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Second argument of function {} must be constant string", getName()); + + + /// Check if the first argument is string column(const or not) + const ColumnConst * col_const = typeid_cast(&*column); + const ColumnString * col = nullptr; + if (col_const) + col = typeid_cast(&col_const->getDataColumn()); + else + col = typeid_cast(&*column); + if (!col) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of argument of function {}", arguments[0].column->getName(), getName()); + + auto col_res = ColumnString::create(); + ColumnString::Chars & vec_res = col_res->getChars(); + ColumnString::Offsets & offsets_res = col_res->getOffsets(); + + if (col_const) + constantVector(col_const->getValue(), col_pattern->getValue(), column_index, vec_res, offsets_res); + else if (!column_index || isColumnConst(*column_index)) + { + const auto * col_const_index = typeid_cast(&*column_index); + ssize_t index = !col_const_index ? 1 : col_const_index->getInt(0); + vectorConstant(col->getChars(), col->getOffsets(), col_pattern->getValue(), index, vec_res, offsets_res); + } + else + vectorVector(col->getChars(), col->getOffsets(), col_pattern->getValue(), column_index, vec_res, offsets_res); + + return col_res; + } + + private: + static void vectorConstant( + const ColumnString::Chars & data, + const ColumnString::Offsets & offsets, + const std::string & pattern, + ssize_t index, + ColumnString::Chars & res_data, + ColumnString::Offsets & res_offsets) + { + const Regexps::Regexp regexp = Regexps::createRegexp(pattern); + unsigned capture = regexp.getNumberOfSubpatterns(); + if (index < 0 || index >= capture + 1) + throw Exception( + ErrorCodes::INDEX_OF_POSITIONAL_ARGUMENT_IS_OUT_OF_RANGE, + "Index value {} is out of range, should be in [0, {})", + index, + capture + 1); + + OptimizedRegularExpression::MatchVec matches; + matches.reserve(index + 1); + + res_data.reserve(data.size() / 5); + res_offsets.resize(offsets.size()); + size_t prev_offset = 0; + size_t res_offset = 0; + for (size_t i = 0; i < offsets.size(); ++i) + { + size_t cur_offset = offsets[i]; + unsigned count = regexp.match( + reinterpret_cast(&data[prev_offset]), + cur_offset - prev_offset - 1, + matches, + static_cast(index + 1)); + + if (count == index + 1 && matches[index].offset != std::string::npos) + { + const auto & match = matches[index]; + res_data.resize(res_offset + match.length + 1); + memcpySmallAllowReadWriteOverflow15(&res_data[res_offset], &data[prev_offset + match.offset], match.length); + res_offset += match.length; + } + else + res_data.resize(res_offset + 1); + + res_data[res_offset] = 0; + ++res_offset; + res_offsets[i] = res_offset; + + prev_offset = cur_offset; + } + } + + static void vectorVector( + const ColumnString::Chars & data, + const ColumnString::Offsets & offsets, + const std::string & pattern, + const ColumnPtr & column_index, + ColumnString::Chars & res_data, + ColumnString::Offsets & res_offsets) + { + res_data.reserve(data.size() / 5); + res_offsets.resize(offsets.size()); + + const Regexps::Regexp regexp = Regexps::createRegexp(pattern); + unsigned capture = regexp.getNumberOfSubpatterns(); + + OptimizedRegularExpression::MatchVec matches; + matches.reserve(capture + 1); + size_t prev_offset = 0; + size_t res_offset = 0; + for (size_t i = 0; i < offsets.size(); ++i) + { + size_t cur_offset = offsets[i]; + + ssize_t index = column_index->getInt(i); + if (index < 0 || index >= capture + 1) + throw Exception( + ErrorCodes::INDEX_OF_POSITIONAL_ARGUMENT_IS_OUT_OF_RANGE, + "Index value {} is out of range, should be in [0, {})", + index, + capture + 1); + + unsigned count = regexp.match( + reinterpret_cast(&data[prev_offset]), + cur_offset - prev_offset - 1, + matches, + static_cast(index + 1)); + + if (count == index + 1 && matches[index].offset != std::string::npos) + { + const auto & match = matches[index]; + res_data.resize(res_offset + match.length + 1); + memcpySmallAllowReadWriteOverflow15(&res_data[res_offset], &data[prev_offset + match.offset], match.length); + res_offset += match.length; + } + else + res_data.resize(res_offset + 1); + + res_data[res_offset] = 0; + ++res_offset; + res_offsets[i] = res_offset; + + prev_offset = cur_offset; + } + } + + static void constantVector( + const std::string & str, + const std::string & pattern, + const ColumnPtr & column_index, + ColumnString::Chars & res_data, + ColumnString::Offsets & res_offsets) + { + size_t rows = column_index->size(); + res_data.reserve(str.size() / 5); + res_offsets.resize(rows); + + const Regexps::Regexp regexp = Regexps::createRegexp(pattern); + unsigned capture = regexp.getNumberOfSubpatterns(); + OptimizedRegularExpression::MatchVec matches; + matches.reserve(capture + 1); + unsigned count = regexp.match(str.data(), str.size(), matches, static_cast(capture + 1)); + bool found = count == capture + 1; + + size_t res_offset = 0; + for (size_t i = 0; i < rows; ++i) + { + ssize_t index = column_index->getInt(i); + if (index < 0 || index >= capture + 1) + throw Exception( + ErrorCodes::INDEX_OF_POSITIONAL_ARGUMENT_IS_OUT_OF_RANGE, + "Index value {} is out of range, should be in [0, {})", + index, + capture + 1); + + if (found && matches[index].offset != std::string::npos) + { + const auto & match = matches[index]; + res_data.resize(res_offset + match.length + 1); + memcpySmallAllowReadWriteOverflow15(&res_data[res_offset], str.data() + match.offset, match.length); + res_offset += match.length; + } + else + res_data.resize(res_offset + 1); + + res_data[res_offset] = 0; + ++res_offset; + res_offsets[i] = res_offset; + } + } + }; +} + +REGISTER_FUNCTION(RegexpExtract) +{ + factory.registerFunction(); + + /// For Spark compatibility. + factory.registerAlias("REGEXP_EXTRACT", "regexpExtract", FunctionFactory::CaseInsensitive); +} + +} diff --git a/tests/queries/0_stateless/02665_regexp_extract.reference b/tests/queries/0_stateless/02665_regexp_extract.reference new file mode 100644 index 00000000000..812ed7ac8f5 --- /dev/null +++ b/tests/queries/0_stateless/02665_regexp_extract.reference @@ -0,0 +1,30 @@ +100 +100 +200 +100 + +\N +\N +\N +100 +100 +200 +100 + +\N +\N +\N +100 +200 +100 + +\N +\N +\N +100 +200 +100 + +\N +\N +\N diff --git a/tests/queries/0_stateless/02665_regexp_extract.sql b/tests/queries/0_stateless/02665_regexp_extract.sql new file mode 100644 index 00000000000..9966bda50f4 --- /dev/null +++ b/tests/queries/0_stateless/02665_regexp_extract.sql @@ -0,0 +1,40 @@ +select regexpExtract('100-200', '(\\d+)-(\\d+)', 1); +select regexpExtract('100-200', '(\\d+)-(\\d+)'); +select regexpExtract('100-200', '(\\d+)-(\\d+)', 2); +select regexpExtract('100-200', '(\\d+).*', 1); +select regexpExtract('100-200', '([a-z])', 1); +select regexpExtract(null, '([a-z])', 1); +select regexpExtract('100-200', null, 1); +select regexpExtract('100-200', '([a-z])', null); + +select regexpExtract(materialize('100-200'), '(\\d+)-(\\d+)'); +select regexpExtract(materialize('100-200'), '(\\d+)-(\\d+)', 1); +select regexpExtract(materialize('100-200'), '(\\d+)-(\\d+)', 2); +select regexpExtract(materialize('100-200'), '(\\d+).*', 1); +select regexpExtract(materialize('100-200'), '([a-z])', 1); +select regexpExtract(materialize(null), '([a-z])', 1); +select regexpExtract(materialize('100-200'), null, 1); +select regexpExtract(materialize('100-200'), '([a-z])', null); + +select regexpExtract('100-200', '(\\d+)-(\\d+)', materialize(1)); +select regexpExtract('100-200', '(\\d+)-(\\d+)', materialize(2)); +select regexpExtract('100-200', '(\\d+).*', materialize(1)); +select regexpExtract('100-200', '([a-z])', materialize(1)); +select regexpExtract(null, '([a-z])', materialize(1)); +select regexpExtract('100-200', null, materialize(1)); +select regexpExtract('100-200', '([a-z])', materialize(null)); + +select regexpExtract(materialize('100-200'), '(\\d+)-(\\d+)', materialize(1)); +select regexpExtract(materialize('100-200'), '(\\d+)-(\\d+)', materialize(2)); +select regexpExtract(materialize('100-200'), '(\\d+).*', materialize(1)); +select regexpExtract(materialize('100-200'), '([a-z])', materialize(1)); +select regexpExtract(materialize(null), '([a-z])', materialize(1)); +select regexpExtract(materialize('100-200'), null, materialize(1)); +select regexpExtract(materialize('100-200'), '([a-z])', materialize(null)); + + +select regexpExtract('100-200'); -- { serverError 42 } +select regexpExtract('100-200', '(\\d+)-(\\d+)', 1, 2); -- { serverError 42 } +select regexpExtract(cast('100-200' as FixedString(10)), '(\\d+)-(\\d+)', 1); -- { serverError 43 } +select regexpExtract('100-200', cast('(\\d+)-(\\d+)' as FixedString(20)), 1); -- { serverError 43 } +select regexpExtract('100-200', materialize('(\\d+)-(\\d+)'), 1); -- { serverError 44 } From eadac085bf2da1b72c85b70a2634d6e50bb35be8 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Thu, 16 Feb 2023 17:33:51 +0800 Subject: [PATCH 254/717] add docs for function regexExtract --- .../functions/string-search-functions.md | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md index b6b70c7795b..e52eb00ab44 100644 --- a/docs/en/sql-reference/functions/string-search-functions.md +++ b/docs/en/sql-reference/functions/string-search-functions.md @@ -863,3 +863,41 @@ Result: │ 2 │ └───────────────────────────────┘ ``` + +## regexpExtract(haystack, pattern[, index]) + +Extracts the first string in haystack that matches the regexp pattern and corresponds to the regex group index. + +**Syntax** + +``` sql +regexpExtract(haystack, pattern[, index]) +``` + +Alias: `REGEXP_EXTRACT(haystack, pattern[, index])`. + +**Arguments** + +- `haystack` — String, in which regexp pattern will to be matched. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `pattern` — String, regexp expression, must be constant. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `index` – An integer number greater or equal 0 with default 1. It represents which regex group to extract. [UInt or Int](../../sql-reference/data-types/int-uint.md). Optional. + +**Returned values** + +`pattern` may contain multiple regexp groups, `index` indicates which regex group to extract. An index of 0 means matching the entire regular expression. + +Type: `String`. + +**Examples** + +``` sql +SELECT + regexpExtract('100-200', '(\\d+)-(\\d+)', 1), + regexpExtract('100-200', '(\\d+)-(\\d+)', 2), + regexpExtract('100-200', '(\\d+)-(\\d+)', 0), + regexpExtract('100-200', '(\\d+)-(\\d+)') + +┌─regexpExtract('100-200', '(\\d+)-(\\d+)', 1)─┬─regexpExtract('100-200', '(\\d+)-(\\d+)', 2)─┬─regexpExtract('100-200', '(\\d+)-(\\d+)', 0)─┬─regexpExtract('100-200', '(\\d+)-(\\d+)')─┐ +│ 100 │ 200 │ 100-200 │ 100 │ +└──────────────────────────────────────────────┴──────────────────────────────────────────────┴──────────────────────────────────────────────┴───────────────────────────────────────────┘ +``` From 2968cdc8f671da05f70b17314202bf213faeb4f1 Mon Sep 17 00:00:00 2001 From: flynn Date: Thu, 16 Feb 2023 10:18:22 +0000 Subject: [PATCH 255/717] fix --- src/Storages/StorageS3.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/StorageS3.h b/src/Storages/StorageS3.h index 5f7fe2bae0e..dabbb25d828 100644 --- a/src/Storages/StorageS3.h +++ b/src/Storages/StorageS3.h @@ -301,7 +301,7 @@ public: bool supportsPartitionBy() const override; - static StorageS3::Configuration getConfiguration(ASTs & engine_args, ContextPtr local_context, bool get_format_from_file = false); + static StorageS3::Configuration getConfiguration(ASTs & engine_args, ContextPtr local_context, bool get_format_from_file = true); using ObjectInfos = StorageS3Source::ObjectInfos; From d9ffbeb4a2d26aa351ad8c39ed801d7c551a071b Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Thu, 16 Feb 2023 10:51:13 +0000 Subject: [PATCH 256/717] Small improvements --- src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp | 4 ++-- src/Parsers/ParserRenameQuery.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp index 4e54b806bd1..d95a7a42159 100644 --- a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp +++ b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp @@ -591,9 +591,9 @@ ASTs InterpreterRenameImpl::getRewrittenQueries( { elements.push_back(ASTRenameQuery::Element()); elements.back().from.database = std::make_shared(mapped_to_database); - elements.back().from.table = rename_element.from.table; + elements.back().from.table = rename_element.from.table->clone(); elements.back().to.database = std::make_shared(mapped_to_database); - elements.back().to.table = rename_element.to.table; + elements.back().to.table = rename_element.to.table->clone(); } } diff --git a/src/Parsers/ParserRenameQuery.cpp b/src/Parsers/ParserRenameQuery.cpp index cb98554afba..cb595846633 100644 --- a/src/Parsers/ParserRenameQuery.cpp +++ b/src/Parsers/ParserRenameQuery.cpp @@ -61,8 +61,8 @@ bool ParserRenameQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) query->elements.front().if_exists = if_exists; query->elements.front().from.database = from_db; query->elements.front().to.database = to_db; - query->children.push_back(from_db); - query->children.push_back(to_db); + query->children.push_back(std::move(from_db)); + query->children.push_back(std::move(to_db)); query->cluster = cluster_str; node = query; return true; From 96fb6302315f6aac2e1ba4d737b3c8ca68ed2def Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Fri, 20 Jan 2023 10:51:34 +0100 Subject: [PATCH 257/717] Analyzer Planner enable by default --- src/Core/Settings.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 1cd5e93c499..e9db3e5ed06 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -305,7 +305,7 @@ class IColumn; M(Float, opentelemetry_start_trace_probability, 0., "Probability to start an OpenTelemetry trace for an incoming query.", 0) \ M(Bool, opentelemetry_trace_processors, false, "Collect OpenTelemetry spans for processors.", 0) \ M(Bool, prefer_column_name_to_alias, false, "Prefer using column names instead of aliases if possible.", 0) \ - M(Bool, allow_experimental_analyzer, false, "Allow experimental analyzer", 0) \ + M(Bool, allow_experimental_analyzer, true, "Allow experimental analyzer", 0) \ M(Bool, prefer_global_in_and_join, false, "If enabled, all IN/JOIN operators will be rewritten as GLOBAL IN/JOIN. It's useful when the to-be-joined tables are only available on the initiator and we need to always scatter their data on-the-fly during distributed processing with the GLOBAL keyword. It's also useful to reduce the need to access the external sources joining external tables.", 0) \ \ \ From 77fee977052c584bd1c30e39ebcaf4c0fc20e492 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Fri, 20 Jan 2023 14:35:18 +0100 Subject: [PATCH 258/717] MergeTreeIndexFullText fix prepared set index analysis --- src/Analyzer/SetUtils.cpp | 2 +- .../MergeTree/MergeTreeIndexFullText.cpp | 9 +++++--- src/Storages/MergeTree/RPNBuilder.cpp | 21 ++++++++++--------- src/Storages/MergeTree/RPNBuilder.h | 6 ++++++ 4 files changed, 24 insertions(+), 14 deletions(-) diff --git a/src/Analyzer/SetUtils.cpp b/src/Analyzer/SetUtils.cpp index a72879d2145..0fb075e925b 100644 --- a/src/Analyzer/SetUtils.cpp +++ b/src/Analyzer/SetUtils.cpp @@ -170,7 +170,7 @@ SetPtr makeSetForConstantValue(const DataTypePtr & expression_type, const Field value_type->getName()); } - auto set = std::make_shared(size_limits_for_set, false /*fill_set_elements*/, tranform_null_in); + auto set = std::make_shared(size_limits_for_set, true /*fill_set_elements*/, tranform_null_in); set->setHeader(result_block.cloneEmpty().getColumnsWithTypeAndName()); set->insertFromBlock(result_block.getColumnsWithTypeAndName()); diff --git a/src/Storages/MergeTree/MergeTreeIndexFullText.cpp b/src/Storages/MergeTree/MergeTreeIndexFullText.cpp index 35ca484cff0..2745c1e4c36 100644 --- a/src/Storages/MergeTree/MergeTreeIndexFullText.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexFullText.cpp @@ -638,12 +638,15 @@ bool MergeTreeConditionFullText::tryPrepareSetBloomFilter( return false; auto prepared_set = right_argument.tryGetPreparedSet(data_types); - if (!prepared_set) + if (!prepared_set || !prepared_set->hasExplicitSetElements()) return false; - for (const auto & data_type : prepared_set->getDataTypes()) - if (data_type->getTypeId() != TypeIndex::String && data_type->getTypeId() != TypeIndex::FixedString) + for (const auto & prepared_set_data_type : prepared_set->getDataTypes()) + { + auto prepared_set_data_type_id = prepared_set_data_type->getTypeId(); + if (prepared_set_data_type_id != TypeIndex::String && prepared_set_data_type_id != TypeIndex::FixedString) return false; + } std::vector> bloom_filters; std::vector key_position; diff --git a/src/Storages/MergeTree/RPNBuilder.cpp b/src/Storages/MergeTree/RPNBuilder.cpp index b1d726335ae..cee5038ed21 100644 --- a/src/Storages/MergeTree/RPNBuilder.cpp +++ b/src/Storages/MergeTree/RPNBuilder.cpp @@ -29,7 +29,7 @@ namespace ErrorCodes namespace { -void appendColumnNameWithoutAlias(const ActionsDAG::Node & node, WriteBuffer & out, bool legacy = false) +void appendColumnNameWithoutAlias(const ActionsDAG::Node & node, WriteBuffer & out, bool allow_experimental_analyzer, bool legacy = false) { switch (node.type) { @@ -40,19 +40,19 @@ void appendColumnNameWithoutAlias(const ActionsDAG::Node & node, WriteBuffer & o { /// If it was created from ASTLiteral, then result_name can be an alias. /// We need to convert value back to string here. - if (const auto * column_const = typeid_cast(node.column.get())) + const auto * column_const = typeid_cast(node.column.get()); + if (column_const && !allow_experimental_analyzer) writeString(applyVisitor(FieldVisitorToString(), column_const->getField()), out); - /// It may be possible that column is ColumnSet else writeString(node.result_name, out); break; } case ActionsDAG::ActionType::ALIAS: - appendColumnNameWithoutAlias(*node.children.front(), out, legacy); + appendColumnNameWithoutAlias(*node.children.front(), out, allow_experimental_analyzer, legacy); break; case ActionsDAG::ActionType::ARRAY_JOIN: writeCString("arrayJoin(", out); - appendColumnNameWithoutAlias(*node.children.front(), out, legacy); + appendColumnNameWithoutAlias(*node.children.front(), out, allow_experimental_analyzer, legacy); writeChar(')', out); break; case ActionsDAG::ActionType::FUNCTION: @@ -71,17 +71,18 @@ void appendColumnNameWithoutAlias(const ActionsDAG::Node & node, WriteBuffer & o writeCString(", ", out); first = false; - appendColumnNameWithoutAlias(*arg, out, legacy); + appendColumnNameWithoutAlias(*arg, out, allow_experimental_analyzer, legacy); } writeChar(')', out); } } } -String getColumnNameWithoutAlias(const ActionsDAG::Node & node, bool legacy = false) +String getColumnNameWithoutAlias(const ActionsDAG::Node & node, bool allow_experimental_analyzer, bool legacy = false) { WriteBufferFromOwnString out; - appendColumnNameWithoutAlias(node, out, legacy); + appendColumnNameWithoutAlias(node, out, allow_experimental_analyzer, legacy); + return std::move(out.str()); } @@ -116,7 +117,7 @@ std::string RPNBuilderTreeNode::getColumnName() const if (ast_node) return ast_node->getColumnNameWithoutAlias(); else - return getColumnNameWithoutAlias(*dag_node); + return getColumnNameWithoutAlias(*dag_node, getTreeContext().getSettings().allow_experimental_analyzer); } std::string RPNBuilderTreeNode::getColumnNameWithModuloLegacy() const @@ -129,7 +130,7 @@ std::string RPNBuilderTreeNode::getColumnNameWithModuloLegacy() const } else { - return getColumnNameWithoutAlias(*dag_node, true /*legacy*/); + return getColumnNameWithoutAlias(*dag_node, getTreeContext().getSettings().allow_experimental_analyzer, true /*legacy*/); } } diff --git a/src/Storages/MergeTree/RPNBuilder.h b/src/Storages/MergeTree/RPNBuilder.h index 132d3aa44e8..741821f75fb 100644 --- a/src/Storages/MergeTree/RPNBuilder.h +++ b/src/Storages/MergeTree/RPNBuilder.h @@ -31,6 +31,12 @@ public: return query_context; } + /// Get query context settings + const Settings & getSettings() const + { + return query_context->getSettingsRef(); + } + /** Get block with constants. * Valid only for AST tree. */ From c4cb3bb03f1934eb3187abc01d405be7c760e216 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sat, 21 Jan 2023 11:18:00 +0100 Subject: [PATCH 259/717] Analyzer fix combinators with NULL argument --- src/AggregateFunctions/AggregateFunctionNothing.cpp | 4 +++- src/AggregateFunctions/AggregateFunctionNothing.h | 9 ++------- src/AggregateFunctions/AggregateFunctionNull.cpp | 6 ++---- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionNothing.cpp b/src/AggregateFunctions/AggregateFunctionNothing.cpp index b476806da08..ebeffffc71e 100644 --- a/src/AggregateFunctions/AggregateFunctionNothing.cpp +++ b/src/AggregateFunctions/AggregateFunctionNothing.cpp @@ -13,7 +13,9 @@ void registerAggregateFunctionNothing(AggregateFunctionFactory & factory) factory.registerFunction("nothing", [](const std::string & name, const DataTypes & argument_types, const Array & parameters, const Settings *) { assertNoParameters(name, parameters); - return std::make_shared(argument_types, parameters); + + auto result_type = argument_types.empty() ? std::make_shared(std::make_shared()) : argument_types.front(); + return std::make_shared(argument_types, parameters, result_type); }); } diff --git a/src/AggregateFunctions/AggregateFunctionNothing.h b/src/AggregateFunctions/AggregateFunctionNothing.h index de8a5868e04..8491e8edfaa 100644 --- a/src/AggregateFunctions/AggregateFunctionNothing.h +++ b/src/AggregateFunctions/AggregateFunctionNothing.h @@ -19,19 +19,14 @@ struct Settings; class AggregateFunctionNothing final : public IAggregateFunctionHelper { public: - AggregateFunctionNothing(const DataTypes & arguments, const Array & params) - : IAggregateFunctionHelper(arguments, params, createResultType(arguments)) {} + AggregateFunctionNothing(const DataTypes & arguments, const Array & params, const DataTypePtr & result_type_) + : IAggregateFunctionHelper(arguments, params, result_type_) {} String getName() const override { return "nothing"; } - static DataTypePtr createResultType(const DataTypes & arguments) - { - return arguments.empty() ? std::make_shared(std::make_shared()) : arguments.front(); - } - bool allocatesMemoryInArena() const override { return false; } void create(AggregateDataPtr __restrict) const override diff --git a/src/AggregateFunctions/AggregateFunctionNull.cpp b/src/AggregateFunctions/AggregateFunctionNull.cpp index dc522fe472d..3d3d7af3026 100644 --- a/src/AggregateFunctions/AggregateFunctionNull.cpp +++ b/src/AggregateFunctions/AggregateFunctionNull.cpp @@ -72,11 +72,9 @@ public: { /// Currently the only functions that returns not-NULL on all NULL arguments are count and uniq, and they returns UInt64. if (properties.returns_default_when_only_null) - return std::make_shared(DataTypes{ - std::make_shared()}, params); + return std::make_shared(arguments, params, std::make_shared()); else - return std::make_shared(DataTypes{ - std::make_shared(std::make_shared())}, params); + return std::make_shared(arguments, params, std::make_shared(std::make_shared())); } assert(nested_function); From 03f14f7c5020398f76185a0367968c113f7dbbe0 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sat, 21 Jan 2023 11:20:25 +0100 Subject: [PATCH 260/717] Analyzer fix MutationsInterpreter --- src/Analyzer/TableNode.cpp | 7 ++++ src/Analyzer/TableNode.h | 3 ++ src/Interpreters/MutationsInterpreter.cpp | 49 +++++++++++++++++++---- 3 files changed, 51 insertions(+), 8 deletions(-) diff --git a/src/Analyzer/TableNode.cpp b/src/Analyzer/TableNode.cpp index 0d9a351e9a2..1018570c3d4 100644 --- a/src/Analyzer/TableNode.cpp +++ b/src/Analyzer/TableNode.cpp @@ -26,6 +26,13 @@ TableNode::TableNode(StoragePtr storage_, TableLockHolder storage_lock_, Storage { } +TableNode::TableNode(StoragePtr storage_, const ContextPtr & context) + : TableNode(storage_, + storage_->lockForShare(context->getInitialQueryId(), context->getSettingsRef().lock_acquire_timeout), + storage_->getStorageSnapshot(storage_->getInMemoryMetadataPtr(), context)) +{ +} + void TableNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const { buffer << std::string(indent, ' ') << "TABLE id: " << format_state.getNodeId(this); diff --git a/src/Analyzer/TableNode.h b/src/Analyzer/TableNode.h index c7feedd908f..4965de535df 100644 --- a/src/Analyzer/TableNode.h +++ b/src/Analyzer/TableNode.h @@ -29,6 +29,9 @@ public: /// Construct table node with storage, storage lock, storage snapshot explicit TableNode(StoragePtr storage_, TableLockHolder storage_lock_, StorageSnapshotPtr storage_snapshot_); + /// Construct table node with storage, context + explicit TableNode(StoragePtr storage_, const ContextPtr & context); + /// Get storage const StoragePtr & getStorage() const { diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp index c11ee14478c..0b52a1a51bc 100644 --- a/src/Interpreters/MutationsInterpreter.cpp +++ b/src/Interpreters/MutationsInterpreter.cpp @@ -33,6 +33,11 @@ #include #include #include +#include +#include +#include +#include +#include namespace DB @@ -167,6 +172,21 @@ ASTPtr prepareQueryAffectedAST(const std::vector & commands, co return select; } +QueryTreeNodePtr prepareQueryAffectedQueryTree(const std::vector & commands, const StoragePtr & storage, ContextPtr context) +{ + auto ast = prepareQueryAffectedAST(commands, storage, context); + auto query_tree = buildQueryTree(ast, context); + + auto & query_node = query_tree->as(); + query_node.getJoinTree() = std::make_shared(storage, context); + + QueryTreePassManager query_tree_pass_manager(context); + addQueryTreePasses(query_tree_pass_manager); + query_tree_pass_manager.run(query_tree); + + return query_tree; +} + ColumnDependencies getAllColumnDependencies(const StorageMetadataPtr & metadata_snapshot, const NameSet & updated_columns) { NameSet new_updated_columns = updated_columns; @@ -221,16 +241,29 @@ bool isStorageTouchedByMutations( if (all_commands_can_be_skipped) return false; - ASTPtr select_query = prepareQueryAffectedAST(commands, storage.shared_from_this(), context); - auto storage_from_part = std::make_shared(source_part); - /// Interpreter must be alive, when we use result of execute() method. - /// For some reason it may copy context and give it into ExpressionTransform - /// after that we will use context from destroyed stack frame in our stream. - InterpreterSelectQuery interpreter( - select_query, context, storage_from_part, metadata_snapshot, SelectQueryOptions().ignoreLimits().ignoreProjections()); - auto io = interpreter.execute(); + std::optional interpreter_select_query; + BlockIO io; + + if (context->getSettingsRef().allow_experimental_analyzer) + { + auto select_query_tree = prepareQueryAffectedQueryTree(commands, storage.shared_from_this(), context); + InterpreterSelectQueryAnalyzer interpreter(select_query_tree, context, SelectQueryOptions().ignoreLimits().ignoreProjections()); + io = interpreter.execute(); + } + else + { + ASTPtr select_query = prepareQueryAffectedAST(commands, storage.shared_from_this(), context); + /// Interpreter must be alive, when we use result of execute() method. + /// For some reason it may copy context and give it into ExpressionTransform + /// after that we will use context from destroyed stack frame in our stream. + interpreter_select_query.emplace( + select_query, context, storage_from_part, metadata_snapshot, SelectQueryOptions().ignoreLimits().ignoreProjections()); + + io = interpreter_select_query->execute(); + } + PullingAsyncPipelineExecutor executor(io.pipeline); Block block; From 440f2f5834ba07b94693462a192b7c0546d6e98f Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sat, 21 Jan 2023 16:50:28 +0100 Subject: [PATCH 261/717] Function fromModifiedJulianDay argument types fix --- src/Functions/fromModifiedJulianDay.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Functions/fromModifiedJulianDay.cpp b/src/Functions/fromModifiedJulianDay.cpp index 53e5fbffe52..8e76bb27ff1 100644 --- a/src/Functions/fromModifiedJulianDay.cpp +++ b/src/Functions/fromModifiedJulianDay.cpp @@ -165,8 +165,8 @@ namespace DB FunctionBasePtr buildImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & return_type) const override { - const DataTypePtr & from_type = removeNullable(arguments[0].type); - DataTypes argument_types = { from_type }; + DataTypes argument_types = { arguments[0].type }; + const DataTypePtr & from_type_not_null = removeNullable(arguments[0].type); FunctionBasePtr base; auto call = [&](const auto & types) -> bool @@ -178,7 +178,7 @@ namespace DB base = std::make_unique>(argument_types, return_type); return true; }; - bool built = callOnBasicType(from_type->getTypeId(), call); + bool built = callOnBasicType(from_type_not_null->getTypeId(), call); if (built) return base; @@ -188,7 +188,7 @@ namespace DB * here causes a SEGV. So we must somehow create a * dummy implementation and return it. */ - if (WhichDataType(from_type).isNothing()) // Nullable(Nothing) + if (WhichDataType(from_type_not_null).isNothing()) // Nullable(Nothing) return std::make_unique>(argument_types, return_type); else // Should not happen. From 2aef3318465ef39bdb9f59790adb8d8d40cb61ea Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sat, 21 Jan 2023 16:52:05 +0100 Subject: [PATCH 262/717] Analyzer JOIN disable USING for non identifier nodes --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 29ce4079cc3..64e57063afb 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -5667,7 +5667,10 @@ void QueryAnalyzer::resolveQueryJoinTreeNode(QueryTreeNodePtr & join_tree_node, { auto * identifier_node = join_using_node->as(); if (!identifier_node) - continue; + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "JOIN {} USING clause expected identifier. Actual {}", + join_node.formatASTForErrorMessage(), + join_using_node->formatASTForErrorMessage()); const auto & identifier_full_name = identifier_node->getIdentifier().getFullName(); From 945ce7ca63585c91797ad47f01da05e1944c661a Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sun, 22 Jan 2023 12:06:21 +0100 Subject: [PATCH 263/717] Planner support indexHint function --- src/Analyzer/ListNode.h | 4 +++ src/Planner/PlannerActionsVisitor.cpp | 40 +++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/src/Analyzer/ListNode.h b/src/Analyzer/ListNode.h index 4e09512d780..75013f7ee6a 100644 --- a/src/Analyzer/ListNode.h +++ b/src/Analyzer/ListNode.h @@ -17,6 +17,7 @@ class ListNode final : public IQueryTreeNode { public: using iterator = QueryTreeNodes::iterator; + using const_iterator = QueryTreeNodes::const_iterator; /// Initialize list node with empty nodes ListNode(); @@ -44,7 +45,10 @@ public: void dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const override; iterator begin() { return children.begin(); } + const_iterator begin() const { return children.begin(); } + iterator end() { return children.end(); } + const_iterator end() const { return children.end(); } protected: bool isEqualImpl(const IQueryTreeNode & rhs) const override; diff --git a/src/Planner/PlannerActionsVisitor.cpp b/src/Planner/PlannerActionsVisitor.cpp index 95edd93dd9f..a9a4d21721e 100644 --- a/src/Planner/PlannerActionsVisitor.cpp +++ b/src/Planner/PlannerActionsVisitor.cpp @@ -22,6 +22,7 @@ #include #include +#include #include #include @@ -172,6 +173,8 @@ private: NodeNameAndNodeMinLevel makeSetForInFunction(const QueryTreeNodePtr & node); + NodeNameAndNodeMinLevel visitIndexHintFunction(const QueryTreeNodePtr & node); + NodeNameAndNodeMinLevel visitFunction(const QueryTreeNodePtr & node); std::vector actions_stack; @@ -373,11 +376,48 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::ma return {set_key, 0}; } +PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::visitIndexHintFunction(const QueryTreeNodePtr & node) +{ + const auto & function_node = node->as(); + auto function_node_name = calculateActionNodeName(node, *planner_context, node_to_node_name); + + auto index_hint_actions_dag = std::make_shared(); + auto & index_hint_actions_dag_outputs = index_hint_actions_dag->getOutputs(); + std::unordered_set index_hint_actions_dag_output_node_names; + PlannerActionsVisitor actions_visitor(planner_context); + + for (const auto & argument : function_node.getArguments()) + { + auto index_hint_argument_expression_dag_nodes = actions_visitor.visit(index_hint_actions_dag, argument); + + for (auto & expression_dag_node : index_hint_argument_expression_dag_nodes) + { + if (index_hint_actions_dag_output_node_names.contains(expression_dag_node->result_name)) + continue; + + index_hint_actions_dag_output_node_names.insert(expression_dag_node->result_name); + index_hint_actions_dag_outputs.push_back(expression_dag_node); + } + } + + auto index_hint_function = std::make_shared(); + index_hint_function->setActions(std::move(index_hint_actions_dag)); + auto index_hint_function_overload_resolver = std::make_shared(std::move(index_hint_function)); + + size_t index_hint_function_level = actions_stack.size() - 1; + actions_stack[index_hint_function_level].addFunctionIfNecessary(function_node_name, {}, index_hint_function_overload_resolver); + + return {function_node_name, index_hint_function_level}; +} + PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::visitFunction(const QueryTreeNodePtr & node) { const auto & function_node = node->as(); + if (function_node.getFunctionName() == "indexHint") + return visitIndexHintFunction(node); std::optional in_function_second_argument_node_name_with_level; + if (isNameOfInFunction(function_node.getFunctionName())) in_function_second_argument_node_name_with_level = makeSetForInFunction(node); From 7ffee1dafe6d718a0b0c55ffd08f89b9e261b509 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sun, 22 Jan 2023 12:16:19 +0100 Subject: [PATCH 264/717] QueryTreePassManager disable validation for aggregate functions with LowCardinality arguments --- src/Analyzer/QueryTreePassManager.cpp | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/Analyzer/QueryTreePassManager.cpp b/src/Analyzer/QueryTreePassManager.cpp index de68ad42ba4..38d2ae731a1 100644 --- a/src/Analyzer/QueryTreePassManager.cpp +++ b/src/Analyzer/QueryTreePassManager.cpp @@ -8,6 +8,7 @@ #include #include +#include #include @@ -105,14 +106,22 @@ private: if (WhichDataType(expected_argument_types[i]).isFunction()) continue; - if (!expected_argument_types[i]->equals(*actual_argument_columns[i].type)) + const auto & expected_argument_type = expected_argument_types[i]; + const auto & actual_argument_type = actual_argument_columns[i].type; + + if (!expected_argument_type->equals(*actual_argument_type)) { + /// Aggregate functions remove low cardinality for their argument types + if ((function->isAggregateFunction() || function->isWindowFunction()) && + expected_argument_type->equals(*recursiveRemoveLowCardinality(actual_argument_type))) + continue; + throw Exception(ErrorCodes::LOGICAL_ERROR, "Function {} expects {} argument to have {} type but receives {} after running {} pass", function->toAST()->formatForErrorMessage(), i + 1, - expected_argument_types[i]->getName(), - actual_argument_columns[i].type->getName(), + expected_argument_type->getName(), + actual_argument_type->getName(), pass_name); } } From cbcfd404c8a00064b0582b2ebc0f71b9ae2e20fd Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sun, 22 Jan 2023 12:36:20 +0100 Subject: [PATCH 265/717] AggregateFunctionSequenceNextNode fix argument types --- src/AggregateFunctions/AggregateFunctionSequenceNextNode.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AggregateFunctions/AggregateFunctionSequenceNextNode.h b/src/AggregateFunctions/AggregateFunctionSequenceNextNode.h index 51524f23f9b..76610772b22 100644 --- a/src/AggregateFunctions/AggregateFunctionSequenceNextNode.h +++ b/src/AggregateFunctions/AggregateFunctionSequenceNextNode.h @@ -190,7 +190,7 @@ public: SequenceDirection seq_direction_, size_t min_required_args_, UInt64 max_elems_ = std::numeric_limits::max()) - : IAggregateFunctionDataHelper, Self>({data_type_}, parameters_, data_type_) + : IAggregateFunctionDataHelper, Self>(arguments, parameters_, data_type_) , seq_base_kind(seq_base_kind_) , seq_direction(seq_direction_) , min_required_args(min_required_args_) From c7c5e29ff739684a2c5910175f7f802ff6453802 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sun, 22 Jan 2023 14:09:38 +0100 Subject: [PATCH 266/717] Analyzer resolve identifier as table expression crash fix --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 41 ++++++++++++----------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 64e57063afb..1cf7c8354bb 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -4769,29 +4769,32 @@ ProjectionNames QueryAnalyzer::resolveExpressionNode(QueryTreeNodePtr & node, Id { node = tryResolveIdentifier({unresolved_identifier, IdentifierLookupContext::TABLE_EXPRESSION}, scope).resolved_identifier; - /// If table identifier is resolved as CTE clone it and resolve - auto * subquery_node = node->as(); - auto * union_node = node->as(); - bool resolved_as_cte = (subquery_node && subquery_node->isCTE()) || (union_node && union_node->isCTE()); - - if (resolved_as_cte) + if (node) { - node = node->clone(); - subquery_node = node->as(); - union_node = node->as(); + /// If table identifier is resolved as CTE clone it and resolve + auto * subquery_node = node->as(); + auto * union_node = node->as(); + bool resolved_as_cte = (subquery_node && subquery_node->isCTE()) || (union_node && union_node->isCTE()); - if (subquery_node) - subquery_node->setIsCTE(false); - else - union_node->setIsCTE(false); + if (resolved_as_cte) + { + node = node->clone(); + subquery_node = node->as(); + union_node = node->as(); - IdentifierResolveScope subquery_scope(node, &scope /*parent_scope*/); - subquery_scope.subquery_depth = scope.subquery_depth + 1; + if (subquery_node) + subquery_node->setIsCTE(false); + else + union_node->setIsCTE(false); - if (subquery_node) - resolveQuery(node, subquery_scope); - else - resolveUnion(node, subquery_scope); + IdentifierResolveScope subquery_scope(node, &scope /*parent_scope*/); + subquery_scope.subquery_depth = scope.subquery_depth + 1; + + if (subquery_node) + resolveQuery(node, subquery_scope); + else + resolveUnion(node, subquery_scope); + } } } From bb56af0e2b80dad8bb6b3dedbb173b1fa0cd34e2 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sun, 22 Jan 2023 15:31:04 +0100 Subject: [PATCH 267/717] MergeTreeIndexGin crash fix --- src/Storages/MergeTree/MergeTreeIndexInverted.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreeIndexInverted.cpp b/src/Storages/MergeTree/MergeTreeIndexInverted.cpp index 02222aa530c..fa49f53c765 100644 --- a/src/Storages/MergeTree/MergeTreeIndexInverted.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexInverted.cpp @@ -655,7 +655,7 @@ bool MergeTreeConditionInverted::tryPrepareSetGinFilter( return false; ConstSetPtr prepared_set = rhs.tryGetPreparedSet(); - if (!prepared_set && !prepared_set->hasExplicitSetElements()) + if (!prepared_set || !prepared_set->hasExplicitSetElements()) return false; for (const auto & data_type : prepared_set->getDataTypes()) From 84f3f8e18b835d8636a562d4a71ba3892df9e495 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sun, 22 Jan 2023 16:23:44 +0100 Subject: [PATCH 268/717] Planner UNION different number of columns exception fix --- src/Planner/Planner.cpp | 2 +- src/Planner/Utils.cpp | 18 +++++++++++++++--- src/Planner/Utils.h | 3 ++- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp index 02cf0961920..f9d54f9db65 100644 --- a/src/Planner/Planner.cpp +++ b/src/Planner/Planner.cpp @@ -998,7 +998,7 @@ void Planner::buildPlanForUnionNode() query_plans.push_back(std::move(query_node_plan)); } - Block union_common_header = buildCommonHeaderForUnion(query_plans_headers); + Block union_common_header = buildCommonHeaderForUnion(query_plans_headers, union_mode); DataStreams query_plans_streams; query_plans_streams.reserve(query_plans.size()); diff --git a/src/Planner/Utils.cpp b/src/Planner/Utils.cpp index 221e683f4e2..a991816def9 100644 --- a/src/Planner/Utils.cpp +++ b/src/Planner/Utils.cpp @@ -28,6 +28,8 @@ namespace ErrorCodes { extern const int TYPE_MISMATCH; extern const int LOGICAL_ERROR; + extern const int UNION_ALL_RESULT_STRUCTURES_MISMATCH; + extern const int INTERSECT_OR_EXCEPT_RESULT_STRUCTURES_MISMATCH; } String dumpQueryPlan(QueryPlan & query_plan) @@ -47,7 +49,7 @@ String dumpQueryPipeline(QueryPlan & query_plan) return query_pipeline_buffer.str(); } -Block buildCommonHeaderForUnion(const Blocks & queries_headers) +Block buildCommonHeaderForUnion(const Blocks & queries_headers, SelectUnionMode union_mode) { size_t num_selects = queries_headers.size(); Block common_header = queries_headers.front(); @@ -55,9 +57,19 @@ Block buildCommonHeaderForUnion(const Blocks & queries_headers) for (size_t query_number = 1; query_number < num_selects; ++query_number) { + int error_code = 0; + + if (union_mode == SelectUnionMode::UNION_DEFAULT || + union_mode == SelectUnionMode::UNION_ALL || + union_mode == SelectUnionMode::UNION_DISTINCT) + error_code = ErrorCodes::UNION_ALL_RESULT_STRUCTURES_MISMATCH; + else + error_code = ErrorCodes::INTERSECT_OR_EXCEPT_RESULT_STRUCTURES_MISMATCH; + if (queries_headers.at(query_number).columns() != columns_size) - throw Exception(ErrorCodes::TYPE_MISMATCH, - "Different number of columns in UNION elements: {} and {}", + throw Exception(error_code, + "Different number of columns in {} elements: {} and {}", + toString(union_mode), common_header.dumpNames(), queries_headers[query_number].dumpNames()); } diff --git a/src/Planner/Utils.h b/src/Planner/Utils.h index 3ec1ed3a947..f8b3f7a96d2 100644 --- a/src/Planner/Utils.h +++ b/src/Planner/Utils.h @@ -4,6 +4,7 @@ #include #include +#include #include #include @@ -26,7 +27,7 @@ String dumpQueryPlan(QueryPlan & query_plan); String dumpQueryPipeline(QueryPlan & query_plan); /// Build common header for UNION query -Block buildCommonHeaderForUnion(const Blocks & queries_headers); +Block buildCommonHeaderForUnion(const Blocks & queries_headers, SelectUnionMode union_mode); /// Convert query node to ASTSelectQuery ASTPtr queryNodeToSelectQuery(const QueryTreeNodePtr & query_node); From 8eb93b216ba96113128b283e7e59b9a333d1c27d Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sun, 22 Jan 2023 16:36:13 +0100 Subject: [PATCH 269/717] Planner JOIN TREE unused columns drop fix --- src/Planner/PlannerJoinTree.cpp | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index 6cac504557d..a66145074f1 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -759,14 +759,24 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_ auto drop_unused_columns_after_join_actions_dag = std::make_shared(result_plan.getCurrentDataStream().header.getColumnsWithTypeAndName()); ActionsDAG::NodeRawConstPtrs updated_outputs; std::unordered_set updated_outputs_names; + std::optional first_skipped_column_node_index; - for (auto & output : drop_unused_columns_after_join_actions_dag->getOutputs()) + auto & drop_unused_columns_after_join_actions_dag_outputs = drop_unused_columns_after_join_actions_dag->getOutputs(); + size_t drop_unused_columns_after_join_actions_dag_outputs_size = drop_unused_columns_after_join_actions_dag_outputs.size(); + + for (size_t i = 0; i < drop_unused_columns_after_join_actions_dag_outputs_size; ++i) { + const auto & output = drop_unused_columns_after_join_actions_dag_outputs[i]; + const auto & global_planner_context = planner_context->getGlobalPlannerContext(); if (updated_outputs_names.contains(output->result_name) - || !global_planner_context->hasColumnIdentifier(output->result_name) - || !outer_scope_columns.contains(output->result_name)) + || !global_planner_context->hasColumnIdentifier(output->result_name)) + continue; + + if (!outer_scope_columns.contains(output->result_name)) { + if (!first_skipped_column_node_index) + first_skipped_column_node_index = i; continue; } @@ -774,7 +784,14 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_ updated_outputs_names.insert(output->result_name); } - drop_unused_columns_after_join_actions_dag->getOutputs() = std::move(updated_outputs); + /** It is expected that JOIN TREE query plan will contain at least 1 column, even if there are no columns in outer scope. + * + * Example: SELECT count() FROM test_table_1 AS t1, test_table_2 AS t2; + */ + if (updated_outputs.empty() && first_skipped_column_node_index) + updated_outputs.push_back(drop_unused_columns_after_join_actions_dag_outputs[*first_skipped_column_node_index]); + + drop_unused_columns_after_join_actions_dag_outputs = std::move(updated_outputs); auto drop_unused_columns_after_join_transform_step = std::make_unique(result_plan.getCurrentDataStream(), std::move(drop_unused_columns_after_join_actions_dag)); drop_unused_columns_after_join_transform_step->setStepDescription("DROP unused columns after JOIN"); From 9b7710ba40d0fd1b6d6775ffc11509c6a23ec27d Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sun, 22 Jan 2023 16:43:57 +0100 Subject: [PATCH 270/717] Analyzer count function handling with unqualified matcher fix --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 1cf7c8354bb..3d60cd43af8 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -4073,7 +4073,7 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi is_special_function_exists = function_name == "exists"; /// Handle SELECT count(*) FROM test_table - if (function_name == "count" && function_node_ptr->getArguments().getNodes().size() == 1) + if (Poco::toLower(function_name) == "count" && function_node_ptr->getArguments().getNodes().size() == 1) { auto * matcher_node = function_node_ptr->getArguments().getNodes().front()->as(); if (matcher_node && matcher_node->isUnqualified()) From 45f45b290cf83eb3aa104be43bba1a93e9c5ad16 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sun, 22 Jan 2023 17:15:46 +0100 Subject: [PATCH 271/717] Analyzer recursive SQL user defined functions fix --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 3d60cd43af8..01ac128d505 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -1107,7 +1107,7 @@ private: static QueryTreeNodePtr wrapExpressionNodeInTupleElement(QueryTreeNodePtr expression_node, IdentifierView nested_path); - static QueryTreeNodePtr tryGetLambdaFromSQLUserDefinedFunctions(const std::string & function_name, ContextPtr context); + QueryTreeNodePtr tryGetLambdaFromSQLUserDefinedFunctions(const std::string & function_name, ContextPtr context); void evaluateScalarSubqueryIfNeeded(QueryTreeNodePtr & query_tree_node, size_t subquery_depth, ContextPtr context); @@ -1207,6 +1207,9 @@ private: /// Lambdas that are currently in resolve process std::unordered_set lambdas_in_resolve_process; + /// Function name to user defined lambda map + std::unordered_map function_name_to_user_defined_lambda; + /// Array join expressions counter size_t array_join_expressions_counter = 0; @@ -1710,6 +1713,10 @@ QueryTreeNodePtr QueryAnalyzer::tryGetLambdaFromSQLUserDefinedFunctions(const st if (!user_defined_function) return {}; + auto it = function_name_to_user_defined_lambda.find(function_name); + if (it != function_name_to_user_defined_lambda.end()) + return it->second; + const auto & create_function_query = user_defined_function->as(); auto result_node = buildQueryTree(create_function_query->function_core, context); if (result_node->getNodeType() != QueryTreeNodeType::LAMBDA) @@ -1718,6 +1725,8 @@ QueryTreeNodePtr QueryAnalyzer::tryGetLambdaFromSQLUserDefinedFunctions(const st function_name, create_function_query->function_core->formatForErrorMessage()); + function_name_to_user_defined_lambda.emplace(function_name, result_node); + return result_node; } @@ -2271,11 +2280,11 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromAliases(const Identifier identifier_bind_part, scope.scope_node->formatASTForErrorMessage()); - if (auto root_expression_wih_alias = scope.expressions_in_resolve_process_stack.getExpressionWithAlias(identifier_bind_part)) + if (auto root_expression_with_alias = scope.expressions_in_resolve_process_stack.getExpressionWithAlias(identifier_bind_part)) { const auto top_expression = scope.expressions_in_resolve_process_stack.getTop(); - if (!isNodePartOfTree(top_expression.get(), root_expression_wih_alias.get())) + if (!isNodePartOfTree(top_expression.get(), root_expression_with_alias.get())) throw Exception(ErrorCodes::CYCLIC_ALIASES, "Cyclic aliases for identifier '{}'. In scope {}", identifier_lookup.identifier.getFullName(), From 0fe3ba36c4c12dea6efb7571d8aca16fb8cc8812 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sun, 22 Jan 2023 18:02:38 +0100 Subject: [PATCH 272/717] Analyzer added ParitionPruner support --- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 10 ++++-- src/Storages/MergeTree/PartitionPruner.cpp | 31 +++++++++++++++++-- src/Storages/MergeTree/PartitionPruner.h | 26 ++++++---------- 3 files changed, 46 insertions(+), 21 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 19d7b803bf6..a108ee18931 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -794,14 +794,18 @@ void MergeTreeDataSelectExecutor::filterPartsByPartition( std::optional minmax_idx_condition; DataTypes minmax_columns_types; - if (metadata_snapshot->hasPartitionKey() && !settings.allow_experimental_analyzer) + if (metadata_snapshot->hasPartitionKey()) { const auto & partition_key = metadata_snapshot->getPartitionKey(); auto minmax_columns_names = data.getMinMaxColumnsNames(partition_key); + auto minmax_expression_actions = data.getMinMaxExpr(partition_key, ExpressionActionsSettings::fromContext(context)); minmax_columns_types = data.getMinMaxColumnsTypes(partition_key); - minmax_idx_condition.emplace( - query_info, context, minmax_columns_names, data.getMinMaxExpr(partition_key, ExpressionActionsSettings::fromContext(context))); + if (context->getSettingsRef().allow_experimental_analyzer) + minmax_idx_condition.emplace(query_info.filter_actions_dag, context, minmax_columns_names, minmax_expression_actions, NameSet()); + else + minmax_idx_condition.emplace(query_info, context, minmax_columns_names, minmax_expression_actions); + partition_pruner.emplace(metadata_snapshot, query_info, context, false /* strict */); if (settings.force_index_by_date && (minmax_idx_condition->alwaysUnknownOrTrue() && partition_pruner->isUseless())) diff --git a/src/Storages/MergeTree/PartitionPruner.cpp b/src/Storages/MergeTree/PartitionPruner.cpp index dce52ef64e9..61293888f10 100644 --- a/src/Storages/MergeTree/PartitionPruner.cpp +++ b/src/Storages/MergeTree/PartitionPruner.cpp @@ -3,14 +3,38 @@ namespace DB { -bool PartitionPruner::canBePruned(const DataPart & part) +namespace +{ + +KeyCondition buildKeyCondition(const KeyDescription & partition_key, const SelectQueryInfo & query_info, ContextPtr context, bool strict) +{ + if (context->getSettingsRef().allow_experimental_analyzer) + return {query_info.filter_actions_dag, context, partition_key.column_names, partition_key.expression, {}, true /* single_point */, strict}; + + return {query_info, context, partition_key.column_names, partition_key.expression, true /* single_point */, strict}; +} + +} + +PartitionPruner::PartitionPruner(const StorageMetadataPtr & metadata, const SelectQueryInfo & query_info, ContextPtr context, bool strict) + : partition_key(MergeTreePartition::adjustPartitionKey(metadata, context)) + , partition_condition(buildKeyCondition(partition_key, query_info, context, strict)) + , useless(strict ? partition_condition.anyUnknownOrAlwaysTrue() : partition_condition.alwaysUnknownOrTrue()) +{ +} + +bool PartitionPruner::canBePruned(const IMergeTreeDataPart & part) { if (part.isEmpty()) return true; + const auto & partition_id = part.info.partition_id; - bool is_valid; + bool is_valid = false; + if (auto it = partition_filter_map.find(partition_id); it != partition_filter_map.end()) + { is_valid = it->second; + } else { const auto & partition_value = part.partition.value; @@ -21,9 +45,11 @@ bool PartitionPruner::canBePruned(const DataPart & part) if (field.isNull()) field = POSITIVE_INFINITY; } + is_valid = partition_condition.mayBeTrueInRange( partition_value.size(), index_value.data(), index_value.data(), partition_key.data_types); partition_filter_map.emplace(partition_id, is_valid); + if (!is_valid) { WriteBufferFromOwnString buf; @@ -31,6 +57,7 @@ bool PartitionPruner::canBePruned(const DataPart & part) LOG_TRACE(&Poco::Logger::get("PartitionPruner"), "Partition {} gets pruned", buf.str()); } } + return !is_valid; } diff --git a/src/Storages/MergeTree/PartitionPruner.h b/src/Storages/MergeTree/PartitionPruner.h index 9953c52b593..3a986923321 100644 --- a/src/Storages/MergeTree/PartitionPruner.h +++ b/src/Storages/MergeTree/PartitionPruner.h @@ -12,6 +12,15 @@ namespace DB /// Pruning partitions in verbatim way using KeyCondition class PartitionPruner { +public: + PartitionPruner(const StorageMetadataPtr & metadata, const SelectQueryInfo & query_info, ContextPtr context, bool strict); + + bool canBePruned(const IMergeTreeDataPart & part); + + bool isUseless() const { return useless; } + + const KeyCondition & getKeyCondition() const { return partition_condition; } + private: std::unordered_map partition_filter_map; @@ -19,23 +28,8 @@ private: KeyDescription partition_key; KeyCondition partition_condition; - bool useless; - using DataPart = IMergeTreeDataPart; - using DataPartPtr = std::shared_ptr; -public: - PartitionPruner(const StorageMetadataPtr & metadata, const SelectQueryInfo & query_info, ContextPtr context, bool strict) - : partition_key(MergeTreePartition::adjustPartitionKey(metadata, context)) - , partition_condition(query_info, context, partition_key.column_names, partition_key.expression, true /* single_point */, strict) - , useless(strict ? partition_condition.anyUnknownOrAlwaysTrue() : partition_condition.alwaysUnknownOrTrue()) - { - } - - bool canBePruned(const DataPart & part); - - bool isUseless() const { return useless; } - - const KeyCondition & getKeyCondition() const { return partition_condition; } + bool useless = false; }; } From fa0ef340b6c92cccda64b2ee880c659e1e3b2192 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sun, 22 Jan 2023 18:03:50 +0100 Subject: [PATCH 273/717] Analyzer IN function invalid arguments size error fix --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 01ac128d505..76b0370ef23 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -4151,7 +4151,7 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi { auto & function_in_arguments_nodes = function_node.getArguments().getNodes(); if (function_in_arguments_nodes.size() != 2) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Function '{}' expects 2 arguments", function_name); + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function '{}' expects 2 arguments", function_name); auto & in_second_argument = function_in_arguments_nodes[1]; auto * table_node = in_second_argument->as(); From 75cbb0e0d083c3fc5f4210ac1029c3acbb77703e Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Mon, 23 Jan 2023 10:59:55 +0100 Subject: [PATCH 274/717] Analyzer scalar subqueries execution fix --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 76b0370ef23..f36a751f1dc 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -1765,6 +1765,7 @@ void QueryAnalyzer::evaluateScalarSubqueryIfNeeded(QueryTreeNodePtr & node, size PullingAsyncPipelineExecutor executor(io.pipeline); io.pipeline.setProgressCallback(context->getProgressCallback()); + io.pipeline.setProcessListElement(context->getProcessListElement()); Block block; From 301d9e22c2b1e38fd4243d8947006cbd28b5823a Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Mon, 23 Jan 2023 11:27:48 +0100 Subject: [PATCH 275/717] Analyzer change countDistinct function implementation using count_distinct_implementation setting --- .../Passes/CustomizeFunctionsPass.cpp | 29 ------------------- src/Analyzer/Passes/QueryAnalysisPass.cpp | 20 +++++++++++-- src/Analyzer/Passes/QueryAnalysisPass.h | 2 ++ 3 files changed, 19 insertions(+), 32 deletions(-) diff --git a/src/Analyzer/Passes/CustomizeFunctionsPass.cpp b/src/Analyzer/Passes/CustomizeFunctionsPass.cpp index eef31afdc50..3333008a0c0 100644 --- a/src/Analyzer/Passes/CustomizeFunctionsPass.cpp +++ b/src/Analyzer/Passes/CustomizeFunctionsPass.cpp @@ -38,22 +38,6 @@ public: { auto count_distinct_implementation_function_name = String(settings.count_distinct_implementation); - /// Replace countDistinct with countDistinct implementation - if (function_name_lowercase == "countdistinct") - { - resolveAggregateOrWindowFunctionNode(*function_node, count_distinct_implementation_function_name); - function_name = function_node->getFunctionName(); - function_name_lowercase = Poco::toLower(function_name); - } - - /// Replace countIfDistinct with countDistinctIf implementation - if (function_name_lowercase == "countifdistinct") - { - resolveAggregateOrWindowFunctionNode(*function_node, count_distinct_implementation_function_name + "If"); - function_name = function_node->getFunctionName(); - function_name_lowercase = Poco::toLower(function_name); - } - /// Replace aggregateFunctionIfDistinct into aggregateFunctionDistinctIf to make execution more optimal if (function_name_lowercase.ends_with("ifdistinct")) { @@ -64,19 +48,6 @@ public: function_name_lowercase = Poco::toLower(function_name); } - /// Rewrite all aggregate functions to add -OrNull suffix to them - if (settings.aggregate_functions_null_for_empty && !function_name.ends_with("OrNull")) - { - auto function_properies = AggregateFunctionFactory::instance().tryGetProperties(function_name); - if (function_properies && !function_properies->returns_default_when_only_null) - { - auto updated_function_name = function_name + "OrNull"; - resolveAggregateOrWindowFunctionNode(*function_node, updated_function_name); - function_name = function_node->getFunctionName(); - function_name_lowercase = Poco::toLower(function_name); - } - } - /** Move -OrNull suffix ahead, this should execute after add -OrNull suffix. * Used to rewrite aggregate functions with -OrNull suffix in some cases. * Example: sumIfOrNull. diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index f36a751f1dc..9f7582a5cbe 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -4450,13 +4450,27 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi "Aggregate function '{}' does not support lambda arguments", function_name); + auto function_name_lowercase = Poco::toLower(function_name); + + if (function_name_lowercase == "countdistinct") + { + function_name = scope.context->getSettingsRef().count_distinct_implementation; + } + else if (function_name_lowercase == "countdistinctif" || function_name_lowercase == "countifdistinct") + { + function_name = scope.context->getSettingsRef().count_distinct_implementation; + function_name += "If"; + } + bool need_add_or_null = settings.aggregate_functions_null_for_empty && !function_name.ends_with("OrNull"); + if (need_add_or_null) + function_name += "OrNull"; AggregateFunctionProperties properties; - auto aggregate_function = need_add_or_null - ? AggregateFunctionFactory::instance().get(function_name + "OrNull", argument_types, parameters, properties) - : AggregateFunctionFactory::instance().get(function_name, argument_types, parameters, properties); + auto aggregate_function = AggregateFunctionFactory::instance().get(function_name, argument_types, parameters, properties); + function_node.resolveAsAggregateFunction(std::move(aggregate_function)); + return result_projection_names; } diff --git a/src/Analyzer/Passes/QueryAnalysisPass.h b/src/Analyzer/Passes/QueryAnalysisPass.h index 677a13044f2..8b3a8a116e5 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.h +++ b/src/Analyzer/Passes/QueryAnalysisPass.h @@ -51,6 +51,8 @@ namespace DB * Function `untuple` is handled properly. * Function `arrayJoin` is handled properly. * For functions `dictGet` and its variations and for function `joinGet` identifier as first argument is handled properly. + * Replace `countDistinct` and `countIfDistinct` aggregate functions using setting count_distinct_implementation. + * Add -OrNull suffix to aggregate functions if setting aggregate_functions_null_for_empty is true. * Function `exists` is converted into `in`. * * For function `grouping` arguments are resolved, but it is planner responsibility to initialize it with concrete grouping function From 86ee97aed5819bb7aa98d4b0b76448bdedb98641 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Mon, 23 Jan 2023 11:29:54 +0100 Subject: [PATCH 276/717] AggregateFunctionsArithmeticOperationsPass disable optimization if both arguments are non constant --- .../Passes/AggregateFunctionsArithmericOperationsPass.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Analyzer/Passes/AggregateFunctionsArithmericOperationsPass.cpp b/src/Analyzer/Passes/AggregateFunctionsArithmericOperationsPass.cpp index 7e226c20b56..df96d83316c 100644 --- a/src/Analyzer/Passes/AggregateFunctionsArithmericOperationsPass.cpp +++ b/src/Analyzer/Passes/AggregateFunctionsArithmericOperationsPass.cpp @@ -99,6 +99,9 @@ public: const auto * left_argument_constant_node = arithmetic_function_arguments_nodes[0]->as(); const auto * right_argument_constant_node = arithmetic_function_arguments_nodes[1]->as(); + if (!left_argument_constant_node && !right_argument_constant_node) + return; + /** If we extract negative constant, aggregate function name must be updated. * * Example: SELECT min(-1 * id); From b9cb067153f6f7972447a6b1f7a117c37529468d Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Mon, 23 Jan 2023 18:49:33 +0100 Subject: [PATCH 277/717] Analyzer table function arguments resolution fix --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 33 ++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 9f7582a5cbe..6ec2b5c1b97 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -5570,7 +5570,38 @@ void QueryAnalyzer::resolveQueryJoinTreeNode(QueryTreeNodePtr & join_tree_node, } } - resolveExpressionNodeList(table_function_node.getArgumentsNode(), scope, false /*allow_lambda_expression*/, true /*allow_table_expression*/); + QueryTreeNodes result_nodes; + + for (auto & node : table_function_node.getArguments()) + { + if (auto * identifier_node = node->as()) + { + const auto & unresolved_identifier = identifier_node->getIdentifier(); + auto identifier_resolve_result = tryResolveIdentifier({unresolved_identifier, IdentifierLookupContext::EXPRESSION}, scope); + auto resolved_identifier = std::move(identifier_resolve_result.resolved_identifier); + + if (resolved_identifier && resolved_identifier->getNodeType() == QueryTreeNodeType::CONSTANT) + result_nodes.push_back(resolved_identifier); + else + result_nodes.push_back(node); + } + else + { + resolveExpressionNode(node, scope, false /*allow_lambda_expression*/, true /*allow_table_expression*/); + + if (auto * expression_list = node->as()) + { + for (auto & expression_list_node : expression_list->getNodes()) + result_nodes.push_back(expression_list_node); + } + else + { + result_nodes.push_back(node); + } + } + } + + table_function_node.getArguments().getNodes() = std::move(result_nodes); auto table_function_ast = table_function_node.toAST(); table_function_ptr->parseArguments(table_function_ast, scope_context); From 480619af63b39c5d69db9fcac9cc4408e73f8677 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 24 Jan 2023 10:02:57 +0100 Subject: [PATCH 278/717] Analyzer ARRAY JOIN disable alias validation for subquery or table function --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 6ec2b5c1b97..ac3257237b4 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -5619,7 +5619,6 @@ void QueryAnalyzer::resolveQueryJoinTreeNode(QueryTreeNodePtr & join_tree_node, { auto & array_join_node = join_tree_node->as(); resolveQueryJoinTreeNode(array_join_node.getTableExpression(), scope, expressions_visitor); - validateJoinTableExpressionWithoutAlias(join_tree_node, array_join_node.getTableExpression(), scope); std::unordered_set array_join_column_names; From 75876200af2aaeb7e371a73ba601931e948573aa Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 24 Jan 2023 10:04:44 +0100 Subject: [PATCH 279/717] Planner disable names projection on shards --- src/Planner/Planner.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp index f9d54f9db65..2a1bd93391f 100644 --- a/src/Planner/Planner.cpp +++ b/src/Planner/Planner.cpp @@ -1381,8 +1381,12 @@ void Planner::buildPlanForQueryNode() else if (!limit_applied && apply_offset && query_node.hasOffset()) addOffsetStep(query_plan, query_analysis_result); - const auto & projection_analysis_result = expression_analysis_result.getProjection(); - addExpressionStep(query_plan, projection_analysis_result.project_names_actions, "Project names", result_actions_to_execute); + /// Project names is not done on shards, because initiator will not find columns in blocks + if (!query_processing_info.isToAggregationState()) + { + const auto & projection_analysis_result = expression_analysis_result.getProjection(); + addExpressionStep(query_plan, projection_analysis_result.project_names_actions, "Project names", result_actions_to_execute); + } } addBuildSubqueriesForSetsStepIfNeeded(query_plan, select_query_options, planner_context, result_actions_to_execute); From d703c65dbdd9e1c2481d1984cb57a0520be669cb Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 24 Jan 2023 11:53:47 +0100 Subject: [PATCH 280/717] Analyzer lambda resolve unknown identifier crash fix --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 44 ++++++++++++----------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index ac3257237b4..a9157189b1c 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -4768,61 +4768,61 @@ ProjectionNames QueryAnalyzer::resolveExpressionNode(QueryTreeNodePtr & node, Id auto & identifier_node = node->as(); auto unresolved_identifier = identifier_node.getIdentifier(); auto resolve_identifier_expression_result = tryResolveIdentifier({unresolved_identifier, IdentifierLookupContext::EXPRESSION}, scope); - node = resolve_identifier_expression_result.resolved_identifier; + auto resolved_identifier_node = resolve_identifier_expression_result.resolved_identifier; - if (node && result_projection_names.empty() && + if (resolved_identifier_node && result_projection_names.empty() && (resolve_identifier_expression_result.isResolvedFromJoinTree() || resolve_identifier_expression_result.isResolvedFromExpressionArguments())) { - auto projection_name_it = node_to_projection_name.find(node); + auto projection_name_it = node_to_projection_name.find(resolved_identifier_node); if (projection_name_it != node_to_projection_name.end()) result_projection_names.push_back(projection_name_it->second); } - if (node && !node_alias.empty()) + if (resolved_identifier_node && !node_alias.empty()) scope.alias_name_to_lambda_node.erase(node_alias); - if (!node && allow_lambda_expression) + if (!resolved_identifier_node && allow_lambda_expression) { - node = tryResolveIdentifier({unresolved_identifier, IdentifierLookupContext::FUNCTION}, scope).resolved_identifier; + resolved_identifier_node = tryResolveIdentifier({unresolved_identifier, IdentifierLookupContext::FUNCTION}, scope).resolved_identifier; - if (node && !node_alias.empty()) + if (resolved_identifier_node && !node_alias.empty()) scope.alias_name_to_expression_node.erase(node_alias); } - if (!node && allow_table_expression) + if (!resolved_identifier_node && allow_table_expression) { - node = tryResolveIdentifier({unresolved_identifier, IdentifierLookupContext::TABLE_EXPRESSION}, scope).resolved_identifier; + resolved_identifier_node = tryResolveIdentifier({unresolved_identifier, IdentifierLookupContext::TABLE_EXPRESSION}, scope).resolved_identifier; - if (node) + if (resolved_identifier_node) { /// If table identifier is resolved as CTE clone it and resolve - auto * subquery_node = node->as(); - auto * union_node = node->as(); + auto * subquery_node = resolved_identifier_node->as(); + auto * union_node = resolved_identifier_node->as(); bool resolved_as_cte = (subquery_node && subquery_node->isCTE()) || (union_node && union_node->isCTE()); if (resolved_as_cte) { - node = node->clone(); - subquery_node = node->as(); - union_node = node->as(); + resolved_identifier_node = resolved_identifier_node->clone(); + subquery_node = resolved_identifier_node->as(); + union_node = resolved_identifier_node->as(); if (subquery_node) subquery_node->setIsCTE(false); else union_node->setIsCTE(false); - IdentifierResolveScope subquery_scope(node, &scope /*parent_scope*/); + IdentifierResolveScope subquery_scope(resolved_identifier_node, &scope /*parent_scope*/); subquery_scope.subquery_depth = scope.subquery_depth + 1; if (subquery_node) - resolveQuery(node, subquery_scope); + resolveQuery(resolved_identifier_node, subquery_scope); else - resolveUnion(node, subquery_scope); + resolveUnion(resolved_identifier_node, subquery_scope); } } } - if (!node) + if (!resolved_identifier_node) { std::string message_clarification; if (allow_lambda_expression) @@ -4849,10 +4849,10 @@ ProjectionNames QueryAnalyzer::resolveExpressionNode(QueryTreeNodePtr & node, Id getHintsErrorMessageSuffix(hints)); } - if (node->getNodeType() == QueryTreeNodeType::LIST) + if (resolved_identifier_node->getNodeType() == QueryTreeNodeType::LIST) { result_projection_names.clear(); - resolved_expression_it = resolved_expressions.find(node); + resolved_expression_it = resolved_expressions.find(resolved_identifier_node); if (resolved_expression_it != resolved_expressions.end()) return resolved_expression_it->second; else @@ -4862,6 +4862,8 @@ ProjectionNames QueryAnalyzer::resolveExpressionNode(QueryTreeNodePtr & node, Id scope.scope_node->formatASTForErrorMessage()); } + node = std::move(resolved_identifier_node); + if (result_projection_names.empty()) result_projection_names.push_back(unresolved_identifier.getFullName()); From fb4358833343b31570a895fd9847920d3964f49b Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 24 Jan 2023 12:08:08 +0100 Subject: [PATCH 281/717] Fix test 02483_elapsed_time --- tests/queries/0_stateless/02483_elapsed_time.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02483_elapsed_time.sh b/tests/queries/0_stateless/02483_elapsed_time.sh index 724bd1d297b..f743120175b 100755 --- a/tests/queries/0_stateless/02483_elapsed_time.sh +++ b/tests/queries/0_stateless/02483_elapsed_time.sh @@ -20,12 +20,12 @@ EXCEPTION_BEFORE_START_QUERY="WITH SETTINGS enable_global_with_statement = 0" -# For this query the system.query_log needs to show ExceptionBeforeStart and elapsed seconds >= 1.0 +# For this query the system.query_log needs to show ExceptionBeforeStart and elapsed seconds <= 1.0 QUERY_ID="${CLICKHOUSE_DATABASE}_$(date +%s)_02883_q1" ${CLICKHOUSE_CLIENT} -m --query "$EXCEPTION_BEFORE_START_QUERY" --query_id="$QUERY_ID" >/dev/null 2>&1 ${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS" -${CLICKHOUSE_CLIENT} --query "SELECT type == 'ExceptionBeforeStart' as expected_type, query_duration_ms >= 1000 as elapsed_more_than_one_second FROM system.query_log WHERE query_id='$QUERY_ID'" +${CLICKHOUSE_CLIENT} --query "SELECT type == 'ExceptionBeforeStart' as expected_type, query_duration_ms <= 1000 as elapsed_more_than_one_second FROM system.query_log WHERE query_id='$QUERY_ID'" # Now we test with a query that will take 1+ seconds. The CLI should show that as part of the output format OK_QUERY_JSON=" From 5b7fb9c4583f485cc44a95985b055cf559f3d68a Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 24 Jan 2023 16:20:47 +0100 Subject: [PATCH 282/717] Analyzer support enable_global_with_statement setting --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index a9157189b1c..0c9b1f6ffac 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -2920,6 +2920,9 @@ IdentifierResolveResult QueryAnalyzer::tryResolveIdentifierInParentScopes(const } } + if (!scope.context->getSettingsRef().enable_global_with_statement) + return {}; + /** Nested subqueries cannot access outer subqueries table expressions from JOIN tree because * that can prevent resolution of table expression from CTE. * From c469e10092b7b073cf8758e7545238fbfcab76a0 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 24 Jan 2023 16:32:45 +0100 Subject: [PATCH 283/717] Dictionaries DictionaryStorageFetchRequest fix --- src/Dictionaries/DictionaryHelpers.h | 106 +++++++++++++++------------ 1 file changed, 59 insertions(+), 47 deletions(-) diff --git a/src/Dictionaries/DictionaryHelpers.h b/src/Dictionaries/DictionaryHelpers.h index 6e206cafb4a..4fc080f2960 100644 --- a/src/Dictionaries/DictionaryHelpers.h +++ b/src/Dictionaries/DictionaryHelpers.h @@ -71,47 +71,53 @@ private: class DictionaryStorageFetchRequest { public: - DictionaryStorageFetchRequest( - const DictionaryStructure & structure, - const Strings & attributes_names_to_fetch, - DataTypes attributes_to_fetch_result_types, - Columns attributes_default_values_columns) - : attributes_to_fetch_names_set(attributes_names_to_fetch.begin(), attributes_names_to_fetch.end()) - , attributes_to_fetch_filter(structure.attributes.size(), false) + DictionaryStorageFetchRequest(const DictionaryStructure & structure, + const Strings & attributes_to_fetch_names, + const DataTypes & attributes_to_fetch_types, + const Columns & attributes_to_fetch_default_values_columns) + : attributes_to_fetch_filter(structure.attributes.size(), false) { - assert(attributes_default_values_columns.size() == attributes_names_to_fetch.size()); + size_t attributes_to_fetch_size = attributes_to_fetch_names.size(); - if (attributes_to_fetch_names_set.size() != attributes_names_to_fetch.size()) + assert(attributes_to_fetch_size == attributes_to_fetch_types.size()); + assert(attributes_to_fetch_size == attributes_to_fetch_default_values_columns.size()); + + for (size_t i = 0; i < attributes_to_fetch_size; ++i) + attributes_to_fetch_name_to_index.emplace(attributes_to_fetch_names[i], i); + + if (attributes_to_fetch_name_to_index.size() != attributes_to_fetch_name_to_index.size()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Attribute names to fetch should be unique"); size_t attributes_size = structure.attributes.size(); - dictionary_attributes_types.reserve(attributes_size); - attributes_default_value_providers.reserve(attributes_to_fetch_names_set.size()); + dictionary_attributes_names_and_types.reserve(attributes_size); + attributes_default_value_providers.reserve(attributes_size); - size_t attributes_to_fetch_index = 0; - for (size_t i = 0; i < attributes_size; ++i) + for (size_t attribute_index = 0; attribute_index < attributes_size; ++attribute_index) { - const auto & dictionary_attribute = structure.attributes[i]; - const auto & name = dictionary_attribute.name; - const auto & type = dictionary_attribute.type; - dictionary_attributes_types.emplace_back(type); + const auto & dictionary_attribute = structure.attributes[attribute_index]; + dictionary_attributes_names_and_types.emplace_back(dictionary_attribute.name, dictionary_attribute.type); - if (attributes_to_fetch_names_set.find(name) != attributes_to_fetch_names_set.end()) + auto attribute_to_fetch_index_it = attributes_to_fetch_name_to_index.find(dictionary_attribute.name); + if (attribute_to_fetch_index_it == attributes_to_fetch_name_to_index.end()) { - attributes_to_fetch_filter[i] = true; - auto & attribute_to_fetch_result_type = attributes_to_fetch_result_types[attributes_to_fetch_index]; - - if (!attribute_to_fetch_result_type->equals(*type)) - throw Exception(ErrorCodes::TYPE_MISMATCH, - "Attribute type does not match, expected ({}), found ({})", - attribute_to_fetch_result_type->getName(), - type->getName()); - - attributes_default_value_providers.emplace_back(dictionary_attribute.null_value, attributes_default_values_columns[attributes_to_fetch_index]); - ++attributes_to_fetch_index; - } - else attributes_default_value_providers.emplace_back(dictionary_attribute.null_value); + continue; + } + + attributes_to_fetch_filter[attribute_index] = true; + + size_t attributes_to_fetch_index = attribute_to_fetch_index_it->second; + const auto & attribute_to_fetch_result_type = attributes_to_fetch_types[attributes_to_fetch_index]; + const auto & attribute_to_fetch_default_value_column = attributes_to_fetch_default_values_columns[attributes_to_fetch_index]; + + if (!attribute_to_fetch_result_type->equals(*dictionary_attribute.type)) + throw Exception(ErrorCodes::TYPE_MISMATCH, + "Attribute {} type does not match, expected {}, found {}", + dictionary_attribute.name, + attribute_to_fetch_result_type->getName(), + dictionary_attribute.type->getName()); + + attributes_default_value_providers.emplace_back(dictionary_attribute.null_value, attribute_to_fetch_default_value_column); } } @@ -120,13 +126,13 @@ public: /// Check requested attributes size ALWAYS_INLINE size_t attributesSize() const { - return dictionary_attributes_types.size(); + return dictionary_attributes_names_and_types.size(); } /// Check if attribute with attribute_name was requested to fetch ALWAYS_INLINE bool containsAttribute(const String & attribute_name) const { - return attributes_to_fetch_names_set.find(attribute_name) != attributes_to_fetch_names_set.end(); + return attributes_to_fetch_name_to_index.contains(attribute_name); } /// Check if attribute with attribute_index should be filled during fetch @@ -137,7 +143,7 @@ public: const DataTypePtr & dataTypeAtIndex(size_t attribute_index) const { - return dictionary_attributes_types[attribute_index]; + return dictionary_attributes_names_and_types[attribute_index].type; } const DefaultValueProvider & defaultValueProviderAtIndex(size_t attribute_index) const @@ -149,10 +155,10 @@ public: MutableColumns makeAttributesResultColumns() const { MutableColumns result; - result.reserve(dictionary_attributes_types.size()); + result.reserve(dictionary_attributes_names_and_types.size()); - for (const auto & type : dictionary_attributes_types) - result.emplace_back(type->createColumn()); + for (const auto & name_and_type : dictionary_attributes_names_and_types) + result.emplace_back(name_and_type.type->createColumn()); return result; } @@ -160,10 +166,10 @@ public: Columns makeAttributesResultColumnsNonMutable() const { Columns result; - result.reserve(dictionary_attributes_types.size()); + result.reserve(dictionary_attributes_names_and_types.size()); - for (const auto & type : dictionary_attributes_types) - result.emplace_back(type->createColumn()); + for (const auto & name_and_type : dictionary_attributes_names_and_types) + result.emplace_back(name_and_type.type->createColumn()); return result; } @@ -171,20 +177,26 @@ public: /// Filter only requested columns Columns filterRequestedColumns(MutableColumns & fetched_mutable_columns) const { - Columns result; - result.reserve(dictionary_attributes_types.size()); + Columns result(attributes_to_fetch_name_to_index.size()); + size_t dictionary_attributes_size = dictionary_attributes_names_and_types.size(); - for (size_t fetch_request_index = 0; fetch_request_index < dictionary_attributes_types.size(); ++fetch_request_index) - if (shouldFillResultColumnWithIndex(fetch_request_index)) - result.emplace_back(std::move(fetched_mutable_columns[fetch_request_index])); + for (size_t attribute_index = 0; attribute_index < dictionary_attributes_size; ++attribute_index) + { + if (!shouldFillResultColumnWithIndex(attribute_index)) + continue; + + const auto & dictionary_attribute_name = dictionary_attributes_names_and_types[attribute_index].name; + size_t fetch_attribute_index = attributes_to_fetch_name_to_index.find(dictionary_attribute_name)->second; + result[fetch_attribute_index] = std::move(fetched_mutable_columns[attribute_index]); + } return result; } private: - std::unordered_set attributes_to_fetch_names_set; + NamesAndTypes dictionary_attributes_names_and_types; + std::unordered_map attributes_to_fetch_name_to_index; std::vector attributes_to_fetch_filter; std::vector attributes_default_value_providers; - DataTypes dictionary_attributes_types; }; static inline void insertDefaultValuesIntoColumns( /// NOLINT From d3b604de5a04b069feb1fcacf060d4605ef63564 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 24 Jan 2023 17:34:49 +0100 Subject: [PATCH 284/717] FunctionToSubcolumnsPass mapContains map argument result type fix --- src/Analyzer/Passes/FunctionToSubcolumnsPass.cpp | 3 ++- src/Functions/array/arrayIndex.h | 10 ++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/Analyzer/Passes/FunctionToSubcolumnsPass.cpp b/src/Analyzer/Passes/FunctionToSubcolumnsPass.cpp index 6f95607d0c6..696483862e0 100644 --- a/src/Analyzer/Passes/FunctionToSubcolumnsPass.cpp +++ b/src/Analyzer/Passes/FunctionToSubcolumnsPass.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -181,7 +182,7 @@ public: /// Replace `mapContains(map_argument, argument)` with `has(map_argument.keys, argument)` column.name += ".keys"; - column.type = data_type_map.getKeyType(); + column.type = std::make_shared(data_type_map.getKeyType()); auto has_function_argument = std::make_shared(column, column_source); function_arguments_nodes[0] = std::move(has_function_argument); diff --git a/src/Functions/array/arrayIndex.h b/src/Functions/array/arrayIndex.h index 296cc588aa2..7ecba12cf0d 100644 --- a/src/Functions/array/arrayIndex.h +++ b/src/Functions/array/arrayIndex.h @@ -390,8 +390,9 @@ public: { if (!array_type && !map_type) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "First argument for function {} must be an array or map.", - getName()); + "First argument for function {} must be an array or map. Actual {}", + getName(), + first_argument_type->getName()); inner_type = map_type ? map_type->getKeyType() : array_type->getNestedType(); } @@ -399,8 +400,9 @@ public: { if (!array_type) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "First argument for function {} must be an array.", - getName()); + "First argument for function {} must be an array. Actual {}", + getName(), + first_argument_type->getName()); inner_type = array_type->getNestedType(); } From 25da9dcef713f9a7860eb00c4aa23ea9a3263b9c Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 24 Jan 2023 19:15:41 +0100 Subject: [PATCH 285/717] StorageDistributed Planner initialization fix --- src/Planner/Planner.cpp | 11 +++++++++++ src/Planner/Planner.h | 6 ++++++ src/Storages/StorageDistributed.cpp | 12 ++++++++++-- 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp index 2a1bd93391f..6bf51ee9cf3 100644 --- a/src/Planner/Planner.cpp +++ b/src/Planner/Planner.cpp @@ -959,6 +959,17 @@ Planner::Planner(const QueryTreeNodePtr & query_tree_, { } +Planner::Planner(const QueryTreeNodePtr & query_tree_, + const SelectQueryOptions & select_query_options_, + PlannerContextPtr planner_context_, + PlannerConfiguration planner_configuration_) + : query_tree(query_tree_) + , select_query_options(select_query_options_) + , planner_context(std::move(planner_context_)) + , planner_configuration(std::move(planner_configuration_)) +{ +} + void Planner::buildQueryPlanIfNeeded() { if (query_plan.isInitialized()) diff --git a/src/Planner/Planner.h b/src/Planner/Planner.h index 6e225bbf905..a87880eab6b 100644 --- a/src/Planner/Planner.h +++ b/src/Planner/Planner.h @@ -35,6 +35,12 @@ public: GlobalPlannerContextPtr global_planner_context_, PlannerConfiguration planner_configuration_ = {}); + /// Initialize planner with query tree after query analysis phase and planner context + Planner(const QueryTreeNodePtr & query_tree_, + const SelectQueryOptions & select_query_options_, + PlannerContextPtr planner_context_, + PlannerConfiguration planner_configuration_ = {}); + const QueryPlan & getQueryPlan() const { return query_plan; diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 954dddd07f8..e7468ae9d1e 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -663,8 +663,12 @@ void StorageDistributed::read( StorageID remote_storage_id{remote_database, remote_table}; auto query_tree_with_replaced_distributed_table = buildQueryTreeDistributedTableReplacedWithLocalTable(query_info, remote_storage_id); query_ast = queryNodeToSelectQuery(query_tree_with_replaced_distributed_table); - Planner planner(query_tree_with_replaced_distributed_table, SelectQueryOptions(processed_stage), PlannerConfiguration{.only_analyze = true}); + + Planner planner(query_tree_with_replaced_distributed_table, + SelectQueryOptions(processed_stage), + PlannerConfiguration{.only_analyze = true}); planner.buildQueryPlanIfNeeded(); + header = planner.getQueryPlan().getCurrentDataStream().header; } else @@ -716,8 +720,12 @@ void StorageDistributed::read( if (local_context->getSettingsRef().allow_experimental_analyzer) { - Planner planner(query_info.query_tree, SelectQueryOptions(processed_stage), PlannerConfiguration{.only_analyze = true}); + Planner planner(query_info.query_tree, + SelectQueryOptions(processed_stage), + query_info.planner_context, + PlannerConfiguration{.only_analyze = true}); planner.buildQueryPlanIfNeeded(); + auto expected_header = planner.getQueryPlan().getCurrentDataStream().header; auto rename_actions_dag = ActionsDAG::makeConvertingActions( From 484fc40e642a17f845825b49cc25d83200c7b8fa Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 25 Jan 2023 11:36:21 +0100 Subject: [PATCH 286/717] Analyzer identifier resolution for subcolumns fix --- src/Analyzer/ColumnNode.cpp | 4 +++- src/Analyzer/Passes/QueryAnalysisPass.cpp | 10 +++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/Analyzer/ColumnNode.cpp b/src/Analyzer/ColumnNode.cpp index 52c264ba1c7..1b367829510 100644 --- a/src/Analyzer/ColumnNode.cpp +++ b/src/Analyzer/ColumnNode.cpp @@ -99,7 +99,9 @@ QueryTreeNodePtr ColumnNode::cloneImpl() const ASTPtr ColumnNode::toASTImpl() const { - return std::make_shared(column.name); + auto column_name_identifier = Identifier(column.name); + auto column_name_identifier_parts = column_name_identifier.getParts(); + return std::make_shared(std::move(column_name_identifier_parts)); } } diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 0c9b1f6ffac..7fde467921a 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -408,14 +408,14 @@ struct TableExpressionData ColumnNameToColumnNodeMap column_name_to_column_node; std::unordered_set> column_identifier_first_parts; - bool hasFullIdentifierName(IdentifierView identifier) const + bool hasFullIdentifierName(IdentifierView identifier_view) const { - return column_name_to_column_node.contains(std::string_view(identifier.getFullName())); + return column_name_to_column_node.contains(identifier_view.getFullName()); } - bool canBindIdentifier(IdentifierView identifier) const + bool canBindIdentifier(IdentifierView identifier_view) const { - return column_identifier_first_parts.contains(std::string_view(identifier.at(0))); + return column_identifier_first_parts.contains(identifier_view.at(0)); } [[maybe_unused]] void dump(WriteBuffer & buffer) const @@ -2130,7 +2130,7 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromCompoundExpression(const compound_expression_from_error_message += compound_expression_source; } - throw Exception(ErrorCodes::BAD_ARGUMENTS, + throw Exception(ErrorCodes::UNKNOWN_IDENTIFIER, "Identifier {} nested path {} cannot be resolved from type {}{}. In scope {}{}", expression_identifier, nested_path, From 16d9f3766b59026d9bf4a851209936db57fdd821 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 25 Jan 2023 11:46:22 +0100 Subject: [PATCH 287/717] Fix test 00490_with_select --- tests/queries/0_stateless/00490_with_select.sql | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/queries/0_stateless/00490_with_select.sql b/tests/queries/0_stateless/00490_with_select.sql index c2190187f61..7fcb89b5b22 100644 --- a/tests/queries/0_stateless/00490_with_select.sql +++ b/tests/queries/0_stateless/00490_with_select.sql @@ -1,8 +1,6 @@ with pow(2,2) as four select pow(four, 2), 2 as two, pow(two, 2); -select `pow(four, 2)`, `pow(2, 2)` from (with pow(2,2) as four select pow(four, 2), 2 as two, pow(two, 2)); -with (select pow(2,2)) as four select pow(four, 2), 2 as two, pow(two, 2); -select `pow(four, 2)`, `pow(2, 2)` from (with (select pow(2,2)) as four select pow(four, 2), 2 as two, pow(two, 2)); +select `pow(four, 2)`, `pow(two, 2)` from (with pow(2,2) as four select pow(four, 2), 2 as two, pow(two, 2)); +with (select pow(two,2)) as four select pow(four, 2), 2 as two, pow(two, 2); +select `pow(four, 2)`, `pow(two, 2)` from (with (select pow(2,2)) as four select pow(four, 2), 2 as two, pow(two, 2)); with 'string' as str select str || '_abc'; select `concat(str, \'_abc\')` from (with 'string' as str select str || '_abc'); - - From 77bbe2558b4ccc173070151e8478457ba2cd2561 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 25 Jan 2023 11:51:28 +0100 Subject: [PATCH 288/717] Fix test 01934_constexpr_aggregate_function_parameters --- .../01934_constexpr_aggregate_function_parameters.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01934_constexpr_aggregate_function_parameters.sql b/tests/queries/0_stateless/01934_constexpr_aggregate_function_parameters.sql index defc3d7b686..ac7fdafda82 100644 --- a/tests/queries/0_stateless/01934_constexpr_aggregate_function_parameters.sql +++ b/tests/queries/0_stateless/01934_constexpr_aggregate_function_parameters.sql @@ -1,11 +1,10 @@ SELECT groupArray(2 + 3)(number) FROM numbers(10); SELECT groupArray('5'::UInt8)(number) FROM numbers(10); -SELECT groupArray()(number) FROM numbers(10); -- { serverError 36 } SELECT groupArray(NULL)(number) FROM numbers(10); -- { serverError 36 } SELECT groupArray(NULL + NULL)(number) FROM numbers(10); -- { serverError 36 } SELECT groupArray([])(number) FROM numbers(10); -- { serverError 36 } SELECT groupArray(throwIf(1))(number) FROM numbers(10); -- { serverError 134 } -- Not the best error message, can be improved. -SELECT groupArray(number)(number) FROM numbers(10); -- { serverError 47 } +SELECT groupArray(number)(number) FROM numbers(10); -- { serverError 36 } From 837413312305c2be0902ea5d7e2b5db97dfc9d22 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 25 Jan 2023 16:20:40 +0100 Subject: [PATCH 289/717] Analyzer evaluate scalar subquery fix --- src/Analyzer/HashUtils.h | 6 +- src/Analyzer/Passes/QueryAnalysisPass.cpp | 268 ++++++++++++++-------- 2 files changed, 172 insertions(+), 102 deletions(-) diff --git a/src/Analyzer/HashUtils.h b/src/Analyzer/HashUtils.h index 46222755938..2203e7d5203 100644 --- a/src/Analyzer/HashUtils.h +++ b/src/Analyzer/HashUtils.h @@ -16,11 +16,11 @@ struct QueryTreeNodeWithHash { QueryTreeNodeWithHash(QueryTreeNodePtrType node_) /// NOLINT : node(std::move(node_)) - , hash(node->getTreeHash().first) + , hash(node->getTreeHash()) {} QueryTreeNodePtrType node = nullptr; - size_t hash = 0; + std::pair hash; }; template @@ -55,6 +55,6 @@ struct std::hash> { size_t operator()(const DB::QueryTreeNodeWithHash & node_with_hash) const { - return node_with_hash.hash; + return node_with_hash.hash.first; } }; diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 7fde467921a..c541cd89fcd 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -1109,7 +1110,7 @@ private: QueryTreeNodePtr tryGetLambdaFromSQLUserDefinedFunctions(const std::string & function_name, ContextPtr context); - void evaluateScalarSubqueryIfNeeded(QueryTreeNodePtr & query_tree_node, size_t subquery_depth, ContextPtr context); + void evaluateScalarSubqueryIfNeeded(QueryTreeNodePtr & query_tree_node, IdentifierResolveScope & scope); static void mergeWindowWithParentWindow(const QueryTreeNodePtr & window_node, const QueryTreeNodePtr & parent_window_node, IdentifierResolveScope & scope); @@ -1222,8 +1223,8 @@ private: /// Global resolve expression node to projection names map std::unordered_map resolved_expressions; - /// Results of scalar sub queries - std::unordered_map> scalars; + /// Global scalar subquery to scalar value map + std::unordered_map scalar_subquery_to_scalar_value; }; @@ -1731,7 +1732,7 @@ QueryTreeNodePtr QueryAnalyzer::tryGetLambdaFromSQLUserDefinedFunctions(const st } /// Evaluate scalar subquery and perform constant folding if scalar subquery does not have constant value -void QueryAnalyzer::evaluateScalarSubqueryIfNeeded(QueryTreeNodePtr & node, size_t subquery_depth, ContextPtr context) +void QueryAnalyzer::evaluateScalarSubqueryIfNeeded(QueryTreeNodePtr & node, IdentifierResolveScope & scope) { auto * query_node = node->as(); auto * union_node = node->as(); @@ -1741,102 +1742,171 @@ void QueryAnalyzer::evaluateScalarSubqueryIfNeeded(QueryTreeNodePtr & node, size node->getNodeTypeName(), node->formatASTForErrorMessage()); - auto scalars_iterator = scalars.find(node.get()); - if (scalars_iterator != scalars.end()) + auto & context = scope.context; + + Block scalar_block; + + QueryTreeNodePtrWithHash node_with_hash(node); + auto scalar_value_it = scalar_subquery_to_scalar_value.find(node_with_hash); + + if (scalar_value_it != scalar_subquery_to_scalar_value.end()) { ProfileEvents::increment(ProfileEvents::ScalarSubqueriesGlobalCacheHit); - node = std::make_shared(scalars_iterator->second, node); - return; - } - - ProfileEvents::increment(ProfileEvents::ScalarSubqueriesCacheMiss); - - auto subquery_context = Context::createCopy(context); - - Settings subquery_settings = context->getSettings(); - subquery_settings.max_result_rows = 1; - subquery_settings.extremes = false; - subquery_context->setSettings(subquery_settings); - - auto options = SelectQueryOptions(QueryProcessingStage::Complete, subquery_depth, true /*is_subquery*/); - auto interpreter = std::make_unique(node, subquery_context, options); - - auto io = interpreter->execute(); - - PullingAsyncPipelineExecutor executor(io.pipeline); - io.pipeline.setProgressCallback(context->getProgressCallback()); - io.pipeline.setProcessListElement(context->getProcessListElement()); - - Block block; - - while (block.rows() == 0 && executor.pull(block)) - { - } - - if (block.rows() == 0) - { - auto types = interpreter->getSampleBlock().getDataTypes(); - if (types.size() != 1) - types = {std::make_shared(types)}; - - auto & type = types[0]; - if (!type->isNullable()) - { - if (!type->canBeInsideNullable()) - throw Exception(ErrorCodes::INCORRECT_RESULT_OF_SCALAR_SUBQUERY, - "Scalar subquery returned empty result of type {} which cannot be Nullable.", - type->getName()); - - type = makeNullable(type); - } - - auto constant_value = std::make_shared(Null(), std::move(type)); - node = std::make_shared(std::move(constant_value), node); - return; - } - - if (block.rows() != 1) - throw Exception(ErrorCodes::INCORRECT_RESULT_OF_SCALAR_SUBQUERY, "Scalar subquery returned more than one row"); - - Block tmp_block; - while (tmp_block.rows() == 0 && executor.pull(tmp_block)) - { - } - - if (tmp_block.rows() != 0) - throw Exception(ErrorCodes::INCORRECT_RESULT_OF_SCALAR_SUBQUERY, "Scalar subquery returned more than one row"); - - block = materializeBlock(block); - size_t columns = block.columns(); - - Field scalar_value; - DataTypePtr scalar_type; - - if (columns == 1) - { - auto & column = block.getByPosition(0); - /// Here we wrap type to nullable if we can. - /// It is needed cause if subquery return no rows, it's result will be Null. - /// In case of many columns, do not check it cause tuple can't be nullable. - if (!column.type->isNullable() && column.type->canBeInsideNullable()) - { - column.type = makeNullable(column.type); - column.column = makeNullable(column.column); - } - - column.column->get(0, scalar_value); - scalar_type = column.type; + scalar_block = scalar_value_it->second; } else { - auto tuple_column = ColumnTuple::create(block.getColumns()); - tuple_column->get(0, scalar_value); - scalar_type = std::make_shared(block.getDataTypes(), block.getNames()); + ProfileEvents::increment(ProfileEvents::ScalarSubqueriesCacheMiss); + auto subquery_context = Context::createCopy(context); + + Settings subquery_settings = context->getSettings(); + subquery_settings.max_result_rows = 1; + subquery_settings.extremes = false; + subquery_context->setSettings(subquery_settings); + + auto options = SelectQueryOptions(QueryProcessingStage::Complete, scope.subquery_depth, true /*is_subquery*/); + auto interpreter = std::make_unique(node, subquery_context, options); + + auto io = interpreter->execute(); + + PullingAsyncPipelineExecutor executor(io.pipeline); + io.pipeline.setProgressCallback(context->getProgressCallback()); + io.pipeline.setProcessListElement(context->getProcessListElement()); + + Block block; + + while (block.rows() == 0 && executor.pull(block)) + { + } + + if (block.rows() == 0) + { + auto types = interpreter->getSampleBlock().getDataTypes(); + if (types.size() != 1) + types = {std::make_shared(types)}; + + auto & type = types[0]; + if (!type->isNullable()) + { + if (!type->canBeInsideNullable()) + throw Exception(ErrorCodes::INCORRECT_RESULT_OF_SCALAR_SUBQUERY, + "Scalar subquery returned empty result of type {} which cannot be Nullable", + type->getName()); + + type = makeNullable(type); + } + + auto scalar_column = type->createColumn(); + scalar_column->insert(Null()); + scalar_block.insert({std::move(scalar_column), type, "null"}); + } + else + { + if (block.rows() != 1) + throw Exception(ErrorCodes::INCORRECT_RESULT_OF_SCALAR_SUBQUERY, "Scalar subquery returned more than one row"); + + Block tmp_block; + while (tmp_block.rows() == 0 && executor.pull(tmp_block)) + { + } + + if (tmp_block.rows() != 0) + throw Exception(ErrorCodes::INCORRECT_RESULT_OF_SCALAR_SUBQUERY, "Scalar subquery returned more than one row"); + + block = materializeBlock(block); + size_t columns = block.columns(); + + if (columns == 1) + { + auto & column = block.getByPosition(0); + /// Here we wrap type to nullable if we can. + /// It is needed cause if subquery return no rows, it's result will be Null. + /// In case of many columns, do not check it cause tuple can't be nullable. + if (!column.type->isNullable() && column.type->canBeInsideNullable()) + { + column.type = makeNullable(column.type); + column.column = makeNullable(column.column); + } + + scalar_block = block; + } + else + { + scalar_block.insert({ + ColumnTuple::create(block.getColumns()), + std::make_shared(block.getDataTypes(), block.getNames()), + "tuple"}); + } + } + + scalar_subquery_to_scalar_value.emplace(node_with_hash, scalar_block); } - auto constant_value = std::make_shared(std::move(scalar_value), std::move(scalar_type)); - scalars[node.get()] = constant_value; - node = std::make_shared(std::move(constant_value), node); + const auto & scalar_column_with_type = scalar_block.safeGetByPosition(0); + const auto & scalar_type = scalar_column_with_type.type; + + Field scalar_value; + scalar_column_with_type.column->get(0, scalar_value); + + const auto * scalar_type_name = scalar_block.safeGetByPosition(0).type->getFamilyName(); + static const std::set useless_literal_types = {"Array", "Tuple", "AggregateFunction", "Function", "Set", "LowCardinality"}; + auto * nearest_query_scope = scope.getNearestQueryScope(); + + /// Always convert to literals when there is no query context + if (!context->getSettingsRef().enable_scalar_subquery_optimization || + !useless_literal_types.contains(scalar_type_name) || + !context->hasQueryContext() || + !nearest_query_scope) + { + auto constant_value = std::make_shared(std::move(scalar_value), scalar_type); + auto constant_node = std::make_shared(std::move(constant_value), node); + + if (constant_node->getValue().isNull()) + { + std::string cast_type = constant_node->getResultType()->getName(); + std::string cast_function_name = "__CAST"; + + auto cast_type_constant_value = std::make_shared(std::move(cast_type), std::make_shared()); + auto cast_type_constant_node = std::make_shared(std::move(cast_type_constant_value)); + + auto cast_function_node = std::make_shared(cast_function_name); + cast_function_node->getArguments().getNodes().push_back(constant_node); + cast_function_node->getArguments().getNodes().push_back(std::move(cast_type_constant_node)); + + auto cast_function = FunctionFactory::instance().get(cast_function_name, context); + cast_function_node->resolveAsFunction(cast_function->build(cast_function_node->getArgumentColumns())); + + node = std::move(cast_function_node); + } + else + { + node = std::move(constant_node); + } + return; + } + + auto & nearest_query_scope_query_node = nearest_query_scope->scope_node->as(); + auto & mutable_context = nearest_query_scope_query_node.getMutableContext(); + + auto scalar_query_hash_string = std::to_string(node_with_hash.hash.first) + '_' + std::to_string(node_with_hash.hash.second); + + if (mutable_context->hasQueryContext()) + mutable_context->getQueryContext()->addScalar(scalar_query_hash_string, scalar_block); + + mutable_context->addScalar(scalar_query_hash_string, scalar_block); + + std::string get_scalar_function_name = "__getScalar"; + + auto scalar_query_hash_constant_value = std::make_shared(std::move(scalar_query_hash_string), std::make_shared()); + auto scalar_query_hash_constant_node = std::make_shared(std::move(scalar_query_hash_constant_value)); + + auto get_scalar_function_node = std::make_shared(get_scalar_function_name); + get_scalar_function_node->getArguments().getNodes().push_back(std::move(scalar_query_hash_constant_node)); + + auto get_scalar_function = FunctionFactory::instance().get(get_scalar_function_name, mutable_context); + get_scalar_function_node->resolveAsFunction(get_scalar_function->build(get_scalar_function_node->getArgumentColumns())); + + node = std::move(get_scalar_function_node); } void QueryAnalyzer::mergeWindowWithParentWindow(const QueryTreeNodePtr & window_node, const QueryTreeNodePtr & parent_window_node, IdentifierResolveScope & scope) @@ -4708,7 +4778,7 @@ ProjectionNames QueryAnalyzer::resolveExpressionNode(QueryTreeNodePtr & node, Id IdentifierResolveScope subquery_scope(node, &scope /*parent_scope*/); subquery_scope.subquery_depth = scope.subquery_depth + 1; - evaluateScalarSubqueryIfNeeded(node, subquery_scope.subquery_depth, subquery_scope.context); + evaluateScalarSubqueryIfNeeded(node, subquery_scope); } return resolved_expression_it->second; @@ -4852,10 +4922,12 @@ ProjectionNames QueryAnalyzer::resolveExpressionNode(QueryTreeNodePtr & node, Id getHintsErrorMessageSuffix(hints)); } - if (resolved_identifier_node->getNodeType() == QueryTreeNodeType::LIST) + node = std::move(resolved_identifier_node); + + if (node->getNodeType() == QueryTreeNodeType::LIST) { result_projection_names.clear(); - resolved_expression_it = resolved_expressions.find(resolved_identifier_node); + resolved_expression_it = resolved_expressions.find(node); if (resolved_expression_it != resolved_expressions.end()) return resolved_expression_it->second; else @@ -4865,8 +4937,6 @@ ProjectionNames QueryAnalyzer::resolveExpressionNode(QueryTreeNodePtr & node, Id scope.scope_node->formatASTForErrorMessage()); } - node = std::move(resolved_identifier_node); - if (result_projection_names.empty()) result_projection_names.push_back(unresolved_identifier.getFullName()); @@ -4948,7 +5018,7 @@ ProjectionNames QueryAnalyzer::resolveExpressionNode(QueryTreeNodePtr & node, Id resolveUnion(node, subquery_scope); if (!allow_table_expression) - evaluateScalarSubqueryIfNeeded(node, subquery_scope.subquery_depth, subquery_scope.context); + evaluateScalarSubqueryIfNeeded(node, subquery_scope); if (result_projection_names.empty()) result_projection_names.push_back(std::move(projection_name)); From 2830877e22a9dd5a989ad16f8d5e313d8912debe Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Thu, 26 Jan 2023 11:52:40 +0100 Subject: [PATCH 290/717] Analyzer support table functions --- src/Analyzer/ColumnNode.cpp | 33 +- src/Analyzer/Passes/QueryAnalysisPass.cpp | 559 ++++++++++-------- src/Analyzer/QueryTreePassManager.cpp | 8 + src/Analyzer/Utils.cpp | 2 + .../InterpreterSelectQueryAnalyzer.cpp | 31 +- .../InterpreterSelectQueryAnalyzer.h | 8 + src/TableFunctions/ITableFunction.h | 6 + .../TableFunctionExecutable.cpp | 15 + src/TableFunctions/TableFunctionExecutable.h | 6 + src/TableFunctions/TableFunctionExplain.cpp | 1 - src/TableFunctions/TableFunctionView.cpp | 17 +- src/TableFunctions/TableFunctionView.h | 5 + .../TableFunctionViewIfPermitted.cpp | 17 +- .../TableFunctionViewIfPermitted.h | 5 + 14 files changed, 457 insertions(+), 256 deletions(-) diff --git a/src/Analyzer/ColumnNode.cpp b/src/Analyzer/ColumnNode.cpp index 1b367829510..4ec284f6462 100644 --- a/src/Analyzer/ColumnNode.cpp +++ b/src/Analyzer/ColumnNode.cpp @@ -8,6 +8,8 @@ #include +#include + namespace DB { @@ -99,9 +101,36 @@ QueryTreeNodePtr ColumnNode::cloneImpl() const ASTPtr ColumnNode::toASTImpl() const { + std::vector additional_column_qualification_parts; + + auto column_source = getColumnSourceOrNull(); + if (column_source) + { + auto node_type = column_source->getNodeType(); + if (node_type == QueryTreeNodeType::TABLE || + node_type == QueryTreeNodeType::TABLE_FUNCTION || + node_type == QueryTreeNodeType::QUERY || + node_type == QueryTreeNodeType::UNION) + { + if (column_source->hasAlias()) + { + additional_column_qualification_parts = {column_source->getAlias()}; + } + else if (auto * table_node = column_source->as()) + { + const auto & table_storage_id = table_node->getStorageID(); + additional_column_qualification_parts = {table_storage_id.getDatabaseName(), table_storage_id.getTableName()}; + } + } + } + auto column_name_identifier = Identifier(column.name); - auto column_name_identifier_parts = column_name_identifier.getParts(); - return std::make_shared(std::move(column_name_identifier_parts)); + const auto & column_name_identifier_parts = column_name_identifier.getParts(); + additional_column_qualification_parts.insert(additional_column_qualification_parts.end(), + column_name_identifier_parts.begin(), + column_name_identifier_parts.end()); + + return std::make_shared(std::move(additional_column_qualification_parts)); } } diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index c541cd89fcd..7685fed6948 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -1199,6 +1199,12 @@ private: void initializeTableExpressionColumns(const QueryTreeNodePtr & table_expression_node, IdentifierResolveScope & scope); + void resolveTableFunction(QueryTreeNodePtr & table_function_node, IdentifierResolveScope & scope, QueryExpressionsAliasVisitor & expressions_visitor, bool nested_table_function); + + void resolveArrayJoin(QueryTreeNodePtr & array_join_node, IdentifierResolveScope & scope, QueryExpressionsAliasVisitor & expressions_visitor); + + void resolveJoin(QueryTreeNodePtr & join_node, IdentifierResolveScope & scope, QueryExpressionsAliasVisitor & expressions_visitor); + void resolveQueryJoinTreeNode(QueryTreeNodePtr & join_tree_node, IdentifierResolveScope & scope, QueryExpressionsAliasVisitor & expressions_visitor); void resolveQuery(const QueryTreeNodePtr & query_node, IdentifierResolveScope & scope); @@ -5579,27 +5585,300 @@ void QueryAnalyzer::initializeTableExpressionColumns(const QueryTreeNodePtr & ta scope.table_expression_node_to_data.emplace(table_expression_node, std::move(table_expression_data)); } +/// Resolve table function node in scope +void QueryAnalyzer::resolveTableFunction(QueryTreeNodePtr & table_function_node, + IdentifierResolveScope & scope, + QueryExpressionsAliasVisitor & expressions_visitor, + bool nested_table_function) +{ + auto & table_function_node_typed = table_function_node->as(); + + if (!nested_table_function) + expressions_visitor.visit(table_function_node_typed.getArgumentsNode()); + + const auto & table_function_factory = TableFunctionFactory::instance(); + const auto & table_function_name = table_function_node_typed.getTableFunctionName(); + + auto & scope_context = scope.context; + + TableFunctionPtr table_function_ptr = table_function_factory.tryGet(table_function_name, scope_context); + if (!table_function_ptr) + { + auto hints = TableFunctionFactory::instance().getHints(table_function_name); + if (!hints.empty()) + throw Exception(ErrorCodes::UNKNOWN_FUNCTION, + "Unknown table function {}. Maybe you meant: {}", + table_function_name, + DB::toString(hints)); + else + throw Exception(ErrorCodes::UNKNOWN_FUNCTION, + "Unknown table function {}", + table_function_name); + } + + if (!nested_table_function && + scope_context->getSettingsRef().use_structure_from_insertion_table_in_table_functions && + scope_context->hasInsertionTable() && + table_function_ptr->needStructureHint()) + { + const auto & insertion_table = scope_context->getInsertionTable(); + if (!insertion_table.empty()) + { + auto insertion_table_storage = DatabaseCatalog::instance().getTable(insertion_table, scope_context); + const auto & structure_hint = insertion_table_storage->getInMemoryMetadataPtr()->columns; + table_function_ptr->setStructureHint(structure_hint); + } + } + + QueryTreeNodes result_table_function_arguments; + + auto skip_analysis_arguments_indexes = table_function_ptr->skipAnalysisForArguments(table_function_node, scope_context); + + auto & table_function_arguments = table_function_node_typed.getArguments().getNodes(); + size_t table_function_arguments_size = table_function_arguments.size(); + + for (size_t table_function_argument_index = 0; table_function_argument_index < table_function_arguments_size; ++table_function_argument_index) + { + auto & table_function_argument = table_function_arguments[table_function_argument_index]; + + auto skip_argument_index_it = std::find(skip_analysis_arguments_indexes.begin(), + skip_analysis_arguments_indexes.end(), + table_function_argument_index); + if (skip_argument_index_it != skip_analysis_arguments_indexes.end()) + { + result_table_function_arguments.push_back(table_function_argument); + continue; + } + + if (auto * identifier_node = table_function_argument->as()) + { + const auto & unresolved_identifier = identifier_node->getIdentifier(); + auto identifier_resolve_result = tryResolveIdentifier({unresolved_identifier, IdentifierLookupContext::EXPRESSION}, scope); + auto resolved_identifier = std::move(identifier_resolve_result.resolved_identifier); + + if (resolved_identifier && resolved_identifier->getNodeType() == QueryTreeNodeType::CONSTANT) + result_table_function_arguments.push_back(std::move(resolved_identifier)); + else + result_table_function_arguments.push_back(table_function_argument); + + continue; + } + + if (auto * table_function_argument_function = table_function_argument->as()) + { + const auto & table_function_argument_function_name = table_function_argument_function->getFunctionName(); + if (TableFunctionFactory::instance().isTableFunctionName(table_function_argument_function_name)) + { + auto table_function_node_to_resolve_typed = std::make_shared(table_function_argument_function_name); + table_function_node_to_resolve_typed->getArgumentsNode() = table_function_argument_function->getArgumentsNode(); + + QueryTreeNodePtr table_function_node_to_resolve = std::move(table_function_node_to_resolve_typed); + resolveTableFunction(table_function_node_to_resolve, scope, expressions_visitor, true /*nested_table_function*/); + + result_table_function_arguments.push_back(std::move(table_function_node_to_resolve)); + continue; + } + } + + resolveExpressionNode(table_function_argument, scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); + + if (auto * expression_list = table_function_argument->as()) + { + for (auto & expression_list_node : expression_list->getNodes()) + result_table_function_arguments.push_back(expression_list_node); + } + else + { + result_table_function_arguments.push_back(table_function_argument); + } + } + + table_function_node_typed.getArguments().getNodes() = std::move(result_table_function_arguments); + + auto table_function_ast = table_function_node_typed.toAST(); + table_function_ptr->parseArguments(table_function_ast, scope_context); + + auto table_function_storage = table_function_ptr->execute(table_function_ast, scope_context, table_function_ptr->getName()); + table_function_node_typed.resolve(std::move(table_function_ptr), std::move(table_function_storage), scope_context); +} + +/// Resolve array join node in scope +void QueryAnalyzer::resolveArrayJoin(QueryTreeNodePtr & array_join_node, IdentifierResolveScope & scope, QueryExpressionsAliasVisitor & expressions_visitor) +{ + auto & array_join_node_typed = array_join_node->as(); + resolveQueryJoinTreeNode(array_join_node_typed.getTableExpression(), scope, expressions_visitor); + + std::unordered_set array_join_column_names; + + /// Wrap array join expressions into column nodes, where array join expression is inner expression + + auto & array_join_nodes = array_join_node_typed.getJoinExpressions().getNodes(); + size_t array_join_nodes_size = array_join_nodes.size(); + + std::vector array_join_column_expressions; + array_join_column_expressions.reserve(array_join_nodes_size); + + for (auto & array_join_expression : array_join_nodes) + { + auto array_join_expression_alias = array_join_expression->getAlias(); + if (!array_join_expression_alias.empty() && scope.alias_name_to_expression_node.contains(array_join_expression_alias)) + throw Exception(ErrorCodes::MULTIPLE_EXPRESSIONS_FOR_ALIAS, + "ARRAY JOIN expression {} with duplicate alias {}. In scope {}", + array_join_expression->formatASTForErrorMessage(), + array_join_expression_alias, + scope.scope_node->formatASTForErrorMessage()); + + /// Add array join expression into scope + expressions_visitor.visit(array_join_expression); + + resolveExpressionNode(array_join_expression, scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); + + auto result_type = array_join_expression->getResultType(); + + if (!isArray(result_type)) + throw Exception(ErrorCodes::TYPE_MISMATCH, + "ARRAY JOIN {} requires expression {} with Array type. Actual {}. In scope {}", + array_join_node_typed.formatASTForErrorMessage(), + array_join_expression->formatASTForErrorMessage(), + result_type->getName(), + scope.scope_node->formatASTForErrorMessage()); + + result_type = assert_cast(*result_type).getNestedType(); + + String array_join_column_name; + + if (!array_join_expression_alias.empty()) + { + array_join_column_name = array_join_expression_alias; + } + else if (auto * array_join_expression_inner_column = array_join_expression->as()) + { + array_join_column_name = array_join_expression_inner_column->getColumnName(); + } + else + { + array_join_column_name = "__array_join_expression_" + std::to_string(array_join_expressions_counter); + ++array_join_expressions_counter; + } + + if (array_join_column_names.contains(array_join_column_name)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "ARRAY JOIN {} multiple columns with name {}. In scope {}", + array_join_node_typed.formatASTForErrorMessage(), + array_join_column_name, + scope.scope_node->formatASTForErrorMessage()); + array_join_column_names.emplace(array_join_column_name); + + NameAndTypePair array_join_column(array_join_column_name, result_type); + auto array_join_column_node = std::make_shared(std::move(array_join_column), array_join_expression, array_join_node); + array_join_column_node->setAlias(array_join_expression_alias); + array_join_column_expressions.push_back(std::move(array_join_column_node)); + } + + /** Allow to resolve ARRAY JOIN columns from aliases with types after ARRAY JOIN only after ARRAY JOIN expression list is resolved, because + * during resolution of ARRAY JOIN expression list we must use column type before ARRAY JOIN. + * + * Example: SELECT id, value_element FROM test_table ARRAY JOIN [[1,2,3]] AS value_element, value_element AS value + * It is expected that `value_element AS value` expression inside ARRAY JOIN expression list will be + * resolved as `value_element` expression with type before ARRAY JOIN. + * And it is expected that `value_element` inside projection expression list will be resolved as `value_element` expression + * with type after ARRAY JOIN. + */ + for (size_t i = 0; i < array_join_nodes_size; ++i) + { + auto & array_join_expression = array_join_nodes[i]; + array_join_expression = std::move(array_join_column_expressions[i]); + auto it = scope.alias_name_to_expression_node.find(array_join_expression->getAlias()); + if (it != scope.alias_name_to_expression_node.end()) + it->second = array_join_nodes[i]; + } +} + +/// Resolve join node in scope +void QueryAnalyzer::resolveJoin(QueryTreeNodePtr & join_node, IdentifierResolveScope & scope, QueryExpressionsAliasVisitor & expressions_visitor) +{ + auto & join_node_typed = join_node->as(); + + resolveQueryJoinTreeNode(join_node_typed.getLeftTableExpression(), scope, expressions_visitor); + validateJoinTableExpressionWithoutAlias(join_node, join_node_typed.getLeftTableExpression(), scope); + + resolveQueryJoinTreeNode(join_node_typed.getRightTableExpression(), scope, expressions_visitor); + validateJoinTableExpressionWithoutAlias(join_node, join_node_typed.getRightTableExpression(), scope); + + if (join_node_typed.isOnJoinExpression()) + { + expressions_visitor.visit(join_node_typed.getJoinExpression()); + auto join_expression = join_node_typed.getJoinExpression(); + resolveExpressionNode(join_expression, scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); + join_node_typed.getJoinExpression() = std::move(join_expression); + } + else if (join_node_typed.isUsingJoinExpression()) + { + auto & join_using_list = join_node_typed.getJoinExpression()->as(); + std::unordered_set join_using_identifiers; + + for (auto & join_using_node : join_using_list.getNodes()) + { + auto * identifier_node = join_using_node->as(); + if (!identifier_node) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "JOIN {} USING clause expected identifier. Actual {}", + join_node_typed.formatASTForErrorMessage(), + join_using_node->formatASTForErrorMessage()); + + const auto & identifier_full_name = identifier_node->getIdentifier().getFullName(); + + if (join_using_identifiers.contains(identifier_full_name)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "JOIN {} identifier '{}' appears more than once in USING clause", + join_node_typed.formatASTForErrorMessage(), + identifier_full_name); + + join_using_identifiers.insert(identifier_full_name); + + IdentifierLookup identifier_lookup{identifier_node->getIdentifier(), IdentifierLookupContext::EXPRESSION}; + auto result_left_table_expression = tryResolveIdentifierFromJoinTreeNode(identifier_lookup, join_node_typed.getLeftTableExpression(), scope); + if (!result_left_table_expression) + throw Exception(ErrorCodes::UNKNOWN_IDENTIFIER, + "JOIN {} using identifier '{}' cannot be resolved from left table expression. In scope {}", + join_node_typed.formatASTForErrorMessage(), + identifier_full_name, + scope.scope_node->formatASTForErrorMessage()); + + auto result_right_table_expression = tryResolveIdentifierFromJoinTreeNode(identifier_lookup, join_node_typed.getRightTableExpression(), scope); + if (!result_right_table_expression) + throw Exception(ErrorCodes::UNKNOWN_IDENTIFIER, + "JOIN {} using identifier '{}' cannot be resolved from right table expression. In scope {}", + join_node_typed.formatASTForErrorMessage(), + identifier_full_name, + scope.scope_node->formatASTForErrorMessage()); + + auto expression_types = DataTypes{result_left_table_expression->getResultType(), result_right_table_expression->getResultType()}; + DataTypePtr common_type = tryGetLeastSupertype(expression_types); + + if (!common_type) + throw Exception(ErrorCodes::NO_COMMON_TYPE, + "JOIN {} cannot infer common type for {} and {} in USING for identifier '{}'. In scope {}", + join_node_typed.formatASTForErrorMessage(), + result_left_table_expression->getResultType()->getName(), + result_right_table_expression->getResultType()->getName(), + identifier_full_name, + scope.scope_node->formatASTForErrorMessage()); + + NameAndTypePair join_using_column(identifier_full_name, common_type); + ListNodePtr join_using_expression = std::make_shared(QueryTreeNodes{result_left_table_expression, result_right_table_expression}); + auto join_using_column_node = std::make_shared(std::move(join_using_column), std::move(join_using_expression), join_node); + join_using_node = std::move(join_using_column_node); + } + } +} + /** Resolve query join tree. * * Query join tree must be initialized before calling this function. */ void QueryAnalyzer::resolveQueryJoinTreeNode(QueryTreeNodePtr & join_tree_node, IdentifierResolveScope & scope, QueryExpressionsAliasVisitor & expressions_visitor) { - auto add_table_expression_alias_into_scope = [&](const QueryTreeNodePtr & table_expression_node) - { - const auto & alias_name = table_expression_node->getAlias(); - if (alias_name.empty()) - return; - - auto [it, inserted] = scope.alias_name_to_table_expression_node.emplace(alias_name, table_expression_node); - if (!inserted) - throw Exception(ErrorCodes::UNSUPPORTED_METHOD, - "Duplicate aliases {} for table expressions in FROM section are not allowed. Try to register {}. Already registered {}.", - alias_name, - table_expression_node->formatASTForErrorMessage(), - it->second->formatASTForErrorMessage()); - }; - auto from_node_type = join_tree_node->getNodeType(); switch (from_node_type) @@ -5613,77 +5892,7 @@ void QueryAnalyzer::resolveQueryJoinTreeNode(QueryTreeNodePtr & join_tree_node, } case QueryTreeNodeType::TABLE_FUNCTION: { - auto & table_function_node = join_tree_node->as(); - expressions_visitor.visit(table_function_node.getArgumentsNode()); - - const auto & table_function_factory = TableFunctionFactory::instance(); - const auto & table_function_name = table_function_node.getTableFunctionName(); - - auto & scope_context = scope.context; - - TableFunctionPtr table_function_ptr = table_function_factory.tryGet(table_function_name, scope_context); - if (!table_function_ptr) - { - auto hints = TableFunctionFactory::instance().getHints(table_function_name); - if (!hints.empty()) - throw Exception(ErrorCodes::UNKNOWN_FUNCTION, - "Unknown table function {}. Maybe you meant: {}", - table_function_name, - DB::toString(hints)); - else - throw Exception(ErrorCodes::UNKNOWN_FUNCTION, "Unknown table function {}", table_function_name); - } - - if (scope_context->getSettingsRef().use_structure_from_insertion_table_in_table_functions && table_function_ptr->needStructureHint()) - { - const auto & insertion_table = scope_context->getInsertionTable(); - if (!insertion_table.empty()) - { - const auto & structure_hint - = DatabaseCatalog::instance().getTable(insertion_table, scope_context)->getInMemoryMetadataPtr()->columns; - table_function_ptr->setStructureHint(structure_hint); - } - } - - QueryTreeNodes result_nodes; - - for (auto & node : table_function_node.getArguments()) - { - if (auto * identifier_node = node->as()) - { - const auto & unresolved_identifier = identifier_node->getIdentifier(); - auto identifier_resolve_result = tryResolveIdentifier({unresolved_identifier, IdentifierLookupContext::EXPRESSION}, scope); - auto resolved_identifier = std::move(identifier_resolve_result.resolved_identifier); - - if (resolved_identifier && resolved_identifier->getNodeType() == QueryTreeNodeType::CONSTANT) - result_nodes.push_back(resolved_identifier); - else - result_nodes.push_back(node); - } - else - { - resolveExpressionNode(node, scope, false /*allow_lambda_expression*/, true /*allow_table_expression*/); - - if (auto * expression_list = node->as()) - { - for (auto & expression_list_node : expression_list->getNodes()) - result_nodes.push_back(expression_list_node); - } - else - { - result_nodes.push_back(node); - } - } - } - - table_function_node.getArguments().getNodes() = std::move(result_nodes); - - auto table_function_ast = table_function_node.toAST(); - table_function_ptr->parseArguments(table_function_ast, scope_context); - - auto table_function_storage = table_function_ptr->execute(table_function_ast, scope_context, table_function_ptr->getName()); - table_function_node.resolve(std::move(table_function_ptr), std::move(table_function_storage), scope_context); - + resolveTableFunction(join_tree_node, scope, expressions_visitor, false /*nested_table_function*/); break; } case QueryTreeNodeType::TABLE: @@ -5692,171 +5901,12 @@ void QueryAnalyzer::resolveQueryJoinTreeNode(QueryTreeNodePtr & join_tree_node, } case QueryTreeNodeType::ARRAY_JOIN: { - auto & array_join_node = join_tree_node->as(); - resolveQueryJoinTreeNode(array_join_node.getTableExpression(), scope, expressions_visitor); - - std::unordered_set array_join_column_names; - - /// Wrap array join expressions into column nodes, where array join expression is inner expression - - auto & array_join_nodes = array_join_node.getJoinExpressions().getNodes(); - size_t array_join_nodes_size = array_join_nodes.size(); - - std::vector array_join_column_expressions; - array_join_column_expressions.reserve(array_join_nodes_size); - - for (auto & array_join_expression : array_join_nodes) - { - auto array_join_expression_alias = array_join_expression->getAlias(); - if (!array_join_expression_alias.empty() && scope.alias_name_to_expression_node.contains(array_join_expression_alias)) - throw Exception(ErrorCodes::MULTIPLE_EXPRESSIONS_FOR_ALIAS, - "ARRAY JOIN expression {} with duplicate alias {}. In scope {}", - array_join_expression->formatASTForErrorMessage(), - array_join_expression_alias, - scope.scope_node->formatASTForErrorMessage()); - - /// Add array join expression into scope - expressions_visitor.visit(array_join_expression); - - resolveExpressionNode(array_join_expression, scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); - - auto result_type = array_join_expression->getResultType(); - - if (!isArray(result_type)) - throw Exception(ErrorCodes::TYPE_MISMATCH, - "ARRAY JOIN {} requires expression with Array type. Actual {}. In scope {}", - array_join_node.formatASTForErrorMessage(), - result_type->getName(), - scope.scope_node->formatASTForErrorMessage()); - - result_type = assert_cast(*result_type).getNestedType(); - - String array_join_column_name; - - if (!array_join_expression_alias.empty()) - { - array_join_column_name = array_join_expression_alias; - } - else if (auto * array_join_expression_inner_column = array_join_expression->as()) - { - array_join_column_name = array_join_expression_inner_column->getColumnName(); - } - else - { - array_join_column_name = "__array_join_expression_" + std::to_string(array_join_expressions_counter); - ++array_join_expressions_counter; - } - - if (array_join_column_names.contains(array_join_column_name)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "ARRAY JOIN {} multiple columns with name {}. In scope {}", - array_join_node.formatASTForErrorMessage(), - array_join_column_name, - scope.scope_node->formatASTForErrorMessage()); - array_join_column_names.emplace(array_join_column_name); - - auto array_join_column = std::make_shared(NameAndTypePair{array_join_column_name, result_type}, array_join_expression, join_tree_node); - array_join_column->setAlias(array_join_expression_alias); - array_join_column_expressions.push_back(std::move(array_join_column)); - } - - /** Allow to resolve ARRAY JOIN columns from aliases with types after ARRAY JOIN only after ARRAY JOIN expression list is resolved, because - * during resolution of ARRAY JOIN expression list we must use column type before ARRAY JOIN. - * - * Example: SELECT id, value_element FROM test_table ARRAY JOIN [[1,2,3]] AS value_element, value_element AS value - * It is expected that `value_element AS value` expression inside ARRAY JOIN expression list will be - * resolved as `value_element` expression with type before ARRAY JOIN. - * And it is expected that `value_element` inside projection expression list will be resolved as `value_element` expression - * with type after ARRAY JOIN. - */ - for (size_t i = 0; i < array_join_nodes_size; ++i) - { - auto & array_join_expression = array_join_nodes[i]; - array_join_expression = std::move(array_join_column_expressions[i]); - auto it = scope.alias_name_to_expression_node.find(array_join_expression->getAlias()); - if (it != scope.alias_name_to_expression_node.end()) - it->second = array_join_nodes[i]; - } - + resolveArrayJoin(join_tree_node, scope, expressions_visitor); break; } case QueryTreeNodeType::JOIN: { - auto & join_node = join_tree_node->as(); - - resolveQueryJoinTreeNode(join_node.getLeftTableExpression(), scope, expressions_visitor); - validateJoinTableExpressionWithoutAlias(join_tree_node, join_node.getLeftTableExpression(), scope); - - resolveQueryJoinTreeNode(join_node.getRightTableExpression(), scope, expressions_visitor); - validateJoinTableExpressionWithoutAlias(join_tree_node, join_node.getRightTableExpression(), scope); - - if (join_node.isUsingJoinExpression()) - { - auto & join_using_list = join_node.getJoinExpression()->as(); - std::unordered_set join_using_identifiers; - - for (auto & join_using_node : join_using_list.getNodes()) - { - auto * identifier_node = join_using_node->as(); - if (!identifier_node) - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "JOIN {} USING clause expected identifier. Actual {}", - join_node.formatASTForErrorMessage(), - join_using_node->formatASTForErrorMessage()); - - const auto & identifier_full_name = identifier_node->getIdentifier().getFullName(); - - if (join_using_identifiers.contains(identifier_full_name)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "JOIN {} identifier '{}' appears more than once in USING clause", - join_node.formatASTForErrorMessage(), - identifier_full_name); - - join_using_identifiers.insert(identifier_full_name); - - IdentifierLookup identifier_lookup {identifier_node->getIdentifier(), IdentifierLookupContext::EXPRESSION}; - auto result_left_table_expression = tryResolveIdentifierFromJoinTreeNode(identifier_lookup, join_node.getLeftTableExpression(), scope); - if (!result_left_table_expression) - throw Exception(ErrorCodes::UNKNOWN_IDENTIFIER, "JOIN {} using identifier '{}' cannot be resolved from left table expression. In scope {}", - join_node.formatASTForErrorMessage(), - identifier_full_name, - scope.scope_node->formatASTForErrorMessage()); - - auto result_right_table_expression = tryResolveIdentifierFromJoinTreeNode(identifier_lookup, join_node.getRightTableExpression(), scope); - if (!result_right_table_expression) - throw Exception(ErrorCodes::UNKNOWN_IDENTIFIER, - "JOIN {} using identifier '{}' cannot " - "be resolved from right table expression. In scope {}", - join_node.formatASTForErrorMessage(), - identifier_full_name, - scope.scope_node->formatASTForErrorMessage()); - - DataTypePtr common_type = tryGetLeastSupertype(DataTypes{result_left_table_expression->getResultType(), result_right_table_expression->getResultType()}); - - if (!common_type) - throw Exception(ErrorCodes::NO_COMMON_TYPE, - "JOIN {} cannot infer common type for {} and {} in USING for identifier '{}'. In scope {}", - join_node.formatASTForErrorMessage(), - result_left_table_expression->getResultType()->getName(), - result_right_table_expression->getResultType()->getName(), - identifier_full_name, - scope.scope_node->formatASTForErrorMessage()); - - NameAndTypePair join_using_columns_common_name_and_type(identifier_full_name, common_type); - ListNodePtr join_using_expression = std::make_shared(QueryTreeNodes{result_left_table_expression, result_right_table_expression}); - auto join_using_column = std::make_shared(join_using_columns_common_name_and_type, std::move(join_using_expression), join_tree_node); - - join_using_node = std::move(join_using_column); - } - } - else if (join_node.getJoinExpression()) - { - expressions_visitor.visit(join_node.getJoinExpression()); - auto join_expression = join_node.getJoinExpression(); - resolveExpressionNode(join_expression, scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); - join_node.getJoinExpression() = std::move(join_expression); - } - + resolveJoin(join_tree_node, scope, expressions_visitor); break; } case QueryTreeNodeType::IDENTIFIER: @@ -5882,6 +5932,21 @@ void QueryAnalyzer::resolveQueryJoinTreeNode(QueryTreeNodePtr & join_tree_node, initializeTableExpressionColumns(join_tree_node, scope); } + auto add_table_expression_alias_into_scope = [&](const QueryTreeNodePtr & table_expression_node) + { + const auto & alias_name = table_expression_node->getAlias(); + if (alias_name.empty()) + return; + + auto [it, inserted] = scope.alias_name_to_table_expression_node.emplace(alias_name, table_expression_node); + if (!inserted) + throw Exception(ErrorCodes::UNSUPPORTED_METHOD, + "Duplicate aliases {} for table expressions in FROM section are not allowed. Try to register {}. Already registered {}.", + alias_name, + table_expression_node->formatASTForErrorMessage(), + it->second->formatASTForErrorMessage()); + }; + add_table_expression_alias_into_scope(join_tree_node); scope.table_expressions_in_resolve_process.erase(join_tree_node.get()); } diff --git a/src/Analyzer/QueryTreePassManager.cpp b/src/Analyzer/QueryTreePassManager.cpp index 38d2ae731a1..de67f5c2d49 100644 --- a/src/Analyzer/QueryTreePassManager.cpp +++ b/src/Analyzer/QueryTreePassManager.cpp @@ -62,6 +62,14 @@ public: : pass_name(std::move(pass_name_)) {} + bool needChildVisit(VisitQueryTreeNodeType & parent, VisitQueryTreeNodeType &) + { + if (parent->getNodeType() == QueryTreeNodeType::TABLE_FUNCTION) + return false; + + return true; + } + void visitImpl(QueryTreeNodePtr & node) const { if (auto * column = node->as()) diff --git a/src/Analyzer/Utils.cpp b/src/Analyzer/Utils.cpp index dcdab330728..bd859ecd379 100644 --- a/src/Analyzer/Utils.cpp +++ b/src/Analyzer/Utils.cpp @@ -90,6 +90,8 @@ static ASTPtr convertIntoTableExpressionAST(const QueryTreeNodePtr & table_expre throw Exception(ErrorCodes::LOGICAL_ERROR, "Identifier for table expression must contain 1 or 2 parts. Actual '{}'", identifier.getFullName()); + + table_expression_node_ast->setAlias(identifier_node.getAlias()); } else { diff --git a/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp b/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp index 6b6f3560c5a..2d869cf0d5d 100644 --- a/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp +++ b/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp @@ -63,6 +63,11 @@ QueryTreeNodePtr buildQueryTreeAndRunPasses(const ASTPtr & query, const SelectQu return query_tree; } +PlannerConfiguration buildPlannerConfiguration(const SelectQueryOptions & select_query_options) +{ + return {.only_analyze = select_query_options.only_analyze}; +} + } InterpreterSelectQueryAnalyzer::InterpreterSelectQueryAnalyzer( @@ -73,7 +78,7 @@ InterpreterSelectQueryAnalyzer::InterpreterSelectQueryAnalyzer( , context(Context::createCopy(context_)) , select_query_options(select_query_options_) , query_tree(buildQueryTreeAndRunPasses(query, select_query_options, context)) - , planner(query_tree, select_query_options) + , planner(query_tree, select_query_options, buildPlannerConfiguration(select_query_options)) { } @@ -85,10 +90,32 @@ InterpreterSelectQueryAnalyzer::InterpreterSelectQueryAnalyzer( , context(Context::createCopy(context_)) , select_query_options(select_query_options_) , query_tree(query_tree_) - , planner(query_tree, select_query_options) + , planner(query_tree, select_query_options, buildPlannerConfiguration(select_query_options)) { } +Block InterpreterSelectQueryAnalyzer::getSampleBlock(const ASTPtr & query, + const ContextPtr & context, + const SelectQueryOptions & select_query_options) +{ + auto select_query_options_copy = select_query_options; + select_query_options_copy.only_analyze = true; + InterpreterSelectQueryAnalyzer interpreter(query, context, select_query_options_copy); + + return interpreter.getSampleBlock(); +} + +Block InterpreterSelectQueryAnalyzer::getSampleBlock(const QueryTreeNodePtr & query, + const ContextPtr & context, + const SelectQueryOptions & select_query_options) +{ + auto select_query_options_copy = select_query_options; + select_query_options_copy.only_analyze = true; + InterpreterSelectQueryAnalyzer interpreter(query, context, select_query_options_copy); + + return interpreter.getSampleBlock(); +} + Block InterpreterSelectQueryAnalyzer::getSampleBlock() { planner.buildQueryPlanIfNeeded(); diff --git a/src/Interpreters/InterpreterSelectQueryAnalyzer.h b/src/Interpreters/InterpreterSelectQueryAnalyzer.h index 7fe64a7fb99..bd6a96f597b 100644 --- a/src/Interpreters/InterpreterSelectQueryAnalyzer.h +++ b/src/Interpreters/InterpreterSelectQueryAnalyzer.h @@ -32,6 +32,14 @@ public: Block getSampleBlock(); + static Block getSampleBlock(const ASTPtr & query, + const ContextPtr & context, + const SelectQueryOptions & select_query_options = {}); + + static Block getSampleBlock(const QueryTreeNodePtr & query_, + const ContextPtr & context_, + const SelectQueryOptions & select_query_options = {}); + BlockIO execute() override; QueryPlan && extractQueryPlan() &&; diff --git a/src/TableFunctions/ITableFunction.h b/src/TableFunctions/ITableFunction.h index dfcf9012068..f59092acf69 100644 --- a/src/TableFunctions/ITableFunction.h +++ b/src/TableFunctions/ITableFunction.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -49,6 +50,11 @@ public: /// Returns false if storage returned by table function supports type conversion (e.g. StorageDistributed) virtual bool needStructureConversion() const { return true; } + /** Return array of table function arguments indexes for which query tree analysis must be skipped. + * It is important for table functions that take subqueries, because otherwise analyzer will resolve them. + */ + virtual std::vector skipAnalysisForArguments(const QueryTreeNodePtr & /*query_node_table_function*/, ContextPtr /*context*/) const { return {}; } + virtual void parseArguments(const ASTPtr & /*ast_function*/, ContextPtr /*context*/) {} /// Returns actual table structure probably requested from remote server, may fail diff --git a/src/TableFunctions/TableFunctionExecutable.cpp b/src/TableFunctions/TableFunctionExecutable.cpp index 260b80c40f3..ca769a2ff0a 100644 --- a/src/TableFunctions/TableFunctionExecutable.cpp +++ b/src/TableFunctions/TableFunctionExecutable.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -27,6 +28,20 @@ namespace ErrorCodes extern const int UNSUPPORTED_METHOD; } +std::vector TableFunctionExecutable::skipAnalysisForArguments(const QueryTreeNodePtr & query_node_table_function, ContextPtr) const +{ + const auto & table_function_node = query_node_table_function->as(); + const auto & table_function_node_arguments = table_function_node.getArguments().getNodes(); + size_t table_function_node_arguments_size = table_function_node_arguments.size(); + + std::vector result_indexes; + + for (size_t i = 3; i < table_function_node_arguments_size; ++i) + result_indexes.push_back(i); + + return result_indexes; +} + void TableFunctionExecutable::parseArguments(const ASTPtr & ast_function, ContextPtr context) { const auto * function = ast_function->as(); diff --git a/src/TableFunctions/TableFunctionExecutable.h b/src/TableFunctions/TableFunctionExecutable.h index 820da077ca2..2d9f86e14e6 100644 --- a/src/TableFunctions/TableFunctionExecutable.h +++ b/src/TableFunctions/TableFunctionExecutable.h @@ -18,14 +18,20 @@ class TableFunctionExecutable : public ITableFunction { public: static constexpr auto name = "executable"; + std::string getName() const override { return name; } + bool hasStaticStructure() const override { return true; } private: StoragePtr executeImpl(const ASTPtr & ast_function, ContextPtr context, const std::string & table_name, ColumnsDescription cached_columns) const override; + const char * getStorageTypeName() const override { return "Executable"; } ColumnsDescription getActualTableStructure(ContextPtr context) const override; + + std::vector skipAnalysisForArguments(const QueryTreeNodePtr & query_node_table_function, ContextPtr context) const override; + void parseArguments(const ASTPtr & ast_function, ContextPtr context) override; String script_name; diff --git a/src/TableFunctions/TableFunctionExplain.cpp b/src/TableFunctions/TableFunctionExplain.cpp index 5b2d0e7b78f..e28d6e7c7fc 100644 --- a/src/TableFunctions/TableFunctionExplain.cpp +++ b/src/TableFunctions/TableFunctionExplain.cpp @@ -1,4 +1,3 @@ -#include #include #include #include diff --git a/src/TableFunctions/TableFunctionView.cpp b/src/TableFunctions/TableFunctionView.cpp index 578c497e720..6b50e7e0611 100644 --- a/src/TableFunctions/TableFunctionView.cpp +++ b/src/TableFunctions/TableFunctionView.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -21,6 +22,11 @@ const ASTSelectWithUnionQuery & TableFunctionView::getSelectQuery() const return *create.select; } +std::vector TableFunctionView::skipAnalysisForArguments(const QueryTreeNodePtr &, ContextPtr) const +{ + return {0}; +} + void TableFunctionView::parseArguments(const ASTPtr & ast_function, ContextPtr /*context*/) { const auto * function = ast_function->as(); @@ -40,8 +46,15 @@ ColumnsDescription TableFunctionView::getActualTableStructure(ContextPtr context assert(create.select); assert(create.children.size() == 1); assert(create.children[0]->as()); - auto sample = InterpreterSelectWithUnionQuery::getSampleBlock(create.children[0], context); - return ColumnsDescription(sample.getNamesAndTypesList()); + + Block sample_block; + + if (context->getSettingsRef().allow_experimental_analyzer) + sample_block = InterpreterSelectQueryAnalyzer::getSampleBlock(create.children[0], context); + else + sample_block = InterpreterSelectWithUnionQuery::getSampleBlock(create.children[0], context); + + return ColumnsDescription(sample_block.getNamesAndTypesList()); } StoragePtr TableFunctionView::executeImpl( diff --git a/src/TableFunctions/TableFunctionView.h b/src/TableFunctions/TableFunctionView.h index 4afb049e738..bbf072655ed 100644 --- a/src/TableFunctions/TableFunctionView.h +++ b/src/TableFunctions/TableFunctionView.h @@ -15,15 +15,20 @@ class TableFunctionView : public ITableFunction { public: static constexpr auto name = "view"; + std::string getName() const override { return name; } const ASTSelectWithUnionQuery & getSelectQuery() const; private: StoragePtr executeImpl(const ASTPtr & ast_function, ContextPtr context, const String & table_name, ColumnsDescription cached_columns) const override; + const char * getStorageTypeName() const override { return "View"; } + std::vector skipAnalysisForArguments(const QueryTreeNodePtr & query_node_table_function, ContextPtr context) const override; + void parseArguments(const ASTPtr & ast_function, ContextPtr context) override; + ColumnsDescription getActualTableStructure(ContextPtr context) const override; ASTCreateQuery create; diff --git a/src/TableFunctions/TableFunctionViewIfPermitted.cpp b/src/TableFunctions/TableFunctionViewIfPermitted.cpp index 6128fe0a36f..12762e24f7e 100644 --- a/src/TableFunctions/TableFunctionViewIfPermitted.cpp +++ b/src/TableFunctions/TableFunctionViewIfPermitted.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -28,6 +29,11 @@ const ASTSelectWithUnionQuery & TableFunctionViewIfPermitted::getSelectQuery() c return *create.select; } +std::vector TableFunctionViewIfPermitted::skipAnalysisForArguments(const QueryTreeNodePtr &, ContextPtr) const +{ + return {0}; +} + void TableFunctionViewIfPermitted::parseArguments(const ASTPtr & ast_function, ContextPtr context) { const auto * function = ast_function->as(); @@ -79,8 +85,15 @@ bool TableFunctionViewIfPermitted::isPermitted(const ContextPtr & context, const try { - /// Will throw ACCESS_DENIED if the current user is not allowed to execute the SELECT query. - sample_block = InterpreterSelectWithUnionQuery::getSampleBlock(create.children[0], context); + if (context->getSettingsRef().allow_experimental_analyzer) + { + sample_block = InterpreterSelectQueryAnalyzer::getSampleBlock(create.children[0], context); + } + else + { + /// Will throw ACCESS_DENIED if the current user is not allowed to execute the SELECT query. + sample_block = InterpreterSelectWithUnionQuery::getSampleBlock(create.children[0], context); + } } catch (Exception & e) { diff --git a/src/TableFunctions/TableFunctionViewIfPermitted.h b/src/TableFunctions/TableFunctionViewIfPermitted.h index 0fd0b050eaf..9fdb34f30ab 100644 --- a/src/TableFunctions/TableFunctionViewIfPermitted.h +++ b/src/TableFunctions/TableFunctionViewIfPermitted.h @@ -14,15 +14,20 @@ class TableFunctionViewIfPermitted : public ITableFunction { public: static constexpr auto name = "viewIfPermitted"; + std::string getName() const override { return name; } const ASTSelectWithUnionQuery & getSelectQuery() const; private: StoragePtr executeImpl(const ASTPtr & ast_function, ContextPtr context, const String & table_name, ColumnsDescription cached_columns) const override; + const char * getStorageTypeName() const override { return "ViewIfPermitted"; } + std::vector skipAnalysisForArguments(const QueryTreeNodePtr & query_node_table_function, ContextPtr context) const override; + void parseArguments(const ASTPtr & ast_function, ContextPtr context) override; + ColumnsDescription getActualTableStructure(ContextPtr context) const override; bool isPermitted(const ContextPtr & context, const ColumnsDescription & else_columns) const; From cd3e8482ae4a48a44bc2ddee8af90f13e6a0febf Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Thu, 26 Jan 2023 14:46:32 +0100 Subject: [PATCH 291/717] Analyzer TableFunctionExplain fix --- src/Analyzer/MatcherNode.cpp | 8 +++-- src/Analyzer/Passes/QueryAnalysisPass.cpp | 1 + src/Planner/Utils.cpp | 1 - src/TableFunctions/TableFunctionExplain.cpp | 13 ++++++++ src/TableFunctions/TableFunctionExplain.h | 6 +++- .../0_stateless/02421_explain_subquery.sql | 30 ++----------------- 6 files changed, 27 insertions(+), 32 deletions(-) diff --git a/src/Analyzer/MatcherNode.cpp b/src/Analyzer/MatcherNode.cpp index fc74b4ff67e..5c8738e0504 100644 --- a/src/Analyzer/MatcherNode.cpp +++ b/src/Analyzer/MatcherNode.cpp @@ -209,12 +209,14 @@ ASTPtr MatcherNode::toASTImpl() const ASTPtr result; ASTPtr transformers; - if (!children.empty()) + const auto & column_transformers = getColumnTransformers().getNodes(); + + if (!column_transformers.empty()) { transformers = std::make_shared(); - for (const auto & child : children) - transformers->children.push_back(child->toAST()); + for (const auto & column_transformer : column_transformers) + transformers->children.push_back(column_transformer->toAST()); } if (matcher_type == MatcherNodeType::ASTERISK) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 7685fed6948..3a5597f164f 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -108,6 +108,7 @@ namespace ErrorCodes extern const int NOT_IMPLEMENTED; extern const int ALIAS_REQUIRED; extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } /** Query analyzer implementation overview. Please check documentation in QueryAnalysisPass.h before. diff --git a/src/Planner/Utils.cpp b/src/Planner/Utils.cpp index a991816def9..0eac0881609 100644 --- a/src/Planner/Utils.cpp +++ b/src/Planner/Utils.cpp @@ -26,7 +26,6 @@ namespace DB namespace ErrorCodes { - extern const int TYPE_MISMATCH; extern const int LOGICAL_ERROR; extern const int UNION_ALL_RESULT_STRUCTURES_MISMATCH; extern const int INTERSECT_OR_EXCEPT_RESULT_STRUCTURES_MISMATCH; diff --git a/src/TableFunctions/TableFunctionExplain.cpp b/src/TableFunctions/TableFunctionExplain.cpp index e28d6e7c7fc..081aa4fc1dc 100644 --- a/src/TableFunctions/TableFunctionExplain.cpp +++ b/src/TableFunctions/TableFunctionExplain.cpp @@ -10,6 +10,7 @@ #include #include #include +#include namespace DB { @@ -19,6 +20,18 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } +std::vector TableFunctionExplain::skipAnalysisForArguments(const QueryTreeNodePtr & query_node_table_function, ContextPtr /*context*/) const +{ + const auto & table_function_node = query_node_table_function->as(); + const auto & table_function_node_arguments = table_function_node.getArguments().getNodes(); + size_t table_function_node_arguments_size = table_function_node_arguments.size(); + + if (table_function_node_arguments_size == 3) + return {2}; + + return {}; +} + void TableFunctionExplain::parseArguments(const ASTPtr & ast_function, ContextPtr /*context*/) { const auto * function = ast_function->as(); diff --git a/src/TableFunctions/TableFunctionExplain.h b/src/TableFunctions/TableFunctionExplain.h index 9d6dde4760c..99d3e52ee68 100644 --- a/src/TableFunctions/TableFunctionExplain.h +++ b/src/TableFunctions/TableFunctionExplain.h @@ -13,13 +13,18 @@ class TableFunctionExplain : public ITableFunction { public: static constexpr auto name = "viewExplain"; + std::string getName() const override { return name; } private: StoragePtr executeImpl(const ASTPtr & ast_function, ContextPtr context, const String & table_name, ColumnsDescription cached_columns) const override; + const char * getStorageTypeName() const override { return "Explain"; } + std::vector skipAnalysisForArguments(const QueryTreeNodePtr & query_node_table_function, ContextPtr context) const override; + void parseArguments(const ASTPtr & ast_function, ContextPtr context) override; + ColumnsDescription getActualTableStructure(ContextPtr context) const override; InterpreterExplainQuery getInterpreter(ContextPtr context) const; @@ -27,5 +32,4 @@ private: ASTPtr query = nullptr; }; - } diff --git a/tests/queries/0_stateless/02421_explain_subquery.sql b/tests/queries/0_stateless/02421_explain_subquery.sql index 9f83b2f56c7..32631b54d0c 100644 --- a/tests/queries/0_stateless/02421_explain_subquery.sql +++ b/tests/queries/0_stateless/02421_explain_subquery.sql @@ -24,12 +24,13 @@ SELECT * FROM viewExplain('EXPLAIN AST', ''); -- { serverError BAD_ARGUMENTS } SELECT * FROM viewExplain('EXPLAIN AST', '', 1); -- { serverError BAD_ARGUMENTS } SELECT * FROM viewExplain('EXPLAIN AST', '', ''); -- { serverError BAD_ARGUMENTS } +DROP TABLE IF EXISTS t1; CREATE TABLE t1 ( a UInt64 ) Engine = MergeTree ORDER BY tuple() AS SELECT number AS a FROM system.numbers LIMIT 100000; SELECT rows > 1000 FROM (EXPLAIN ESTIMATE SELECT sum(a) FROM t1); SELECT count() == 1 FROM (EXPLAIN ESTIMATE SELECT sum(a) FROM t1); -DROP TABLE IF EXISTS t1; +DROP TABLE t1; SET allow_experimental_analyzer = 1; @@ -38,32 +39,7 @@ SELECT count() > 0 FROM (EXPLAIN PLAN SELECT * FROM system.numbers ORDER BY numb SELECT count() > 0 FROM (EXPLAIN SELECT * FROM system.numbers ORDER BY number DESC) WHERE explain ILIKE '%Sort%'; SELECT count() > 0 FROM (EXPLAIN CURRENT TRANSACTION); SELECT count() == 1 FROM (EXPLAIN SYNTAX SELECT number FROM system.numbers ORDER BY number DESC) WHERE explain ILIKE 'SELECT%'; - --- We have `Identifier number` instead of `Asterisk` because query argument of `viewExplain` table function was analyzed. --- Compare: --- :) EXPLAIN AST SELECT *; --- ┌─explain───────────────────────────┐ --- │ SelectWithUnionQuery (children 1) │ --- │ ExpressionList (children 1) │ --- │ SelectQuery (children 1) │ --- │ ExpressionList (children 1) │ --- │ Asterisk │ --- └───────────────────────────────────┘ --- :) SELECT * FROM (EXPLAIN AST SELECT *); --- ┌─explain─────────────────────────────────────┐ --- │ SelectWithUnionQuery (children 1) │ --- │ ExpressionList (children 1) │ --- │ SelectQuery (children 2) │ --- │ ExpressionList (children 1) │ --- │ Identifier dummy │ --- │ TablesInSelectQuery (children 1) │ --- │ TablesInSelectQueryElement (children 1) │ --- │ TableExpression (children 1) │ --- │ TableIdentifier system.one │ --- └─────────────────────────────────────────────┘ --- TODO: argument of `viewExplain` (and subquery in `EXAPLAN ...`) should not be analyzed. --- See _Support query tree in table functions_ in https://github.com/ClickHouse/ClickHouse/issues/42648 -SELECT trim(explain) == 'Identifier number' FROM (EXPLAIN AST SELECT * FROM system.numbers LIMIT 10) WHERE explain LIKE '%Identifier number%'; +SELECT trim(explain) == 'Asterisk' FROM (EXPLAIN AST SELECT * FROM system.numbers LIMIT 10) WHERE explain LIKE '%Asterisk%'; SELECT * FROM ( EXPLAIN AST SELECT * FROM ( From 8497eba49f95e280ea94c858e195d3a3d23954fd Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Thu, 26 Jan 2023 18:16:36 +0100 Subject: [PATCH 292/717] Fixed tests --- src/Analyzer/ColumnNode.cpp | 10 +--------- src/Analyzer/IQueryTreeNode.cpp | 13 +++++++++++-- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/Analyzer/ColumnNode.cpp b/src/Analyzer/ColumnNode.cpp index 4ec284f6462..a5f7dc9786e 100644 --- a/src/Analyzer/ColumnNode.cpp +++ b/src/Analyzer/ColumnNode.cpp @@ -73,15 +73,7 @@ void ColumnNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & state, size_t bool ColumnNode::isEqualImpl(const IQueryTreeNode & rhs) const { const auto & rhs_typed = assert_cast(rhs); - - auto source = getColumnSourceOrNull(); - auto rhs_source = rhs_typed.getColumnSourceOrNull(); - if (source && !rhs_source) - return false; - if (!source && rhs_source) - return false; - - return column == rhs_typed.column && (!source || source->isEqual(*rhs_source)); + return column == rhs_typed.column; } void ColumnNode::updateTreeHashImpl(HashState & hash_state) const diff --git a/src/Analyzer/IQueryTreeNode.cpp b/src/Analyzer/IQueryTreeNode.cpp index 1970d36a3dd..df841f9a61a 100644 --- a/src/Analyzer/IQueryTreeNode.cpp +++ b/src/Analyzer/IQueryTreeNode.cpp @@ -76,6 +76,9 @@ struct NodePairHash bool IQueryTreeNode::isEqual(const IQueryTreeNode & rhs) const { + if (this == &rhs) + return true; + std::vector nodes_to_process; std::unordered_set equals_pairs; @@ -89,11 +92,17 @@ bool IQueryTreeNode::isEqual(const IQueryTreeNode & rhs) const const auto * lhs_node_to_compare = nodes_to_compare.first; const auto * rhs_node_to_compare = nodes_to_compare.second; + assert(lhs_node_to_compare); + assert(rhs_node_to_compare); + if (equals_pairs.contains(std::make_pair(lhs_node_to_compare, rhs_node_to_compare))) continue; - assert(lhs_node_to_compare); - assert(rhs_node_to_compare); + if (lhs_node_to_compare == rhs_node_to_compare) + { + equals_pairs.emplace(lhs_node_to_compare, rhs_node_to_compare); + continue; + } if (lhs_node_to_compare->getNodeType() != rhs_node_to_compare->getNodeType() || lhs_node_to_compare->alias != rhs_node_to_compare->alias || From a12aea45f64f0786f9ce50f1469521221e4c5255 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Fri, 27 Jan 2023 17:59:54 +0100 Subject: [PATCH 293/717] Analyzer matcher resolve fix --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 253 +++++++++++------- src/Storages/ColumnsDescription.cpp | 26 ++ src/Storages/ColumnsDescription.h | 3 +- ...tcher_alias_materialized_columns.reference | 18 ++ ...zer_matcher_alias_materialized_columns.sql | 33 +++ 5 files changed, 239 insertions(+), 94 deletions(-) create mode 100644 tests/queries/0_stateless/02540_analyzer_matcher_alias_materialized_columns.reference create mode 100644 tests/queries/0_stateless/02540_analyzer_matcher_alias_materialized_columns.sql diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 3a5597f164f..df93696c540 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -405,8 +405,8 @@ struct TableExpressionData { std::string table_expression_name; std::string table_expression_description; - std::string table_name; std::string database_name; + std::string table_name; ColumnNameToColumnNodeMap column_name_to_column_node; std::unordered_set> column_identifier_first_parts; @@ -422,7 +422,18 @@ struct TableExpressionData [[maybe_unused]] void dump(WriteBuffer & buffer) const { - buffer << "Columns size " << column_name_to_column_node.size() << '\n'; + buffer << "Table expression name " << table_expression_name; + + if (!table_expression_description.empty()) + buffer << " table expression description " << table_expression_description; + + if (!database_name.empty()) + buffer << " database name " << database_name; + + if (!table_name.empty()) + buffer << " table name " << table_name; + + buffer << " columns size " << column_name_to_column_node.size() << '\n'; for (const auto & [column_name, column_node] : column_name_to_column_node) buffer << "Column name " << column_name << " column node " << column_node->dumpTree() << '\n'; @@ -749,7 +760,21 @@ struct IdentifierResolveScope return scope_to_check; } - TableExpressionData & getTableExpressionDataOrThrow(QueryTreeNodePtr table_expression_node) + TableExpressionData & getTableExpressionDataOrThrow(const QueryTreeNodePtr & table_expression_node) + { + auto it = table_expression_node_to_data.find(table_expression_node); + if (it == table_expression_node_to_data.end()) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Table expression {} data must be initialized. In scope {}", + table_expression_node->formatASTForErrorMessage(), + scope_node->formatASTForErrorMessage()); + } + + return it->second; + } + + const TableExpressionData & getTableExpressionDataOrThrow(const QueryTreeNodePtr & table_expression_node) const { auto it = table_expression_node_to_data.find(table_expression_node); if (it == table_expression_node_to_data.end()) @@ -1139,13 +1164,13 @@ private: QueryTreeNodePtr tryResolveIdentifierFromExpressionArguments(const IdentifierLookup & identifier_lookup, IdentifierResolveScope & scope); - static bool tryBindIdentifierToAliases(const IdentifierLookup & identifier_lookup, IdentifierResolveScope & scope); + static bool tryBindIdentifierToAliases(const IdentifierLookup & identifier_lookup, const IdentifierResolveScope & scope); QueryTreeNodePtr tryResolveIdentifierFromAliases(const IdentifierLookup & identifier_lookup, IdentifierResolveScope & scope, IdentifierResolveSettings identifier_resolve_settings = {}); QueryTreeNodePtr tryResolveIdentifierFromTableColumns(const IdentifierLookup & identifier_lookup, IdentifierResolveScope & scope); - static bool tryBindIdentifierToTableExpression(const IdentifierLookup & identifier_lookup, const QueryTreeNodePtr & table_expression_node, IdentifierResolveScope & scope); + static bool tryBindIdentifierToTableExpression(const IdentifierLookup & identifier_lookup, const QueryTreeNodePtr & table_expression_node, const IdentifierResolveScope & scope); QueryTreeNodePtr tryResolveIdentifierFromTableExpression(const IdentifierLookup & identifier_lookup, const QueryTreeNodePtr & table_expression_node, IdentifierResolveScope & scope); @@ -1163,11 +1188,18 @@ private: /// Resolve query tree nodes functions + void qualifyColumnNodesWithProjectionNames(const QueryTreeNodes & column_nodes, + const QueryTreeNodePtr & table_expression_node, + const IdentifierResolveScope & scope); + + static GetColumnsOptions buildGetColumnsOptions(QueryTreeNodePtr & matcher_node, const ContextPtr & context); + using QueryTreeNodesWithNames = std::vector>; - void qualifyMatchedColumnsProjectionNamesIfNeeded(QueryTreeNodesWithNames & matched_nodes_with_column_names, + QueryTreeNodesWithNames getMatchedColumnNodesWithNames(const QueryTreeNodePtr & matcher_node, const QueryTreeNodePtr & table_expression_node, - IdentifierResolveScope & scope); + const NamesAndTypes & matched_columns, + const IdentifierResolveScope & scope); QueryTreeNodesWithNames resolveQualifiedMatcher(QueryTreeNodePtr & matcher_node, IdentifierResolveScope & scope); @@ -2268,11 +2300,11 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromExpressionArguments(cons return it->second; } -bool QueryAnalyzer::tryBindIdentifierToAliases(const IdentifierLookup & identifier_lookup, IdentifierResolveScope & scope) +bool QueryAnalyzer::tryBindIdentifierToAliases(const IdentifierLookup & identifier_lookup, const IdentifierResolveScope & scope) { const auto & identifier_bind_part = identifier_lookup.identifier.front(); - auto get_alias_name_to_node_map = [&]() -> std::unordered_map & + auto get_alias_name_to_node_map = [&]() -> const std::unordered_map & { if (identifier_lookup.isExpressionLookup()) return scope.alias_name_to_expression_node; @@ -2282,13 +2314,9 @@ bool QueryAnalyzer::tryBindIdentifierToAliases(const IdentifierLookup & identifi return scope.alias_name_to_table_expression_node; }; - auto & alias_name_to_node_map = get_alias_name_to_node_map(); - auto it = alias_name_to_node_map.find(identifier_bind_part); + const auto & alias_name_to_node_map = get_alias_name_to_node_map(); - if (it == alias_name_to_node_map.end()) - return false; - - return true; + return alias_name_to_node_map.contains(identifier_bind_part); } /** Resolve identifier from scope aliases. @@ -2482,7 +2510,9 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromTableColumns(const Ident return result; } -bool QueryAnalyzer::tryBindIdentifierToTableExpression(const IdentifierLookup & identifier_lookup, const QueryTreeNodePtr & table_expression_node, IdentifierResolveScope & scope) +bool QueryAnalyzer::tryBindIdentifierToTableExpression(const IdentifierLookup & identifier_lookup, + const QueryTreeNodePtr & table_expression_node, + const IdentifierResolveScope & scope) { auto table_expression_node_type = table_expression_node->getNodeType(); @@ -2498,7 +2528,7 @@ bool QueryAnalyzer::tryBindIdentifierToTableExpression(const IdentifierLookup & const auto & identifier = identifier_lookup.identifier; const auto & path_start = identifier.getParts().front(); - auto & table_expression_data = scope.getTableExpressionDataOrThrow(table_expression_node); + const auto & table_expression_data = scope.getTableExpressionDataOrThrow(table_expression_node); const auto & table_name = table_expression_data.table_name; const auto & database_name = table_expression_data.database_name; @@ -3197,12 +3227,13 @@ IdentifierResolveResult QueryAnalyzer::tryResolveIdentifier(const IdentifierLook /// Resolve query tree nodes functions implementation -/** Qualify matched columns projection names for unqualified matcher or qualified matcher resolved nodes +/** Qualify column nodes with projection names. * * Example: SELECT * FROM test_table AS t1, test_table AS t2; */ -void QueryAnalyzer::qualifyMatchedColumnsProjectionNamesIfNeeded(QueryTreeNodesWithNames & matched_nodes_with_column_names, - const QueryTreeNodePtr & table_expression_node, IdentifierResolveScope & scope) +void QueryAnalyzer::qualifyColumnNodesWithProjectionNames(const QueryTreeNodes & column_nodes, + const QueryTreeNodePtr & table_expression_node, + const IdentifierResolveScope & scope) { /// Build additional column qualification parts array std::vector additional_column_qualification_parts; @@ -3219,8 +3250,9 @@ void QueryAnalyzer::qualifyMatchedColumnsProjectionNamesIfNeeded(QueryTreeNodesW */ std::vector column_qualified_identifier_parts; - for (auto & [column_node, column_name] : matched_nodes_with_column_names) + for (const auto & column_node : column_nodes) { + const auto & column_name = column_node->as().getColumnName(); column_qualified_identifier_parts = Identifier(column_name).getParts(); /// Iterate over additional column qualifications and apply them if needed @@ -3230,7 +3262,7 @@ void QueryAnalyzer::qualifyMatchedColumnsProjectionNamesIfNeeded(QueryTreeNodesW auto identifier_to_check = Identifier(column_qualified_identifier_parts); IdentifierLookup lookup{identifier_to_check, IdentifierLookupContext::EXPRESSION}; - for (auto & table_expression_data : scope.table_expression_node_to_data) + for (const auto & table_expression_data : scope.table_expression_node_to_data) { if (table_expression_data.first.get() == table_expression_node.get()) continue; @@ -3262,10 +3294,83 @@ void QueryAnalyzer::qualifyMatchedColumnsProjectionNamesIfNeeded(QueryTreeNodesW } } - node_to_projection_name.emplace(column_node, Identifier(column_qualified_identifier_parts).getFullName()); + auto qualified_node_name = Identifier(column_qualified_identifier_parts).getFullName(); + node_to_projection_name.emplace(column_node, qualified_node_name); } } +/// Build get columns options for matcher +GetColumnsOptions QueryAnalyzer::buildGetColumnsOptions(QueryTreeNodePtr & matcher_node, const ContextPtr & context) +{ + auto & matcher_node_typed = matcher_node->as(); + UInt8 get_columns_options_kind = GetColumnsOptions::AllPhysicalAndAliases; + + if (matcher_node_typed.isAsteriskMatcher()) + { + get_columns_options_kind = GetColumnsOptions::Ordinary; + + const auto & settings = context->getSettingsRef(); + + if (settings.asterisk_include_alias_columns) + get_columns_options_kind |= GetColumnsOptions::Kind::Aliases; + + if (settings.asterisk_include_materialized_columns) + get_columns_options_kind |= GetColumnsOptions::Kind::Materialized; + } + + return GetColumnsOptions(static_cast(get_columns_options_kind)); +} + +QueryAnalyzer::QueryTreeNodesWithNames QueryAnalyzer::getMatchedColumnNodesWithNames(const QueryTreeNodePtr & matcher_node, + const QueryTreeNodePtr & table_expression_node, + const NamesAndTypes & matched_columns, + const IdentifierResolveScope & scope) +{ + auto & matcher_node_typed = matcher_node->as(); + + /** Use resolved columns from table expression data in nearest query scope if available. + * It is important for ALIAS columns to use column nodes with resolved ALIAS expression. + */ + const TableExpressionData * table_expression_data = nullptr; + const auto * nearest_query_scope = scope.getNearestQueryScope(); + if (nearest_query_scope) + table_expression_data = &nearest_query_scope->getTableExpressionDataOrThrow(table_expression_node); + + QueryTreeNodes matched_column_nodes; + + for (const auto & column : matched_columns) + { + const auto & column_name = column.name; + if (!matcher_node_typed.isMatchingColumn(column_name)) + continue; + + if (table_expression_data) + { + auto column_node_it = table_expression_data->column_name_to_column_node.find(column_name); + if (column_node_it != table_expression_data->column_name_to_column_node.end()) + { + matched_column_nodes.emplace_back(column_node_it->second); + continue; + } + } + + matched_column_nodes.emplace_back(std::make_shared(column, table_expression_node)); + } + + qualifyColumnNodesWithProjectionNames(matched_column_nodes, table_expression_node, scope); + + QueryAnalyzer::QueryTreeNodesWithNames matched_column_nodes_with_names; + matched_column_nodes_with_names.reserve(matched_column_nodes.size()); + + for (auto && matched_column_node : matched_column_nodes) + { + auto column_name = matched_column_node->as().getColumnName(); + matched_column_nodes_with_names.emplace_back(std::move(matched_column_node), std::move(column_name)); + } + + return matched_column_nodes_with_names; +} + /** Resolve qualified tree matcher. * * First try to match qualified identifier to expression. If qualified identifier matched expression node then @@ -3277,8 +3382,6 @@ QueryAnalyzer::QueryTreeNodesWithNames QueryAnalyzer::resolveQualifiedMatcher(Qu auto & matcher_node_typed = matcher_node->as(); assert(matcher_node_typed.isQualified()); - QueryTreeNodesWithNames matched_expression_nodes_with_column_names; - auto expression_identifier_lookup = IdentifierLookup{matcher_node_typed.getQualifiedIdentifier(), IdentifierLookupContext::EXPRESSION}; auto expression_identifier_resolve_result = tryResolveIdentifier(expression_identifier_lookup, scope); auto expression_query_tree_node = expression_identifier_resolve_result.resolved_identifier; @@ -3302,6 +3405,7 @@ QueryAnalyzer::QueryTreeNodesWithNames QueryAnalyzer::resolveQualifiedMatcher(Qu scope.scope_node->formatASTForErrorMessage()); const auto & element_names = tuple_data_type->getElementNames(); + QueryTreeNodesWithNames matched_expression_nodes_with_column_names; auto qualified_matcher_element_identifier = matcher_node_typed.getQualifiedIdentifier(); for (const auto & element_name : element_names) @@ -3344,7 +3448,7 @@ QueryAnalyzer::QueryTreeNodesWithNames QueryAnalyzer::resolveQualifiedMatcher(Qu scope.scope_node->formatASTForErrorMessage()); } - NamesAndTypes initial_matcher_columns; + NamesAndTypes matched_columns; auto * table_expression_query_node = table_expression_node->as(); auto * table_expression_union_node = table_expression_node->as(); @@ -3353,15 +3457,16 @@ QueryAnalyzer::QueryTreeNodesWithNames QueryAnalyzer::resolveQualifiedMatcher(Qu if (table_expression_query_node || table_expression_union_node) { - initial_matcher_columns = table_expression_query_node ? table_expression_query_node->getProjectionColumns() + matched_columns = table_expression_query_node ? table_expression_query_node->getProjectionColumns() : table_expression_union_node->computeProjectionColumns(); } else if (table_expression_table_node || table_expression_table_function_node) { const auto & storage_snapshot = table_expression_table_node ? table_expression_table_node->getStorageSnapshot() : table_expression_table_function_node->getStorageSnapshot(); - auto storage_columns_list = storage_snapshot->getColumns(GetColumnsOptions(GetColumnsOptions::All)); - initial_matcher_columns = NamesAndTypes(storage_columns_list.begin(), storage_columns_list.end()); + auto get_columns_options = buildGetColumnsOptions(matcher_node, scope.context); + auto storage_columns_list = storage_snapshot->getColumns(get_columns_options); + matched_columns = NamesAndTypes(storage_columns_list.begin(), storage_columns_list.end()); } else { @@ -3371,19 +3476,14 @@ QueryAnalyzer::QueryTreeNodesWithNames QueryAnalyzer::resolveQualifiedMatcher(Qu scope.scope_node->formatASTForErrorMessage()); } - for (auto & column : initial_matcher_columns) - { - const auto & column_name = column.name; - if (matcher_node_typed.isMatchingColumn(column_name)) - matched_expression_nodes_with_column_names.emplace_back(std::make_shared(column, table_expression_node), column_name); - } + auto result_matched_column_nodes_with_names = getMatchedColumnNodesWithNames(matcher_node, + table_expression_node, + matched_columns, + scope); - qualifyMatchedColumnsProjectionNamesIfNeeded(matched_expression_nodes_with_column_names, table_expression_node, scope); - - return matched_expression_nodes_with_column_names; + return result_matched_column_nodes_with_names; } - /// Resolve non qualified matcher, using scope join tree node. QueryAnalyzer::QueryTreeNodesWithNames QueryAnalyzer::resolveUnqualifiedMatcher(QueryTreeNodePtr & matcher_node, IdentifierResolveScope & scope) { @@ -3421,8 +3521,6 @@ QueryAnalyzer::QueryTreeNodesWithNames QueryAnalyzer::resolveUnqualifiedMatcher( for (auto & table_expression : table_expressions_stack) { - QueryTreeNodesWithNames matched_expression_nodes_with_column_names; - if (auto * array_join_node = table_expression->as()) { if (table_expressions_column_nodes_with_names_stack.empty()) @@ -3486,6 +3584,8 @@ QueryAnalyzer::QueryTreeNodesWithNames QueryAnalyzer::resolveUnqualifiedMatcher( left_table_expression_column_names_to_skip.clear(); right_table_expression_column_names_to_skip.clear(); + QueryTreeNodesWithNames matched_expression_nodes_with_column_names; + /** If there is JOIN with USING we need to match only single USING column and do not use left table expression * and right table expression column with same name. * @@ -3544,37 +3644,37 @@ QueryAnalyzer::QueryTreeNodesWithNames QueryAnalyzer::resolveUnqualifiedMatcher( } } - for (auto && left_table_column : left_table_expression_columns) + for (auto && left_table_column_with_name : left_table_expression_columns) { - if (left_table_expression_column_names_to_skip.contains(left_table_column.second)) + if (left_table_expression_column_names_to_skip.contains(left_table_column_with_name.second)) continue; - matched_expression_nodes_with_column_names.push_back(std::move(left_table_column)); + matched_expression_nodes_with_column_names.push_back(std::move(left_table_column_with_name)); } - for (auto && right_table_column : right_table_expression_columns) + for (auto && right_table_column_with_name : right_table_expression_columns) { - if (right_table_expression_column_names_to_skip.contains(right_table_column.second)) + if (right_table_expression_column_names_to_skip.contains(right_table_column_with_name.second)) continue; - matched_expression_nodes_with_column_names.push_back(std::move(right_table_column)); + matched_expression_nodes_with_column_names.push_back(std::move(right_table_column_with_name)); } table_expressions_column_nodes_with_names_stack.push_back(std::move(matched_expression_nodes_with_column_names)); continue; } + if (table_expression_in_resolve_process) + { + table_expressions_column_nodes_with_names_stack.emplace_back(); + continue; + } + auto * table_node = table_expression->as(); auto * table_function_node = table_expression->as(); auto * query_node = table_expression->as(); auto * union_node = table_expression->as(); - if (table_expression_in_resolve_process) - { - table_expressions_column_nodes_with_names_stack.emplace_back(); - continue; - } - NamesAndTypes table_expression_columns; if (query_node || union_node) @@ -3585,27 +3685,7 @@ QueryAnalyzer::QueryTreeNodesWithNames QueryAnalyzer::resolveUnqualifiedMatcher( { const auto & storage_snapshot = table_node ? table_node->getStorageSnapshot() : table_function_node->getStorageSnapshot(); - - UInt8 get_column_options_kind = 0; - - if (matcher_node_typed.isAsteriskMatcher()) - { - get_column_options_kind = GetColumnsOptions::Ordinary; - const auto & settings = scope.context->getSettingsRef(); - - if (settings.asterisk_include_alias_columns) - get_column_options_kind |= GetColumnsOptions::Kind::Aliases; - - if (settings.asterisk_include_materialized_columns) - get_column_options_kind |= GetColumnsOptions::Kind::Materialized; - } - else - { - /// TODO: Check if COLUMNS select aliases column by default - get_column_options_kind = GetColumnsOptions::All; - } - - auto get_columns_options = GetColumnsOptions(static_cast(get_column_options_kind)); + auto get_columns_options = buildGetColumnsOptions(matcher_node, scope.context); auto storage_columns_list = storage_snapshot->getColumns(get_columns_options); table_expression_columns = NamesAndTypes(storage_columns_list.begin(), storage_columns_list.end()); } @@ -3617,25 +3697,12 @@ QueryAnalyzer::QueryTreeNodesWithNames QueryAnalyzer::resolveUnqualifiedMatcher( scope.scope_node->formatASTForErrorMessage()); } - for (auto & table_expression_column : table_expression_columns) - { - if (!matcher_node_typed.isMatchingColumn(table_expression_column.name)) - continue; + auto matched_column_nodes_with_names = getMatchedColumnNodesWithNames(matcher_node, + table_expression, + table_expression_columns, + scope); - auto matched_column_node = std::make_shared(table_expression_column, table_expression); - matched_expression_nodes_with_column_names.emplace_back(std::move(matched_column_node), table_expression_column.name); - } - - qualifyMatchedColumnsProjectionNamesIfNeeded(matched_expression_nodes_with_column_names, table_expression, scope); - - for (auto & [matched_node, column_name] : matched_expression_nodes_with_column_names) - { - auto node_projection_name_it = node_to_projection_name.find(matcher_node); - if (node_projection_name_it != node_to_projection_name.end()) - column_name = node_projection_name_it->second; - } - - table_expressions_column_nodes_with_names_stack.push_back(std::move(matched_expression_nodes_with_column_names)); + table_expressions_column_nodes_with_names_stack.push_back(std::move(matched_column_nodes_with_names)); } QueryTreeNodesWithNames result; @@ -5478,8 +5545,8 @@ void QueryAnalyzer::initializeTableExpressionColumns(const QueryTreeNodePtr & ta if (table_node) { const auto & table_storage_id = table_node->getStorageID(); - table_expression_data.table_name = table_storage_id.table_name; table_expression_data.database_name = table_storage_id.database_name; + table_expression_data.table_name = table_storage_id.table_name; table_expression_data.table_expression_name = table_storage_id.getFullNameNotQuoted(); table_expression_data.table_expression_description = "table"; } diff --git a/src/Storages/ColumnsDescription.cpp b/src/Storages/ColumnsDescription.cpp index ebbf81f1faa..d401840eec7 100644 --- a/src/Storages/ColumnsDescription.cpp +++ b/src/Storages/ColumnsDescription.cpp @@ -425,23 +425,49 @@ NamesAndTypesList ColumnsDescription::get(const GetColumnsOptions & options) con switch (options.kind) { case GetColumnsOptions::All: + { res = getAll(); break; + } + case GetColumnsOptions::AllPhysicalAndAliases: + { + res = getAllPhysical(); + auto aliases = getAliases(); + res.insert(res.end(), aliases.begin(), aliases.end()); + break; + } case GetColumnsOptions::AllPhysical: + { res = getAllPhysical(); break; + } + case GetColumnsOptions::OrdinaryAndAliases: + { + res = getOrdinary(); + auto aliases = getAliases(); + res.insert(res.end(), aliases.begin(), aliases.end()); + break; + } case GetColumnsOptions::Ordinary: + { res = getOrdinary(); break; + } case GetColumnsOptions::Materialized: + { res = getMaterialized(); break; + } case GetColumnsOptions::Aliases: + { res = getAliases(); break; + } case GetColumnsOptions::Ephemeral: + { res = getEphemeral(); break; + } } if (options.with_subcolumns) diff --git a/src/Storages/ColumnsDescription.h b/src/Storages/ColumnsDescription.h index 32c52bdcb9e..4f874f4b850 100644 --- a/src/Storages/ColumnsDescription.h +++ b/src/Storages/ColumnsDescription.h @@ -35,8 +35,9 @@ struct GetColumnsOptions Materialized = 2, Aliases = 4, Ephemeral = 8, - + OrdinaryAndAliases = Ordinary | Aliases, AllPhysical = Ordinary | Materialized, + AllPhysicalAndAliases = AllPhysical | Aliases, All = AllPhysical | Aliases | Ephemeral, }; diff --git a/tests/queries/0_stateless/02540_analyzer_matcher_alias_materialized_columns.reference b/tests/queries/0_stateless/02540_analyzer_matcher_alias_materialized_columns.reference new file mode 100644 index 00000000000..3e2a8094a96 --- /dev/null +++ b/tests/queries/0_stateless/02540_analyzer_matcher_alias_materialized_columns.reference @@ -0,0 +1,18 @@ +-- { echoOn } + +SELECT * FROM test_table AS test_table_alias; +0 +SELECT test_table_alias.* FROM test_table AS test_table_alias; +0 +SELECT * FROM test_table AS test_table_alias SETTINGS asterisk_include_alias_columns = 1; +0 AliasValue_0 +SELECT test_table_alias.* FROM test_table AS test_table_alias SETTINGS asterisk_include_alias_columns = 1; +0 AliasValue_0 +SELECT * FROM test_table AS test_table_alias SETTINGS asterisk_include_materialized_columns = 1; +0 MaterializedValue_0 +SELECT test_table_alias.* FROM test_table AS test_table_alias SETTINGS asterisk_include_materialized_columns = 1; +0 MaterializedValue_0 +SELECT * FROM test_table AS test_table_alias SETTINGS asterisk_include_alias_columns = 1, asterisk_include_materialized_columns = 1; +0 MaterializedValue_0 AliasValue_0 +SELECT test_table_alias.* FROM test_table AS test_table_alias SETTINGS asterisk_include_alias_columns = 1, asterisk_include_materialized_columns = 1; +0 MaterializedValue_0 AliasValue_0 diff --git a/tests/queries/0_stateless/02540_analyzer_matcher_alias_materialized_columns.sql b/tests/queries/0_stateless/02540_analyzer_matcher_alias_materialized_columns.sql new file mode 100644 index 00000000000..cc622dde8fe --- /dev/null +++ b/tests/queries/0_stateless/02540_analyzer_matcher_alias_materialized_columns.sql @@ -0,0 +1,33 @@ +SET allow_experimental_analyzer = 1; + +DROP TABLE IF EXISTS test_table; +CREATE TABLE test_table +( + id UInt64, + value_alias ALIAS concat('AliasValue_', toString(id)), + value_materialized MATERIALIZED concat('MaterializedValue_', toString(id)) +) ENGINE=MergeTree ORDER BY id; + +INSERT INTO test_table VALUES (0); + +-- { echoOn } + +SELECT * FROM test_table AS test_table_alias; + +SELECT test_table_alias.* FROM test_table AS test_table_alias; + +SELECT * FROM test_table AS test_table_alias SETTINGS asterisk_include_alias_columns = 1; + +SELECT test_table_alias.* FROM test_table AS test_table_alias SETTINGS asterisk_include_alias_columns = 1; + +SELECT * FROM test_table AS test_table_alias SETTINGS asterisk_include_materialized_columns = 1; + +SELECT test_table_alias.* FROM test_table AS test_table_alias SETTINGS asterisk_include_materialized_columns = 1; + +SELECT * FROM test_table AS test_table_alias SETTINGS asterisk_include_alias_columns = 1, asterisk_include_materialized_columns = 1; + +SELECT test_table_alias.* FROM test_table AS test_table_alias SETTINGS asterisk_include_alias_columns = 1, asterisk_include_materialized_columns = 1; + +-- { echoOff } + +DROP TABLE test_table; From 387f035560f3240c63238cefc2dbd89a5cfd6fd1 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sun, 29 Jan 2023 13:49:46 +0100 Subject: [PATCH 294/717] QueryTreePassManager add ComparisonTupleEliminationPass --- .../Passes/ComparisonTupleEliminationPass.cpp | 205 ++++++++++++++++++ .../Passes/ComparisonTupleEliminationPass.h | 24 ++ src/Analyzer/QueryTreePassManager.cpp | 3 + .../0_stateless/02030_tuple_filter.reference | 4 + .../0_stateless/02030_tuple_filter.sql | 12 +- 5 files changed, 242 insertions(+), 6 deletions(-) create mode 100644 src/Analyzer/Passes/ComparisonTupleEliminationPass.cpp create mode 100644 src/Analyzer/Passes/ComparisonTupleEliminationPass.h diff --git a/src/Analyzer/Passes/ComparisonTupleEliminationPass.cpp b/src/Analyzer/Passes/ComparisonTupleEliminationPass.cpp new file mode 100644 index 00000000000..29dec874e2c --- /dev/null +++ b/src/Analyzer/Passes/ComparisonTupleEliminationPass.cpp @@ -0,0 +1,205 @@ +#include + +#include + +#include + +#include + +#include +#include +#include + +namespace DB +{ + +namespace +{ + +class ComparisonTupleEliminationPassVisitor : public InDepthQueryTreeVisitor +{ +public: + explicit ComparisonTupleEliminationPassVisitor(ContextPtr context_) + : context(std::move(context_)) + {} + + void visitImpl(QueryTreeNodePtr & node) const + { + auto * function_node = node->as(); + if (!function_node) + return; + + const auto & comparison_function_name = function_node->getFunctionName(); + if (comparison_function_name != "equals" && comparison_function_name != "notEquals") + return; + + const auto & arguments = function_node->getArguments().getNodes(); + if (arguments.size() != 2) + return; + + const auto & lhs_argument = arguments[0]; + const auto & lhs_argument_result_type = lhs_argument->getResultType(); + if (!isTuple(lhs_argument_result_type)) + return; + + const auto & rhs_argument = arguments[1]; + const auto & rhs_argument_result_type = rhs_argument->getResultType(); + if (!isTuple(rhs_argument_result_type)) + return; + + auto lhs_argument_node_type = lhs_argument->getNodeType(); + auto rhs_argument_node_type = rhs_argument->getNodeType(); + + if (lhs_argument_node_type == QueryTreeNodeType::FUNCTION && rhs_argument_node_type == QueryTreeNodeType::FUNCTION) + tryOptimizeComparisonTupleFunctions(node, lhs_argument, rhs_argument, comparison_function_name); + else if (lhs_argument_node_type == QueryTreeNodeType::FUNCTION && rhs_argument_node_type == QueryTreeNodeType::CONSTANT) + tryOptimizeComparisonTupleFunctionAndConstant(node, lhs_argument, rhs_argument, comparison_function_name); + else if (lhs_argument_node_type == QueryTreeNodeType::CONSTANT && rhs_argument_node_type == QueryTreeNodeType::FUNCTION) + tryOptimizeComparisonTupleFunctionAndConstant(node, rhs_argument, lhs_argument, comparison_function_name); + } + +private: + void tryOptimizeComparisonTupleFunctions(QueryTreeNodePtr & node, + const QueryTreeNodePtr & lhs_function_node, + const QueryTreeNodePtr & rhs_function_node, + const std::string & comparison_function_name) const + { + const auto & lhs_function_node_typed = lhs_function_node->as(); + if (lhs_function_node_typed.getFunctionName() != "tuple") + return; + + const auto & rhs_function_node_typed = rhs_function_node->as(); + if (rhs_function_node_typed.getFunctionName() != "tuple") + return; + + const auto & lhs_tuple_function_arguments_nodes = lhs_function_node_typed.getArguments().getNodes(); + size_t lhs_tuple_function_arguments_nodes_size = lhs_tuple_function_arguments_nodes.size(); + + const auto & rhs_tuple_function_arguments_nodes = rhs_function_node_typed.getArguments().getNodes(); + if (lhs_tuple_function_arguments_nodes_size != rhs_tuple_function_arguments_nodes.size()) + return; + + if (lhs_tuple_function_arguments_nodes_size == 1) + { + node = makeComparisonFunction(lhs_tuple_function_arguments_nodes[0], rhs_tuple_function_arguments_nodes[0], comparison_function_name); + return; + } + + QueryTreeNodes tuple_arguments_equals_functions; + tuple_arguments_equals_functions.reserve(lhs_tuple_function_arguments_nodes_size); + + for (size_t i = 0; i < lhs_tuple_function_arguments_nodes_size; ++i) + { + auto equals_function = makeEqualsFunction(lhs_tuple_function_arguments_nodes[i], rhs_tuple_function_arguments_nodes[i]); + tuple_arguments_equals_functions.push_back(std::move(equals_function)); + } + + node = makeEquivalentTupleComparisonFunction(std::move(tuple_arguments_equals_functions), comparison_function_name); + } + + void tryOptimizeComparisonTupleFunctionAndConstant(QueryTreeNodePtr & node, + const QueryTreeNodePtr & function_node, + const QueryTreeNodePtr & constant_node, + const std::string & comparison_function_name) const + { + const auto & function_node_typed = function_node->as(); + if (function_node_typed.getFunctionName() != "tuple") + return; + + auto & constant_node_typed = constant_node->as(); + const auto & constant_node_value = constant_node_typed.getValue(); + if (constant_node_value.getType() != Field::Types::Which::Tuple) + return; + + const auto & constant_tuple = constant_node_value.get(); + + const auto & function_arguments_nodes = function_node_typed.getArguments().getNodes(); + size_t function_arguments_nodes_size = function_arguments_nodes.size(); + if (function_arguments_nodes_size != constant_tuple.size()) + return; + + auto constant_node_result_type = constant_node_typed.getResultType(); + const auto * tuple_data_type = typeid_cast(constant_node_result_type.get()); + if (!tuple_data_type) + return; + + const auto & tuple_data_type_elements = tuple_data_type->getElements(); + if (tuple_data_type_elements.size() != function_arguments_nodes_size) + return; + + if (function_arguments_nodes_size == 1) + { + auto comparison_argument_constant_value = std::make_shared(constant_tuple[0], tuple_data_type_elements[0]); + auto comparison_argument_constant_node = std::make_shared(std::move(comparison_argument_constant_value)); + node = makeComparisonFunction(function_arguments_nodes[0], std::move(comparison_argument_constant_node), comparison_function_name); + return; + } + + QueryTreeNodes tuple_arguments_equals_functions; + tuple_arguments_equals_functions.reserve(function_arguments_nodes_size); + + for (size_t i = 0; i < function_arguments_nodes_size; ++i) + { + auto equals_argument_constant_value = std::make_shared(constant_tuple[i], tuple_data_type_elements[i]); + auto equals_argument_constant_node = std::make_shared(std::move(equals_argument_constant_value)); + auto equals_function = makeEqualsFunction(function_arguments_nodes[i], std::move(equals_argument_constant_node)); + tuple_arguments_equals_functions.push_back(std::move(equals_function)); + } + + node = makeEquivalentTupleComparisonFunction(std::move(tuple_arguments_equals_functions), comparison_function_name); + } + + QueryTreeNodePtr makeEquivalentTupleComparisonFunction(QueryTreeNodes tuple_arguments_equals_functions, + const std::string & comparison_function_name) const + { + auto result_function = std::make_shared("and"); + result_function->getArguments().getNodes() = std::move(tuple_arguments_equals_functions); + resolveOrdinaryFunctionNode(*result_function, result_function->getFunctionName()); + + if (comparison_function_name == "notEquals") + { + auto not_function = std::make_shared("not"); + not_function->getArguments().getNodes().push_back(std::move(result_function)); + resolveOrdinaryFunctionNode(*not_function, not_function->getFunctionName()); + result_function = std::move(not_function); + } + + return result_function; + } + + inline QueryTreeNodePtr makeEqualsFunction(QueryTreeNodePtr lhs_argument, QueryTreeNodePtr rhs_argument) const + { + return makeComparisonFunction(std::move(lhs_argument), std::move(rhs_argument), "equals"); + } + + QueryTreeNodePtr makeComparisonFunction(QueryTreeNodePtr lhs_argument, + QueryTreeNodePtr rhs_argument, + const std::string & comparison_function_name) const + { + auto comparison_function = std::make_shared(comparison_function_name); + comparison_function->getArguments().getNodes().push_back(std::move(lhs_argument)); + comparison_function->getArguments().getNodes().push_back(std::move(rhs_argument)); + + resolveOrdinaryFunctionNode(*comparison_function, comparison_function->getFunctionName()); + + return comparison_function; + } + + void resolveOrdinaryFunctionNode(FunctionNode & function_node, const String & function_name) const + { + auto function = FunctionFactory::instance().get(function_name, context); + function_node.resolveAsFunction(function->build(function_node.getArgumentColumns())); + } + + ContextPtr context; +}; + +} + +void ComparisonTupleEliminationPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context) +{ + ComparisonTupleEliminationPassVisitor visitor(std::move(context)); + visitor.visit(query_tree_node); +} + +} diff --git a/src/Analyzer/Passes/ComparisonTupleEliminationPass.h b/src/Analyzer/Passes/ComparisonTupleEliminationPass.h new file mode 100644 index 00000000000..954a9d6a2f0 --- /dev/null +++ b/src/Analyzer/Passes/ComparisonTupleEliminationPass.h @@ -0,0 +1,24 @@ +#pragma once + +#include + +namespace DB +{ + +/** Rewrite tuples comparison into equivalent comparison of tuples arguments. + * + * Example: SELECT id FROM test_table WHERE (id, value) = (1, 'Value'); + * Result: SELECT id FROM test_table WHERE id = 1 AND value = 'Value'; + */ +class ComparisonTupleEliminationPass final : public IQueryTreePass +{ +public: + String getName() override { return "ComparisonTupleEliminationPass"; } + + String getDescription() override { return "Rewrite tuples comparison into equivalent comparison of tuples arguments"; } + + void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override; + +}; + +} diff --git a/src/Analyzer/QueryTreePassManager.cpp b/src/Analyzer/QueryTreePassManager.cpp index de67f5c2d49..ee68d58bb33 100644 --- a/src/Analyzer/QueryTreePassManager.cpp +++ b/src/Analyzer/QueryTreePassManager.cpp @@ -37,6 +37,7 @@ #include #include #include +#include namespace DB { @@ -248,6 +249,8 @@ void addQueryTreePasses(QueryTreePassManager & manager) manager.addPass(std::make_unique()); manager.addPass(std::make_unique()); + manager.addPass(std::make_unique()); + manager.addPass(std::make_unique()); manager.addPass(std::make_unique()); diff --git a/tests/queries/0_stateless/02030_tuple_filter.reference b/tests/queries/0_stateless/02030_tuple_filter.reference index a2e1290c0e1..50e3b2928b3 100644 --- a/tests/queries/0_stateless/02030_tuple_filter.reference +++ b/tests/queries/0_stateless/02030_tuple_filter.reference @@ -6,3 +6,7 @@ 1 A 2021-01-01 1 A 2021-01-01 1 A 2021-01-01 +1 A 2021-01-01 +1 A 2021-01-01 +1 A 2021-01-01 +1 A 2021-01-01 diff --git a/tests/queries/0_stateless/02030_tuple_filter.sql b/tests/queries/0_stateless/02030_tuple_filter.sql index c19f538b8e1..d2a114a89f9 100644 --- a/tests/queries/0_stateless/02030_tuple_filter.sql +++ b/tests/queries/0_stateless/02030_tuple_filter.sql @@ -1,3 +1,5 @@ +SET allow_experimental_analyzer = 1; + DROP TABLE IF EXISTS test_tuple_filter; CREATE TABLE test_tuple_filter (id UInt32, value String, log_date Date) Engine=MergeTree() ORDER BY id PARTITION BY log_date SETTINGS index_granularity = 3; @@ -16,13 +18,11 @@ SELECT * FROM test_tuple_filter WHERE ((id, value), log_date) = ((1, 'A'), '2021 -- not supported functions (concat) do not lost SELECT * FROM test_tuple_filter WHERE (id, value, value||'foo') = ('1', 'A', 'A'); --- Condition fully moved to PREWHERE and such conditions does not supported yet. -SELECT * FROM test_tuple_filter WHERE (1, (1, (1, (1, (id, value))))) = (1, (1, (1, (1, (1, 'A'))))); -- { serverError INDEX_NOT_USED } +SELECT * FROM test_tuple_filter WHERE (1, (1, (1, (1, (id, value))))) = (1, (1, (1, (1, (1, 'A'))))); --- not implemented yet -SELECT * FROM test_tuple_filter WHERE (1, value) = (id, 'A'); -- { serverError INDEX_NOT_USED } -SELECT * FROM test_tuple_filter WHERE (1, (1, (1, (1, tuple(id))))) = (1, (1, (1, (1, tuple(1))))); -- { serverError INDEX_NOT_USED } -SELECT * FROM test_tuple_filter WHERE ((id, value), tuple(log_date)) = ((1, 'A'), tuple('2021-01-01')); -- { serverError INDEX_NOT_USED } +SELECT * FROM test_tuple_filter WHERE (1, value) = (id, 'A'); +SELECT * FROM test_tuple_filter WHERE (1, (1, (1, (1, tuple(id))))) = (1, (1, (1, (1, tuple(1))))); +SELECT * FROM test_tuple_filter WHERE ((id, value), tuple(log_date)) = ((1, 'A'), tuple('2021-01-01')); SET force_index_by_date = 1; SET force_primary_key = 0; From caaff8f3878513277e362e3d9f210a0f059eceb7 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sun, 29 Jan 2023 15:36:29 +0100 Subject: [PATCH 295/717] GroupingFunctionsResolvePass crash fix --- src/Analyzer/Passes/GroupingFunctionsResolvePass.cpp | 5 +++-- src/Analyzer/Passes/QueryAnalysisPass.cpp | 2 +- .../02541_analyzer_grouping_sets_crash_fix.reference | 2 ++ .../0_stateless/02541_analyzer_grouping_sets_crash_fix.sql | 5 +++++ 4 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/02541_analyzer_grouping_sets_crash_fix.reference create mode 100644 tests/queries/0_stateless/02541_analyzer_grouping_sets_crash_fix.sql diff --git a/src/Analyzer/Passes/GroupingFunctionsResolvePass.cpp b/src/Analyzer/Passes/GroupingFunctionsResolvePass.cpp index 55152fccee9..20f1713f8c2 100644 --- a/src/Analyzer/Passes/GroupingFunctionsResolvePass.cpp +++ b/src/Analyzer/Passes/GroupingFunctionsResolvePass.cpp @@ -149,8 +149,9 @@ void resolveGroupingFunctions(QueryTreeNodePtr & query_node, ContextPtr context) /// It is expected by execution layer that if there are only 1 grouping set it will be removed if (query_node_typed.isGroupByWithGroupingSets() && query_node_typed.getGroupBy().getNodes().size() == 1) { - auto & grouping_set_list_node = query_node_typed.getGroupBy().getNodes().front()->as(); - query_node_typed.getGroupBy().getNodes() = std::move(grouping_set_list_node.getNodes()); + auto grouping_set_list_node = query_node_typed.getGroupBy().getNodes().front(); + auto & grouping_set_list_node_typed = grouping_set_list_node->as(); + query_node_typed.getGroupBy().getNodes() = std::move(grouping_set_list_node_typed.getNodes()); query_node_typed.setIsGroupByWithGroupingSets(false); } diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index df93696c540..387c472c948 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -1804,7 +1804,7 @@ void QueryAnalyzer::evaluateScalarSubqueryIfNeeded(QueryTreeNodePtr & node, Iden subquery_context->setSettings(subquery_settings); auto options = SelectQueryOptions(QueryProcessingStage::Complete, scope.subquery_depth, true /*is_subquery*/); - auto interpreter = std::make_unique(node, subquery_context, options); + auto interpreter = std::make_unique(node->toAST(), subquery_context, options); auto io = interpreter->execute(); diff --git a/tests/queries/0_stateless/02541_analyzer_grouping_sets_crash_fix.reference b/tests/queries/0_stateless/02541_analyzer_grouping_sets_crash_fix.reference new file mode 100644 index 00000000000..57782bd7c09 --- /dev/null +++ b/tests/queries/0_stateless/02541_analyzer_grouping_sets_crash_fix.reference @@ -0,0 +1,2 @@ +\N +2 diff --git a/tests/queries/0_stateless/02541_analyzer_grouping_sets_crash_fix.sql b/tests/queries/0_stateless/02541_analyzer_grouping_sets_crash_fix.sql new file mode 100644 index 00000000000..d7af475bbe7 --- /dev/null +++ b/tests/queries/0_stateless/02541_analyzer_grouping_sets_crash_fix.sql @@ -0,0 +1,5 @@ +SET allow_experimental_analyzer = 1; + +WITH pow(NULL, 256) AS four SELECT NULL AS two GROUP BY GROUPING SETS ((pow(two, 65536))); + +WITH (SELECT pow(two, 1) GROUP BY GROUPING SETS ((pow(1, 9)))) AS four SELECT 2 AS two GROUP BY pow(1, two); From 70eb41c6a8b2adc13b0f6d1b790d493e52b96ebe Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 31 Jan 2023 14:54:04 +0100 Subject: [PATCH 296/717] Analyzer add single_join_prefer_left_table setting --- src/Analyzer/IQueryTreeNode.cpp | 8 +- src/Analyzer/IQueryTreeNode.h | 9 +- src/Analyzer/Passes/QueryAnalysisPass.cpp | 90 ++++++++++++++----- src/Analyzer/QueryNode.cpp | 19 +++- src/Core/Settings.h | 1 + src/Interpreters/ActionsDAG.cpp | 4 +- src/Interpreters/ActionsDAG.h | 2 +- src/Planner/Planner.cpp | 6 +- src/Planner/PlannerJoinTree.cpp | 2 +- src/Planner/PlannerJoins.cpp | 4 +- .../0_stateless/02371_analyzer_join_cross.sql | 1 + .../0_stateless/02372_analyzer_join.sql.j2 | 1 + ...analyzer_compound_expression_crash_fix.sql | 2 +- .../02495_analyzer_storage_join.sql | 1 + 14 files changed, 112 insertions(+), 38 deletions(-) diff --git a/src/Analyzer/IQueryTreeNode.cpp b/src/Analyzer/IQueryTreeNode.cpp index df841f9a61a..2b6fd6d8fa6 100644 --- a/src/Analyzer/IQueryTreeNode.cpp +++ b/src/Analyzer/IQueryTreeNode.cpp @@ -74,7 +74,7 @@ struct NodePairHash } -bool IQueryTreeNode::isEqual(const IQueryTreeNode & rhs) const +bool IQueryTreeNode::isEqual(const IQueryTreeNode & rhs, CompareOptions compare_options) const { if (this == &rhs) return true; @@ -105,11 +105,11 @@ bool IQueryTreeNode::isEqual(const IQueryTreeNode & rhs) const } if (lhs_node_to_compare->getNodeType() != rhs_node_to_compare->getNodeType() || - lhs_node_to_compare->alias != rhs_node_to_compare->alias || !lhs_node_to_compare->isEqualImpl(*rhs_node_to_compare)) - { return false; - } + + if (compare_options.compare_aliases && lhs_node_to_compare->alias != rhs_node_to_compare->alias) + return false; const auto & lhs_children = lhs_node_to_compare->children; const auto & rhs_children = rhs_node_to_compare->children; diff --git a/src/Analyzer/IQueryTreeNode.h b/src/Analyzer/IQueryTreeNode.h index 8aa834e60b7..0f0614aa39a 100644 --- a/src/Analyzer/IQueryTreeNode.h +++ b/src/Analyzer/IQueryTreeNode.h @@ -90,12 +90,17 @@ public: throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Method getResultType is not supported for {} query node", getNodeTypeName()); } + struct CompareOptions + { + bool compare_aliases = true; + }; + /** Is tree equal to other tree with node root. * - * Aliases of query tree nodes are compared during isEqual call. + * With default compare options aliases of query tree nodes are compared during isEqual call. * Original ASTs of query tree nodes are not compared during isEqual call. */ - bool isEqual(const IQueryTreeNode & rhs) const; + bool isEqual(const IQueryTreeNode & rhs, CompareOptions compare_options = {true}) const; using Hash = std::pair; using HashState = SipHash; diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 387c472c948..f958c5a5f5e 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -1,5 +1,6 @@ #include +#include #include #include @@ -77,8 +78,6 @@ namespace ProfileEvents extern const Event ScalarSubqueriesCacheMiss; } -#include - namespace DB { @@ -407,6 +406,7 @@ struct TableExpressionData std::string table_expression_description; std::string database_name; std::string table_name; + NamesAndTypes column_names_and_types; ColumnNameToColumnNodeMap column_name_to_column_node; std::unordered_set> column_identifier_first_parts; @@ -724,6 +724,9 @@ struct IdentifierResolveScope /// Use identifier lookup to result cache bool use_identifier_lookup_to_result_cache = true; + /// JOINs count + size_t joins_count = 0; + /// Subquery depth size_t subquery_depth = 0; @@ -1166,25 +1169,38 @@ private: static bool tryBindIdentifierToAliases(const IdentifierLookup & identifier_lookup, const IdentifierResolveScope & scope); - QueryTreeNodePtr tryResolveIdentifierFromAliases(const IdentifierLookup & identifier_lookup, IdentifierResolveScope & scope, IdentifierResolveSettings identifier_resolve_settings = {}); + QueryTreeNodePtr tryResolveIdentifierFromAliases(const IdentifierLookup & identifier_lookup, + IdentifierResolveScope & scope, + IdentifierResolveSettings identifier_resolve_settings); QueryTreeNodePtr tryResolveIdentifierFromTableColumns(const IdentifierLookup & identifier_lookup, IdentifierResolveScope & scope); static bool tryBindIdentifierToTableExpression(const IdentifierLookup & identifier_lookup, const QueryTreeNodePtr & table_expression_node, const IdentifierResolveScope & scope); - QueryTreeNodePtr tryResolveIdentifierFromTableExpression(const IdentifierLookup & identifier_lookup, const QueryTreeNodePtr & table_expression_node, IdentifierResolveScope & scope); + QueryTreeNodePtr tryResolveIdentifierFromTableExpression(const IdentifierLookup & identifier_lookup, + const QueryTreeNodePtr & table_expression_node, + IdentifierResolveScope & scope); - QueryTreeNodePtr tryResolveIdentifierFromJoin(const IdentifierLookup & identifier_lookup, const QueryTreeNodePtr & table_expression_node, IdentifierResolveScope & scope); + QueryTreeNodePtr tryResolveIdentifierFromJoin(const IdentifierLookup & identifier_lookup, + const QueryTreeNodePtr & table_expression_node, + IdentifierResolveScope & scope); - QueryTreeNodePtr tryResolveIdentifierFromArrayJoin(const IdentifierLookup & identifier_lookup, const QueryTreeNodePtr & table_expression_node, IdentifierResolveScope & scope); + QueryTreeNodePtr tryResolveIdentifierFromArrayJoin(const IdentifierLookup & identifier_lookup, + const QueryTreeNodePtr & table_expression_node, + IdentifierResolveScope & scope); - QueryTreeNodePtr tryResolveIdentifierFromJoinTreeNode(const IdentifierLookup & identifier_lookup, const QueryTreeNodePtr & join_tree_node, IdentifierResolveScope & scope); + QueryTreeNodePtr tryResolveIdentifierFromJoinTreeNode(const IdentifierLookup & identifier_lookup, + const QueryTreeNodePtr & join_tree_node, + IdentifierResolveScope & scope); - QueryTreeNodePtr tryResolveIdentifierFromJoinTree(const IdentifierLookup & identifier_lookup, IdentifierResolveScope & scope); + QueryTreeNodePtr tryResolveIdentifierFromJoinTree(const IdentifierLookup & identifier_lookup, + IdentifierResolveScope & scope); IdentifierResolveResult tryResolveIdentifierInParentScopes(const IdentifierLookup & identifier_lookup, IdentifierResolveScope & scope); - IdentifierResolveResult tryResolveIdentifier(const IdentifierLookup & identifier_lookup, IdentifierResolveScope & scope, IdentifierResolveSettings identifier_resolve_settings = {}); + IdentifierResolveResult tryResolveIdentifier(const IdentifierLookup & identifier_lookup, + IdentifierResolveScope & scope, + IdentifierResolveSettings identifier_resolve_settings = {}); /// Resolve query tree nodes functions @@ -2360,7 +2376,9 @@ bool QueryAnalyzer::tryBindIdentifierToAliases(const IdentifierLookup & identifi * * 5. If identifier is compound and identifier lookup is in expression context, use `tryResolveIdentifierFromCompoundExpression`. */ -QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromAliases(const IdentifierLookup & identifier_lookup, IdentifierResolveScope & scope, IdentifierResolveSettings identifier_resolve_settings) +QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromAliases(const IdentifierLookup & identifier_lookup, + IdentifierResolveScope & scope, + IdentifierResolveSettings identifier_resolve_settings) { const auto & identifier_bind_part = identifier_lookup.identifier.front(); @@ -2568,7 +2586,9 @@ bool QueryAnalyzer::tryBindIdentifierToTableExpression(const IdentifierLookup & return false; } -QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromTableExpression(const IdentifierLookup & identifier_lookup, const QueryTreeNodePtr & table_expression_node, IdentifierResolveScope & scope) +QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromTableExpression(const IdentifierLookup & identifier_lookup, + const QueryTreeNodePtr & table_expression_node, + IdentifierResolveScope & scope) { auto table_expression_node_type = table_expression_node->getNodeType(); @@ -2648,7 +2668,11 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromTableExpression(const Id if (result_column && !match_full_identifier && compound_identifier) { size_t identifier_bind_size = identifier_column_qualifier_parts + 1; - result_expression = tryResolveIdentifierFromCompoundExpression(identifier_lookup.identifier, identifier_bind_size, result_column, table_expression_source, scope); + result_expression = tryResolveIdentifierFromCompoundExpression(identifier_lookup.identifier, + identifier_bind_size, + result_column, + table_expression_source, + scope); clone_is_needed = false; } @@ -2662,7 +2686,7 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromTableExpression(const Id auto hints = collectIdentifierTypoHints(identifier, valid_identifiers); - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Identifier '{}' cannot be resolved from {}. In scope {}{}", + throw Exception(ErrorCodes::UNKNOWN_IDENTIFIER, "Identifier '{}' cannot be resolved from {}. In scope {}{}", identifier.getFullName(), table_expression_source, scope.scope_node->formatASTForErrorMessage(), @@ -2744,7 +2768,9 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromTableExpression(const Id return {}; } -QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromJoin(const IdentifierLookup & identifier_lookup, const QueryTreeNodePtr & table_expression_node, IdentifierResolveScope & scope) +QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromJoin(const IdentifierLookup & identifier_lookup, + const QueryTreeNodePtr & table_expression_node, + IdentifierResolveScope & scope) { const auto & from_join_node = table_expression_node->as(); auto left_resolved_identifier = tryResolveIdentifierFromJoinTreeNode(identifier_lookup, from_join_node.getLeftTableExpression(), scope); @@ -2804,6 +2830,14 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromJoin(const IdentifierLoo resolved_identifier = std::move(result_column_node); } + else if (scope.joins_count == 1 && scope.context->getSettingsRef().single_join_prefer_left_table) + { + return left_resolved_identifier; + } + else if (left_resolved_identifier->isEqual(*right_resolved_identifier, IQueryTreeNode::CompareOptions{.compare_aliases = false})) + { + return left_resolved_identifier; + } else { throw Exception(ErrorCodes::AMBIGUOUS_IDENTIFIER, @@ -2870,12 +2904,14 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromJoin(const IdentifierLoo return resolved_identifier; } -QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromArrayJoin(const IdentifierLookup & identifier_lookup, const QueryTreeNodePtr & table_expression_node, IdentifierResolveScope & scope) +QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromArrayJoin(const IdentifierLookup & identifier_lookup, + const QueryTreeNodePtr & table_expression_node, + IdentifierResolveScope & scope) { const auto & from_array_join_node = table_expression_node->as(); auto resolved_identifier = tryResolveIdentifierFromJoinTreeNode(identifier_lookup, from_array_join_node.getTableExpression(), scope); - if (scope.table_expressions_in_resolve_process.contains(table_expression_node.get())) + if (scope.table_expressions_in_resolve_process.contains(table_expression_node.get()) || !identifier_lookup.isExpressionLookup()) return resolved_identifier; const auto & array_join_column_expressions = from_array_join_node.getJoinExpressions(); @@ -2924,7 +2960,9 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromArrayJoin(const Identifi return resolved_identifier; } -QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromJoinTreeNode(const IdentifierLookup & identifier_lookup, const QueryTreeNodePtr & join_tree_node, IdentifierResolveScope & scope) +QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromJoinTreeNode(const IdentifierLookup & identifier_lookup, + const QueryTreeNodePtr & join_tree_node, + IdentifierResolveScope & scope) { auto join_tree_node_type = join_tree_node->getNodeType(); @@ -2973,7 +3011,8 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromJoinTreeNode(const Ident * Start with identifier first part, if it match some column name in table try to get column with full identifier name. * TODO: Need to check if it is okay to throw exception if compound identifier first part bind to column but column is not valid. */ -QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromJoinTree(const IdentifierLookup & identifier_lookup, IdentifierResolveScope & scope) +QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromJoinTree(const IdentifierLookup & identifier_lookup, + IdentifierResolveScope & scope) { if (identifier_lookup.isFunctionLookup()) return {}; @@ -3117,7 +3156,9 @@ IdentifierResolveResult QueryAnalyzer::tryResolveIdentifierInParentScopes(const * Example: Try to lookup identifier as expression, if it is not found, lookup as function. * Example: Try to lookup identifier as expression, if it is not found, lookup as table. */ -IdentifierResolveResult QueryAnalyzer::tryResolveIdentifier(const IdentifierLookup & identifier_lookup, IdentifierResolveScope & scope, IdentifierResolveSettings identifier_resolve_settings) +IdentifierResolveResult QueryAnalyzer::tryResolveIdentifier(const IdentifierLookup & identifier_lookup, + IdentifierResolveScope & scope, + IdentifierResolveSettings identifier_resolve_settings) { auto it = scope.identifier_lookup_to_result.find(identifier_lookup); if (it != scope.identifier_lookup_to_result.end()) @@ -5508,6 +5549,7 @@ void QueryAnalyzer::initializeQueryJoinTreeNode(QueryTreeNodePtr & join_tree_nod join_tree_node_ptrs_to_process_queue.push_back(&join.getLeftTableExpression()); join_tree_node_ptrs_to_process_queue.push_back(&join.getRightTableExpression()); scope.table_expressions_in_resolve_process.insert(current_join_tree_node.get()); + ++scope.joins_count; break; } default: @@ -5572,6 +5614,8 @@ void QueryAnalyzer::initializeTableExpressionColumns(const QueryTreeNodePtr & ta get_column_options.withSubcolumns(); auto column_names_and_types = storage_snapshot->getColumns(get_column_options); + table_expression_data.column_names_and_types = NamesAndTypes(column_names_and_types.begin(), column_names_and_types.end()); + const auto & columns_description = storage_snapshot->metadata->getColumns(); std::vector> alias_columns_to_resolve; @@ -5585,7 +5629,7 @@ void QueryAnalyzer::initializeTableExpressionColumns(const QueryTreeNodePtr & ta * For each alias column we build identifier resolve scope, initialize it with table column name to node map * and resolve alias column. */ - for (const auto & column_name_and_type : column_names_and_types) + for (const auto & column_name_and_type : table_expression_data.column_names_and_types) { const auto & column_default = columns_description.getDefault(column_name_and_type.name); @@ -5632,10 +5676,10 @@ void QueryAnalyzer::initializeTableExpressionColumns(const QueryTreeNodePtr & ta } else if (query_node || union_node) { - auto column_names_and_types = query_node ? query_node->getProjectionColumns() : union_node->computeProjectionColumns(); - table_expression_data.column_name_to_column_node.reserve(column_names_and_types.size()); + table_expression_data.column_names_and_types = query_node ? query_node->getProjectionColumns() : union_node->computeProjectionColumns(); + table_expression_data.column_name_to_column_node.reserve(table_expression_data.column_names_and_types.size()); - for (const auto & column_name_and_type : column_names_and_types) + for (const auto & column_name_and_type : table_expression_data.column_names_and_types) { auto column_node = std::make_shared(column_name_and_type, table_expression_node); table_expression_data.column_name_to_column_node.emplace(column_name_and_type.name, column_node); diff --git a/src/Analyzer/QueryNode.cpp b/src/Analyzer/QueryNode.cpp index 91fb3a1851d..be773582745 100644 --- a/src/Analyzer/QueryNode.cpp +++ b/src/Analyzer/QueryNode.cpp @@ -268,7 +268,24 @@ ASTPtr QueryNode::toASTImpl() const if (hasWith()) select_query->setExpression(ASTSelectQuery::Expression::WITH, getWith().toAST()); - select_query->setExpression(ASTSelectQuery::Expression::SELECT, getProjection().toAST()); + auto projection_ast = getProjection().toAST(); + auto & projection_expression_list_ast = projection_ast->as(); + size_t projection_expression_list_ast_children_size = projection_expression_list_ast.children.size(); + if (projection_expression_list_ast_children_size != getProjection().getNodes().size()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Query node invalid projection conversion to AST"); + + if (!projection_columns.empty()) + { + for (size_t i = 0; i < projection_expression_list_ast_children_size; ++i) + { + auto * ast_with_alias = dynamic_cast(projection_expression_list_ast.children[i].get()); + + if (ast_with_alias) + ast_with_alias->setAlias(projection_columns[i].name); + } + } + + select_query->setExpression(ASTSelectQuery::Expression::SELECT, std::move(projection_ast)); ASTPtr tables_in_select_query_ast = std::make_shared(); addTableExpressionOrJoinIntoTablesInSelectQuery(tables_in_select_query_ast, getJoinTree()); diff --git a/src/Core/Settings.h b/src/Core/Settings.h index e9db3e5ed06..25eb1d4a75a 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -257,6 +257,7 @@ class IColumn; \ M(JoinStrictness, join_default_strictness, JoinStrictness::All, "Set default strictness in JOIN query. Possible values: empty string, 'ANY', 'ALL'. If empty, query without strictness will throw exception.", 0) \ M(Bool, any_join_distinct_right_table_keys, false, "Enable old ANY JOIN logic with many-to-one left-to-right table keys mapping for all ANY JOINs. It leads to confusing not equal results for 't1 ANY LEFT JOIN t2' and 't2 ANY RIGHT JOIN t1'. ANY RIGHT JOIN needs one-to-many keys mapping to be consistent with LEFT one.", IMPORTANT) \ + M(Bool, single_join_prefer_left_table, true, "For single JOIN in case of identifier ambiguity prefer left table", IMPORTANT) \ \ M(UInt64, preferred_block_size_bytes, 1000000, "This setting adjusts the data block size for query processing and represents additional fine tune to the more rough 'max_block_size' setting. If the columns are large and with 'max_block_size' rows the block size is likely to be larger than the specified amount of bytes, its size will be lowered for better CPU cache locality.", 0) \ \ diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index 5f1398fed39..6fe394e9785 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -217,7 +217,7 @@ const ActionsDAG::Node & ActionsDAG::addFunction( all_const); } -const ActionsDAG::Node & ActionsDAG::addCast(const Node & node_to_cast, const DataTypePtr & cast_type) +const ActionsDAG::Node & ActionsDAG::addCast(const Node & node_to_cast, const DataTypePtr & cast_type, std::string result_name) { Field cast_type_constant_value(cast_type->getName()); @@ -230,7 +230,7 @@ const ActionsDAG::Node & ActionsDAG::addCast(const Node & node_to_cast, const Da ActionsDAG::NodeRawConstPtrs children = {&node_to_cast, cast_type_constant_node}; FunctionOverloadResolverPtr func_builder_cast = CastInternalOverloadResolver::createImpl(); - return addFunction(func_builder_cast, std::move(children), node_to_cast.result_name); + return addFunction(func_builder_cast, std::move(children), result_name); } const ActionsDAG::Node & ActionsDAG::addFunctionImpl( diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index 40bc76fe057..80a165a6462 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -143,7 +143,7 @@ public: const FunctionBasePtr & function_base, NodeRawConstPtrs children, std::string result_name); - const Node & addCast(const Node & node_to_cast, const DataTypePtr & cast_type); + const Node & addCast(const Node & node_to_cast, const DataTypePtr & cast_type, std::string result_name); /// Find first column by name in output nodes. This search is linear. const Node & findInOutputs(const std::string & name) const; diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp index 6bf51ee9cf3..71c74448c3a 100644 --- a/src/Planner/Planner.cpp +++ b/src/Planner/Planner.cpp @@ -591,7 +591,11 @@ void addWithFillStepIfNeeded(QueryPlan & query_plan, const auto * interpolate_expression = interpolate_expression_nodes[0]; if (!interpolate_expression->result_type->equals(*expression_to_interpolate->result_type)) - interpolate_expression = &interpolate_actions_dag->addCast(*interpolate_expression, expression_to_interpolate->result_type); + { + interpolate_expression = &interpolate_actions_dag->addCast(*interpolate_expression, + expression_to_interpolate->result_type, + interpolate_expression->result_name); + } const auto * alias_node = &interpolate_actions_dag->addAlias(*interpolate_expression, expression_to_interpolate_name); interpolate_actions_dag->getOutputs().push_back(alias_node); diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index a66145074f1..4f94f2419e1 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -475,7 +475,7 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_ continue; const auto & cast_type = it->second; - output_node = &cast_actions_dag->addCast(*output_node, cast_type); + output_node = &cast_actions_dag->addCast(*output_node, cast_type, output_node->result_name); } cast_actions_dag->projectInput(); diff --git a/src/Planner/PlannerJoins.cpp b/src/Planner/PlannerJoins.cpp index 4546e84a02b..8d87ebd466b 100644 --- a/src/Planner/PlannerJoins.cpp +++ b/src/Planner/PlannerJoins.cpp @@ -464,10 +464,10 @@ JoinClausesAndActions buildJoinClausesAndActions(const ColumnsWithTypeAndName & } if (!left_key_node->result_type->equals(*common_type)) - left_key_node = &join_expression_actions->addCast(*left_key_node, common_type); + left_key_node = &join_expression_actions->addCast(*left_key_node, common_type, {}); if (!right_key_node->result_type->equals(*common_type)) - right_key_node = &join_expression_actions->addCast(*right_key_node, common_type); + right_key_node = &join_expression_actions->addCast(*right_key_node, common_type, {}); } join_expression_actions->addOrReplaceInOutputs(*left_key_node); diff --git a/tests/queries/0_stateless/02371_analyzer_join_cross.sql b/tests/queries/0_stateless/02371_analyzer_join_cross.sql index 8261572cdf2..17388de68ab 100644 --- a/tests/queries/0_stateless/02371_analyzer_join_cross.sql +++ b/tests/queries/0_stateless/02371_analyzer_join_cross.sql @@ -1,4 +1,5 @@ SET allow_experimental_analyzer = 1; +SET single_join_prefer_left_table = 0; DROP TABLE IF EXISTS test_table_join_1; CREATE TABLE test_table_join_1 diff --git a/tests/queries/0_stateless/02372_analyzer_join.sql.j2 b/tests/queries/0_stateless/02372_analyzer_join.sql.j2 index 9b3c212562b..f6032a96b33 100644 --- a/tests/queries/0_stateless/02372_analyzer_join.sql.j2 +++ b/tests/queries/0_stateless/02372_analyzer_join.sql.j2 @@ -1,6 +1,7 @@ -- Tags: long SET allow_experimental_analyzer = 1; +SET single_join_prefer_left_table = 0; DROP TABLE IF EXISTS test_table_join_1; CREATE TABLE test_table_join_1 diff --git a/tests/queries/0_stateless/02494_analyzer_compound_expression_crash_fix.sql b/tests/queries/0_stateless/02494_analyzer_compound_expression_crash_fix.sql index 2af556ce9ab..4eef7792180 100644 --- a/tests/queries/0_stateless/02494_analyzer_compound_expression_crash_fix.sql +++ b/tests/queries/0_stateless/02494_analyzer_compound_expression_crash_fix.sql @@ -11,6 +11,6 @@ INSERT INTO test_table VALUES (0, [[1]], ['1']); SELECT fields.name FROM (SELECT fields.name FROM test_table); -SELECT fields.name, fields.value FROM (SELECT fields.name FROM test_table); -- { serverError 36 } +SELECT fields.name, fields.value FROM (SELECT fields.name FROM test_table); -- { serverError 47 } DROP TABLE IF EXISTS test_table; diff --git a/tests/queries/0_stateless/02495_analyzer_storage_join.sql b/tests/queries/0_stateless/02495_analyzer_storage_join.sql index 6a4c1e45d69..aeab15862c3 100644 --- a/tests/queries/0_stateless/02495_analyzer_storage_join.sql +++ b/tests/queries/0_stateless/02495_analyzer_storage_join.sql @@ -3,6 +3,7 @@ DROP TABLE IF EXISTS t1; DROP TABLE IF EXISTS tj; SET allow_experimental_analyzer = 1; +SET single_join_prefer_left_table = 0; CREATE TABLE tj (key2 UInt64, key1 Int64, a UInt64, b UInt64, x UInt64, y UInt64) ENGINE = Join(ALL, RIGHT, key1, key2); INSERT INTO tj VALUES (2, -2, 20, 200, 2000, 20000), (3, -3, 30, 300, 3000, 30000), (4, -4, 40, 400, 4000, 40000), (5, -5, 50, 500, 5000, 50000), (6, -6, 60, 600, 6000, 60000); From 7e58e23d3a63d3a398c468cc3ed93db52e773623 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 31 Jan 2023 14:59:35 +0100 Subject: [PATCH 297/717] Analyzer subquery resolve fix --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index f958c5a5f5e..c7cb2f03597 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -1087,6 +1087,8 @@ private: static bool isFunctionExpressionNodeType(QueryTreeNodeType node_type); + static bool isSubqueryNodeType(QueryTreeNodeType node_type); + static bool isTableExpressionNodeType(QueryTreeNodeType node_type); static DataTypePtr getExpressionNodeResultTypeOrNull(const QueryTreeNodePtr & query_tree_node); @@ -1297,10 +1299,15 @@ bool QueryAnalyzer::isFunctionExpressionNodeType(QueryTreeNodeType node_type) return node_type == QueryTreeNodeType::LAMBDA; } +bool QueryAnalyzer::isSubqueryNodeType(QueryTreeNodeType node_type) +{ + return node_type == QueryTreeNodeType::QUERY || node_type == QueryTreeNodeType::UNION; +} + bool QueryAnalyzer::isTableExpressionNodeType(QueryTreeNodeType node_type) { return node_type == QueryTreeNodeType::TABLE || node_type == QueryTreeNodeType::TABLE_FUNCTION || - node_type == QueryTreeNodeType::QUERY || node_type == QueryTreeNodeType::UNION; + isSubqueryNodeType(node_type); } DataTypePtr QueryAnalyzer::getExpressionNodeResultTypeOrNull(const QueryTreeNodePtr & query_tree_node) @@ -4923,7 +4930,7 @@ ProjectionNames QueryAnalyzer::resolveExpressionNode(QueryTreeNodePtr & node, Id * alias table because in alias table subquery could be evaluated as scalar. */ bool use_alias_table = true; - if (scope.nodes_with_duplicated_aliases.contains(node) || (allow_table_expression && node->getNodeType() == QueryTreeNodeType::QUERY)) + if (scope.nodes_with_duplicated_aliases.contains(node) || (allow_table_expression && isSubqueryNodeType(node->getNodeType()))) use_alias_table = false; if (!node_alias.empty() && use_alias_table) From 5cc56eb108dca990c776a83a81f7b7985dc5931c Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Thu, 2 Feb 2023 17:26:39 +0100 Subject: [PATCH 298/717] Analyzer better support for single_join_prefer_left_table setting --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 68 ++++++++++++------- src/Analyzer/Utils.cpp | 52 ++++++++++++++ src/Analyzer/Utils.h | 3 + .../02378_analyzer_projection_names.reference | 2 +- .../02378_analyzer_projection_names.sql | 3 +- 5 files changed, 102 insertions(+), 26 deletions(-) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index c7cb2f03597..eec3e8646c2 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -406,6 +406,7 @@ struct TableExpressionData std::string table_expression_description; std::string database_name; std::string table_name; + bool should_qualify_columns = true; NamesAndTypes column_names_and_types; ColumnNameToColumnNodeMap column_name_to_column_node; std::unordered_set> column_identifier_first_parts; @@ -433,6 +434,7 @@ struct TableExpressionData if (!table_name.empty()) buffer << " table name " << table_name; + buffer << " should qualify columns " << should_qualify_columns; buffer << " columns size " << column_name_to_column_node.size() << '\n'; for (const auto & [column_name, column_node] : column_name_to_column_node) @@ -1060,7 +1062,7 @@ public: scope.expression_join_tree_node = table_expression; validateTableExpressionModifiers(scope.expression_join_tree_node, scope); - initializeTableExpressionColumns(scope.expression_join_tree_node, scope); + initializeTableExpressionData(scope.expression_join_tree_node, scope); if (node_type == QueryTreeNodeType::LIST) resolveExpressionNodeList(node, scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); @@ -1248,7 +1250,7 @@ private: void initializeQueryJoinTreeNode(QueryTreeNodePtr & join_tree_node, IdentifierResolveScope & scope); - void initializeTableExpressionColumns(const QueryTreeNodePtr & table_expression_node, IdentifierResolveScope & scope); + void initializeTableExpressionData(const QueryTreeNodePtr & table_expression_node, IdentifierResolveScope & scope); void resolveTableFunction(QueryTreeNodePtr & table_function_node, IdentifierResolveScope & scope, QueryExpressionsAliasVisitor & expressions_visitor, bool nested_table_function); @@ -2704,6 +2706,7 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromTableExpression(const Id result_expression = result_expression->clone(); auto qualified_identifier = identifier; + for (size_t i = 0; i < identifier_column_qualifier_parts; ++i) { auto qualified_identifier_with_removed_part = qualified_identifier; @@ -2718,19 +2721,22 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromTableExpression(const Id bool can_remove_qualificator = true; - for (auto & table_expression_to_check_data : scope.table_expression_node_to_data) + if (table_expression_data.should_qualify_columns) { - const auto & table_expression_to_check = table_expression_to_check_data.first; - if (table_expression_to_check.get() == table_expression_node.get()) - continue; - - IdentifierLookup column_identifier_lookup{qualified_identifier_with_removed_part, IdentifierLookupContext::EXPRESSION}; - bool can_bind_identifier_to_table_expression = tryBindIdentifierToTableExpression(column_identifier_lookup, table_expression_to_check, scope); - - if (can_bind_identifier_to_table_expression) + for (auto & table_expression_to_check_data : scope.table_expression_node_to_data) { - can_remove_qualificator = false; - break; + const auto & table_expression_to_check = table_expression_to_check_data.first; + if (table_expression_to_check.get() == table_expression_node.get()) + continue; + + IdentifierLookup column_identifier_lookup{qualified_identifier_with_removed_part, IdentifierLookupContext::EXPRESSION}; + bool can_bind_identifier_to_table_expression = tryBindIdentifierToTableExpression(column_identifier_lookup, table_expression_to_check, scope); + + if (can_bind_identifier_to_table_expression) + { + can_remove_qualificator = false; + break; + } } } @@ -3292,6 +3298,7 @@ void QueryAnalyzer::qualifyColumnNodesWithProjectionNames(const QueryTreeNodes & additional_column_qualification_parts = {table_node->getStorageID().getDatabaseName(), table_node->getStorageID().getTableName()}; size_t additional_column_qualification_parts_size = additional_column_qualification_parts.size(); + const auto & table_expression_data = scope.getTableExpressionDataOrThrow(table_expression_node); /** For each matched column node iterate over additional column qualifications and apply them if column needs to be qualified. * To check if column needs to be qualified we check if column name can bind to any other table expression in scope or to scope aliases. @@ -3310,15 +3317,18 @@ void QueryAnalyzer::qualifyColumnNodesWithProjectionNames(const QueryTreeNodes & auto identifier_to_check = Identifier(column_qualified_identifier_parts); IdentifierLookup lookup{identifier_to_check, IdentifierLookupContext::EXPRESSION}; - for (const auto & table_expression_data : scope.table_expression_node_to_data) + if (table_expression_data.should_qualify_columns) { - if (table_expression_data.first.get() == table_expression_node.get()) - continue; - - if (tryBindIdentifierToTableExpression(lookup, table_expression_data.first, scope)) + for (const auto & scope_table_expression_data : scope.table_expression_node_to_data) { - need_to_qualify = true; - break; + if (scope_table_expression_data.first.get() == table_expression_node.get()) + continue; + + if (tryBindIdentifierToTableExpression(lookup, scope_table_expression_data.first, scope)) + { + need_to_qualify = true; + break; + } } } @@ -3405,7 +3415,8 @@ QueryAnalyzer::QueryTreeNodesWithNames QueryAnalyzer::getMatchedColumnNodesWithN matched_column_nodes.emplace_back(std::make_shared(column, table_expression_node)); } - qualifyColumnNodesWithProjectionNames(matched_column_nodes, table_expression_node, scope); + const auto & qualify_matched_column_nodes_scope = nearest_query_scope ? *nearest_query_scope : scope; + qualifyColumnNodesWithProjectionNames(matched_column_nodes, table_expression_node, qualify_matched_column_nodes_scope); QueryAnalyzer::QueryTreeNodesWithNames matched_column_nodes_with_names; matched_column_nodes_with_names.reserve(matched_column_nodes.size()); @@ -5571,8 +5582,8 @@ void QueryAnalyzer::initializeQueryJoinTreeNode(QueryTreeNodePtr & join_tree_nod } } -/// Initialize table expression columns for table expression node -void QueryAnalyzer::initializeTableExpressionColumns(const QueryTreeNodePtr & table_expression_node, IdentifierResolveScope & scope) +/// Initialize table expression data for table expression node +void QueryAnalyzer::initializeTableExpressionData(const QueryTreeNodePtr & table_expression_node, IdentifierResolveScope & scope) { auto * table_node = table_expression_node->as(); auto * query_node = table_expression_node->as(); @@ -5701,6 +5712,15 @@ void QueryAnalyzer::initializeTableExpressionColumns(const QueryTreeNodePtr & ta table_expression_data.column_identifier_first_parts.insert(column_name_identifier.at(0)); } + if (auto * scope_query_node = scope.scope_node->as()) + { + auto left_table_expression = extractLeftTableExpression(scope_query_node->getJoinTree()); + if (table_expression_node.get() == left_table_expression.get() && + scope.joins_count == 1 && + scope.context->getSettingsRef().single_join_prefer_left_table) + table_expression_data.should_qualify_columns = false; + } + scope.table_expression_node_to_data.emplace(table_expression_node, std::move(table_expression_data)); } @@ -6048,7 +6068,7 @@ void QueryAnalyzer::resolveQueryJoinTreeNode(QueryTreeNodePtr & join_tree_node, if (isTableExpressionNodeType(join_tree_node_type)) { validateTableExpressionModifiers(join_tree_node, scope); - initializeTableExpressionColumns(join_tree_node, scope); + initializeTableExpressionData(join_tree_node, scope); } auto add_table_expression_alias_into_scope = [&](const QueryTreeNodePtr & table_expression_node) diff --git a/src/Analyzer/Utils.cpp b/src/Analyzer/Utils.cpp index bd859ecd379..790bfe0369a 100644 --- a/src/Analyzer/Utils.cpp +++ b/src/Analyzer/Utils.cpp @@ -244,6 +244,58 @@ QueryTreeNodes extractTableExpressions(const QueryTreeNodePtr & join_tree_node) return result; } +QueryTreeNodePtr extractLeftTableExpression(const QueryTreeNodePtr & join_tree_node) +{ + QueryTreeNodePtr result; + + std::deque nodes_to_process; + nodes_to_process.push_back(join_tree_node); + + while (!nodes_to_process.empty()) + { + auto node_to_process = std::move(nodes_to_process.front()); + nodes_to_process.pop_front(); + + auto node_type = node_to_process->getNodeType(); + + switch (node_type) + { + case QueryTreeNodeType::TABLE: + [[fallthrough]]; + case QueryTreeNodeType::QUERY: + [[fallthrough]]; + case QueryTreeNodeType::UNION: + [[fallthrough]]; + case QueryTreeNodeType::TABLE_FUNCTION: + { + result = std::move(node_to_process); + break; + } + case QueryTreeNodeType::ARRAY_JOIN: + { + auto & array_join_node = node_to_process->as(); + nodes_to_process.push_front(array_join_node.getTableExpression()); + break; + } + case QueryTreeNodeType::JOIN: + { + auto & join_node = node_to_process->as(); + nodes_to_process.push_front(join_node.getLeftTableExpression()); + break; + } + default: + { + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Unexpected node type for table expression. " + "Expected table, table function, query, union, join or array join. Actual {}", + node_to_process->getNodeTypeName()); + } + } + } + + return result; +} + namespace { diff --git a/src/Analyzer/Utils.h b/src/Analyzer/Utils.h index c1a3abd0db7..19800e88d2b 100644 --- a/src/Analyzer/Utils.h +++ b/src/Analyzer/Utils.h @@ -19,6 +19,9 @@ void addTableExpressionOrJoinIntoTablesInSelectQuery(ASTPtr & tables_in_select_q /// Extract table, table function, query, union from join tree QueryTreeNodes extractTableExpressions(const QueryTreeNodePtr & join_tree_node); +/// Extract left table expression from join tree +QueryTreeNodePtr extractLeftTableExpression(const QueryTreeNodePtr & join_tree_node); + /** Build table expressions stack that consists from table, table function, query, union, join, array join from join tree. * * Example: SELECT * FROM t1 INNER JOIN t2 INNER JOIN t3. diff --git a/tests/queries/0_stateless/02378_analyzer_projection_names.reference b/tests/queries/0_stateless/02378_analyzer_projection_names.reference index 1fa79677876..9e72fe0d100 100644 --- a/tests/queries/0_stateless/02378_analyzer_projection_names.reference +++ b/tests/queries/0_stateless/02378_analyzer_projection_names.reference @@ -264,7 +264,7 @@ DESCRIBE (WITH x -> x + 1 AS test_lambda SELECT test_lambda(1)); test_lambda(1) UInt16 SELECT '--'; -- -DESCRIBE (WITH x -> * AS test_lambda SELECT test_lambda(1) AS value, value FROM test_table); +DESCRIBE (WITH x -> * AS test_lambda SELECT test_lambda(1) AS lambda_value, lambda_value FROM test_table); id UInt64 value String id UInt64 diff --git a/tests/queries/0_stateless/02378_analyzer_projection_names.sql b/tests/queries/0_stateless/02378_analyzer_projection_names.sql index 907cc79dcec..c69a1c1ad26 100644 --- a/tests/queries/0_stateless/02378_analyzer_projection_names.sql +++ b/tests/queries/0_stateless/02378_analyzer_projection_names.sql @@ -1,4 +1,5 @@ SET allow_experimental_analyzer = 1; +SET single_join_prefer_left_table = 0; DROP TABLE IF EXISTS test_table; CREATE TABLE test_table @@ -254,7 +255,7 @@ DESCRIBE (WITH x -> x + 1 AS test_lambda SELECT test_lambda(1)); SELECT '--'; -DESCRIBE (WITH x -> * AS test_lambda SELECT test_lambda(1) AS value, value FROM test_table); +DESCRIBE (WITH x -> * AS test_lambda SELECT test_lambda(1) AS lambda_value, lambda_value FROM test_table); SELECT 'Subquery'; From f8442b2a8d225f07d93ab04120f82015b890e226 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Thu, 2 Feb 2023 19:25:32 +0100 Subject: [PATCH 299/717] Analyzer support LiveView --- .../InterpreterSelectQueryAnalyzer.cpp | 35 +- src/Planner/Planner.cpp | 34 + src/Planner/PlannerJoinTree.cpp | 24 + .../QueryPlanOptimizationSettings.h | 2 +- src/Storages/LiveView/LiveViewSink.h | 8 +- src/Storages/LiveView/StorageLiveView.cpp | 941 +++++++++++------- src/Storages/LiveView/StorageLiveView.h | 171 ++-- src/Storages/SelectQueryDescription.h | 2 + src/Storages/StorageDistributed.cpp | 20 - 9 files changed, 732 insertions(+), 505 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp b/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp index 2d869cf0d5d..5f3a83b32ac 100644 --- a/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp +++ b/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp @@ -5,6 +5,8 @@ #include #include +#include + #include #include @@ -48,6 +50,22 @@ ASTPtr normalizeAndValidateQuery(const ASTPtr & query) } } +ContextMutablePtr buildContext(const ContextPtr & context, const SelectQueryOptions & select_query_options) +{ + auto result_context = Context::createCopy(context); + + if (select_query_options.shard_num) + result_context->addSpecialScalar( + "_shard_num", + Block{{DataTypeUInt32().createColumnConst(1, *select_query_options.shard_num), std::make_shared(), "_shard_num"}}); + if (select_query_options.shard_count) + result_context->addSpecialScalar( + "_shard_count", + Block{{DataTypeUInt32().createColumnConst(1, *select_query_options.shard_count), std::make_shared(), "_shard_count"}}); + + return result_context; +} + QueryTreeNodePtr buildQueryTreeAndRunPasses(const ASTPtr & query, const SelectQueryOptions & select_query_options, const ContextPtr & context) { auto query_tree = buildQueryTree(query, context); @@ -75,7 +93,7 @@ InterpreterSelectQueryAnalyzer::InterpreterSelectQueryAnalyzer( const ContextPtr & context_, const SelectQueryOptions & select_query_options_) : query(normalizeAndValidateQuery(query_)) - , context(Context::createCopy(context_)) + , context(buildContext(context_, select_query_options_)) , select_query_options(select_query_options_) , query_tree(buildQueryTreeAndRunPasses(query, select_query_options, context)) , planner(query_tree, select_query_options, buildPlannerConfiguration(select_query_options)) @@ -87,7 +105,7 @@ InterpreterSelectQueryAnalyzer::InterpreterSelectQueryAnalyzer( const ContextPtr & context_, const SelectQueryOptions & select_query_options_) : query(query_tree_->toAST()) - , context(Context::createCopy(context_)) + , context(buildContext(context_, select_query_options_)) , select_query_options(select_query_options_) , query_tree(query_tree_) , planner(query_tree, select_query_options, buildPlannerConfiguration(select_query_options)) @@ -124,15 +142,10 @@ Block InterpreterSelectQueryAnalyzer::getSampleBlock() BlockIO InterpreterSelectQueryAnalyzer::execute() { - planner.buildQueryPlanIfNeeded(); - auto & query_plan = planner.getQueryPlan(); - - QueryPlanOptimizationSettings optimization_settings; - BuildQueryPipelineSettings build_pipeline_settings; - auto pipeline_builder = query_plan.buildQueryPipeline(optimization_settings, build_pipeline_settings); + auto pipeline_builder = buildQueryPipeline(); BlockIO result; - result.pipeline = QueryPipelineBuilder::getPipeline(std::move(*pipeline_builder)); + result.pipeline = QueryPipelineBuilder::getPipeline(std::move(pipeline_builder)); if (!select_query_options.ignore_quota && (select_query_options.to_stage == QueryProcessingStage::Complete)) result.pipeline.setQuota(context->getQuota()); @@ -151,8 +164,8 @@ QueryPipelineBuilder InterpreterSelectQueryAnalyzer::buildQueryPipeline() planner.buildQueryPlanIfNeeded(); auto & query_plan = planner.getQueryPlan(); - QueryPlanOptimizationSettings optimization_settings; - BuildQueryPipelineSettings build_pipeline_settings; + auto optimization_settings = QueryPlanOptimizationSettings::fromContext(context); + auto build_pipeline_settings = BuildQueryPipelineSettings::fromContext(context); return std::move(*query_plan.buildQueryPipeline(optimization_settings, build_pipeline_settings)); } diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp index 71c74448c3a..703de0e9119 100644 --- a/src/Planner/Planner.cpp +++ b/src/Planner/Planner.cpp @@ -79,6 +79,7 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; extern const int TOO_DEEP_SUBQUERIES; extern const int NOT_IMPLEMENTED; + extern const int ILLEGAL_PREWHERE; } /** ClickHouse query planner. @@ -134,6 +135,37 @@ void checkStoragesSupportTransactions(const PlannerContextPtr & planner_context) } } +void checkStorageSupportPrewhere(const QueryTreeNodePtr & query_node) +{ + auto & query_node_typed = query_node->as(); + auto table_expression = extractLeftTableExpression(query_node_typed.getJoinTree()); + + if (auto * table_node = table_expression->as()) + { + auto storage = table_node->getStorage(); + if (!storage->supportsPrewhere()) + throw Exception(ErrorCodes::ILLEGAL_PREWHERE, + "Storage {} (table {}) does not support PREWHERE", + storage->getName(), + storage->getStorageID().getNameForLogs()); + } + else if (auto * table_function_node = table_expression->as()) + { + auto storage = table_function_node->getStorage(); + if (!storage->supportsPrewhere()) + throw Exception(ErrorCodes::ILLEGAL_PREWHERE, + "Table function storage {} (table {}) does not support PREWHERE", + storage->getName(), + storage->getStorageID().getNameForLogs()); + } + else + { + throw Exception(ErrorCodes::ILLEGAL_PREWHERE, + "Subquery {} does not support PREWHERE", + query_node->formatASTForErrorMessage()); + } +} + /// Extend lifetime of query context, storages, and table locks void extendQueryContextAndStoragesLifetime(QueryPlan & query_plan, const PlannerContextPtr & planner_context) { @@ -1090,6 +1122,8 @@ void Planner::buildPlanForQueryNode() if (query_node.hasPrewhere()) { + checkStorageSupportPrewhere(query_tree); + if (query_node.hasWhere()) query_node.getWhere() = mergeConditionNodes({query_node.getPrewhere(), query_node.getWhere()}, query_context); else diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index 4f94f2419e1..505f0e8e3d0 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -367,6 +367,27 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(const QueryTreeNodePtr & tabl rename_step->setStepDescription("Change column names to column identifiers"); query_plan.addStep(std::move(rename_step)); } + else + { + Planner planner(select_query_info.query_tree, + SelectQueryOptions(from_stage), + select_query_info.planner_context, + PlannerConfiguration{.only_analyze = true}); + planner.buildQueryPlanIfNeeded(); + + auto expected_header = planner.getQueryPlan().getCurrentDataStream().header; + materializeBlockInplace(expected_header); + + auto rename_actions_dag = ActionsDAG::makeConvertingActions( + query_plan.getCurrentDataStream().header.getColumnsWithTypeAndName(), + expected_header.getColumnsWithTypeAndName(), + ActionsDAG::MatchColumnsMode::Position, + true /*ignore_constant_values*/); + auto rename_step = std::make_unique(query_plan.getCurrentDataStream(), std::move(rename_actions_dag)); + std::string step_description = table_expression_data.isRemote() ? "Change remote column names to local column names" : "Change column names"; + rename_step->setStepDescription(std::move(step_description)); + query_plan.addStep(std::move(rename_step)); + } return {std::move(query_plan), from_stage}; } @@ -918,6 +939,9 @@ JoinTreeQueryPlan buildJoinTreeQueryPlan(const QueryTreeNodePtr & query_node, select_query_options, planner_context, is_single_table_expression)); + + if (query_plans_stack.back().from_stage != QueryProcessingStage::FetchColumns) + break; } } diff --git a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h index b894e5caf1d..89464e01d3c 100644 --- a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h +++ b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h @@ -31,7 +31,7 @@ struct QueryPlanOptimizationSettings bool aggregation_in_order = false; /// If removing redundant sorting is enabled, for example, ORDER BY clauses in subqueries - bool remove_redundant_sorting = true; + bool remove_redundant_sorting = false; static QueryPlanOptimizationSettings fromSettings(const Settings & from); static QueryPlanOptimizationSettings fromContext(ContextPtr from); diff --git a/src/Storages/LiveView/LiveViewSink.h b/src/Storages/LiveView/LiveViewSink.h index bbb8bf02c45..1d90e35618f 100644 --- a/src/Storages/LiveView/LiveViewSink.h +++ b/src/Storages/LiveView/LiveViewSink.h @@ -40,10 +40,10 @@ public: std::lock_guard lock(storage.mutex); - if (storage.getBlocksHashKey() != key_str) + if (storage.getBlocksHashKey(lock) != key_str) { new_blocks_metadata->hash = key_str; - new_blocks_metadata->version = storage.getBlocksVersion() + 1; + new_blocks_metadata->version = storage.getBlocksVersion(lock) + 1; new_blocks_metadata->time = std::chrono::system_clock::now(); for (auto & block : *new_blocks) @@ -62,8 +62,8 @@ public: else { // only update blocks time - new_blocks_metadata->hash = storage.getBlocksHashKey(); - new_blocks_metadata->version = storage.getBlocksVersion(); + new_blocks_metadata->hash = storage.getBlocksHashKey(lock); + new_blocks_metadata->version = storage.getBlocksVersion(lock); new_blocks_metadata->time = std::chrono::system_clock::now(); (*storage.blocks_metadata_ptr) = new_blocks_metadata; diff --git a/src/Storages/LiveView/StorageLiveView.cpp b/src/Storages/LiveView/StorageLiveView.cpp index 8f36ea4d91d..ba499412b33 100644 --- a/src/Storages/LiveView/StorageLiveView.cpp +++ b/src/Storages/LiveView/StorageLiveView.cpp @@ -15,6 +15,7 @@ limitations under the License. */ #include #include #include +#include #include #include #include @@ -22,6 +23,7 @@ limitations under the License. */ #include #include #include +#include #include #include #include @@ -41,6 +43,12 @@ limitations under the License. */ #include #include +#include +#include +#include +#include +#include + namespace DB { @@ -53,85 +61,10 @@ namespace ErrorCodes extern const int SUPPORT_IS_DISABLED; } - -static StorageID extractDependentTable(ASTPtr & query, ContextPtr context, const String & table_name, ASTPtr & inner_subquery) +namespace { - ASTSelectQuery & select_query = typeid_cast(*query); - if (auto db_and_table = getDatabaseAndTable(select_query, 0)) - { - String select_database_name = context->getCurrentDatabase(); - String select_table_name = db_and_table->table; - - if (db_and_table->database.empty()) - { - db_and_table->database = select_database_name; - AddDefaultDatabaseVisitor visitor(context, select_database_name); - visitor.visit(select_query); - } - else - select_database_name = db_and_table->database; - - select_query.replaceDatabaseAndTable("", table_name + "_blocks"); - return StorageID(select_database_name, select_table_name); - } - else if (auto subquery = extractTableExpression(select_query, 0)) - { - auto * ast_select = subquery->as(); - if (!ast_select) - throw Exception(DB::ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_LIVE_VIEW, - "LIVE VIEWs are only supported for queries from tables, " - "but there is no table name in select query."); - if (ast_select->list_of_selects->children.size() != 1) - throw Exception(ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_LIVE_VIEW, "UNION is not supported for LIVE VIEW"); - - inner_subquery = ast_select->list_of_selects->children.at(0)->clone(); - - return extractDependentTable(ast_select->list_of_selects->children.at(0), context, table_name, inner_subquery); - } - else - { - /// If the table is not specified - use the table `system.one` - return StorageID("system", "one"); - } -} - -MergeableBlocksPtr StorageLiveView::collectMergeableBlocks(ContextPtr local_context) -{ - ASTPtr mergeable_query = inner_query; - - if (inner_subquery) - mergeable_query = inner_subquery; - - MergeableBlocksPtr new_mergeable_blocks = std::make_shared(); - BlocksPtrs new_blocks = std::make_shared>(); - BlocksPtr base_blocks = std::make_shared(); - - InterpreterSelectQuery interpreter(mergeable_query->clone(), local_context, SelectQueryOptions(QueryProcessingStage::WithMergeableState), Names()); - - auto builder = interpreter.buildQueryPipeline(); - builder.addSimpleTransform([&](const Block & cur_header) - { - return std::make_shared(cur_header); - }); - - new_mergeable_blocks->sample_block = builder.getHeader(); - - auto pipeline = QueryPipelineBuilder::getPipeline(std::move(builder)); - PullingAsyncPipelineExecutor executor(pipeline); - Block this_block; - - while (executor.pull(this_block)) - base_blocks->push_back(this_block); - - new_blocks->push_back(base_blocks); - - new_mergeable_blocks->blocks = new_blocks; - - return new_mergeable_blocks; -} - -Pipes StorageLiveView::blocksToPipes(BlocksPtrs blocks, Block & sample_block) +Pipes blocksToPipes(BlocksPtrs blocks, Block & sample_block) { Pipes pipes; for (auto & blocks_for_source : *blocks) @@ -140,137 +73,141 @@ Pipes StorageLiveView::blocksToPipes(BlocksPtrs blocks, Block & sample_block) return pipes; } -/// Complete query using input streams from mergeable blocks -QueryPipelineBuilder StorageLiveView::completeQuery(Pipes pipes) +SelectQueryDescription buildSelectQueryDescription(const ASTPtr & select_query, const ContextPtr & context, std::string replace_table_name = {}) { - //FIXME it's dangerous to create Context on stack - auto block_context = Context::createCopy(getContext()); - block_context->makeQueryContext(); + ASTPtr inner_query = select_query; + std::optional dependent_table_storage_id; - auto creator = [&](const StorageID & blocks_id_global) + while (true) { - auto parent_table_metadata = getParentStorage()->getInMemoryMetadataPtr(); - return StorageBlocks::createStorage( - blocks_id_global, parent_table_metadata->getColumns(), - std::move(pipes), QueryProcessingStage::WithMergeableState); - }; - block_context->addExternalTable(getBlocksTableName(), TemporaryTableHolder(getContext(), creator)); - InterpreterSelectQuery select(getInnerBlocksQuery(), block_context, StoragePtr(), nullptr, SelectQueryOptions(QueryProcessingStage::Complete)); - auto builder = select.buildQueryPipeline(); - builder.addSimpleTransform([&](const Block & cur_header) - { - return std::make_shared(cur_header); - }); + auto * inner_select_with_union_query = inner_query->as(); - /// Squashing is needed here because the view query can generate a lot of blocks - /// even when only one block is inserted into the parent table (e.g. if the query is a GROUP BY - /// and two-level aggregation is triggered). - builder.addSimpleTransform([&](const Block & cur_header) - { - return std::make_shared( - cur_header, - getContext()->getSettingsRef().min_insert_block_size_rows, - getContext()->getSettingsRef().min_insert_block_size_bytes); - }); + if (inner_select_with_union_query) + { + if (inner_select_with_union_query->list_of_selects->children.size() != 1) + throw Exception(ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_LIVE_VIEW, "UNION is not supported for LIVE VIEW"); - return builder; + inner_query = inner_select_with_union_query->list_of_selects->children[0]; + } + + auto * inner_select_query = inner_query->as(); + if (!inner_select_query) + throw Exception(DB::ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_LIVE_VIEW, + "LIVE VIEWs are only supported for queries from tables, " + "but there is no table name in select query."); + + if (auto db_and_table = getDatabaseAndTable(*inner_select_query, 0)) + { + const auto * table_expression = getTableExpression(*inner_select_query, 0); + if (table_expression->database_and_table_name->tryGetAlias().empty()) + table_expression->database_and_table_name->setAlias("__dependent_table"); + + String select_database_name = db_and_table->database; + String select_table_name = db_and_table->table; + + if (select_database_name.empty()) + { + select_database_name = context->getCurrentDatabase(); + db_and_table->database = select_database_name; + AddDefaultDatabaseVisitor visitor(context, select_database_name); + visitor.visit(*inner_select_query); + } + + if (replace_table_name.empty()) + { + dependent_table_storage_id = StorageID(select_database_name, select_table_name); + } + else + { + inner_select_query->replaceDatabaseAndTable("", replace_table_name); + dependent_table_storage_id = StorageID("", replace_table_name); + } + + break; + } + else if (auto subquery = extractTableExpression(*inner_select_query, 0)) + { + inner_query = subquery; + } + else + { + break; + } + } + + if (!dependent_table_storage_id) + { + /// If the table is not specified - use the table `system.one` + dependent_table_storage_id = StorageID("system", "one"); + } + + SelectQueryDescription result; + result.select_table_id = *dependent_table_storage_id; + result.select_query = select_query; + result.inner_query = std::move(inner_query); + + return result; } -void StorageLiveView::writeIntoLiveView( - StorageLiveView & live_view, - const Block & block, - ContextPtr local_context) +struct SelectQueryTreeDescription { - auto output = std::make_shared(live_view); + QueryTreeNodePtr select_query_node; + QueryTreeNodePtr inner_query_node; + QueryTreeNodePtr dependent_table_node; +}; - /// Check if live view has any readers if not - /// just reset blocks to empty and do nothing else - /// When first reader comes the blocks will be read. +SelectQueryTreeDescription buildSelectQueryTreeDescription(const ASTPtr & select_query, const ContextPtr & context) +{ + auto select_query_node = buildQueryTree(select_query, context); + + QueryTreePassManager query_tree_pass_manager(context); + addQueryTreePasses(query_tree_pass_manager); + query_tree_pass_manager.run(select_query_node); + + QueryTreeNodePtr inner_query_node = select_query_node; + QueryTreeNodePtr dependent_table_node; + + while (true) { - std::lock_guard lock(live_view.mutex); - if (!live_view.hasActiveUsers()) + auto & query_node = inner_query_node->as(); + + auto left_table_expression = extractLeftTableExpression(query_node.getJoinTree()); + auto left_table_expression_node_type = left_table_expression->getNodeType(); + + if (left_table_expression_node_type == QueryTreeNodeType::QUERY) { - live_view.reset(); - return; + inner_query_node = left_table_expression; + } + else if (left_table_expression_node_type == QueryTreeNodeType::TABLE) + { + dependent_table_node = left_table_expression; + break; + } + else if (left_table_expression_node_type == QueryTreeNodeType::TABLE_FUNCTION) + { + break; + } + else + { + throw Exception(ErrorCodes::UNSUPPORTED_METHOD, + "LiveView does not support UNION {} subquery", + left_table_expression->formatASTForErrorMessage()); } } - bool is_block_processed = false; - Pipes from; - MergeableBlocksPtr mergeable_blocks; - BlocksPtr new_mergeable_blocks = std::make_shared(); - - { - std::lock_guard lock(live_view.mutex); - - mergeable_blocks = live_view.getMergeableBlocks(); - if (!mergeable_blocks || mergeable_blocks->blocks->size() >= local_context->getGlobalContext()->getSettingsRef().max_live_view_insert_blocks_before_refresh) - { - mergeable_blocks = live_view.collectMergeableBlocks(local_context); - live_view.setMergeableBlocks(mergeable_blocks); - from = live_view.blocksToPipes(mergeable_blocks->blocks, mergeable_blocks->sample_block); - is_block_processed = true; - } - } - - if (!is_block_processed) - { - ASTPtr mergeable_query = live_view.getInnerQuery(); - - if (live_view.getInnerSubQuery()) - mergeable_query = live_view.getInnerSubQuery(); - - Pipes pipes; - pipes.emplace_back(std::make_shared(block)); - - auto creator = [&](const StorageID & blocks_id_global) - { - auto parent_metadata = live_view.getParentStorage()->getInMemoryMetadataPtr(); - return StorageBlocks::createStorage( - blocks_id_global, parent_metadata->getColumns(), - std::move(pipes), QueryProcessingStage::FetchColumns); - }; - TemporaryTableHolder blocks_storage(local_context, creator); - - InterpreterSelectQuery select_block(mergeable_query, local_context, blocks_storage.getTable(), blocks_storage.getTable()->getInMemoryMetadataPtr(), - QueryProcessingStage::WithMergeableState); - - auto builder = select_block.buildQueryPipeline(); - builder.addSimpleTransform([&](const Block & cur_header) - { - return std::make_shared(cur_header); - }); - - auto pipeline = QueryPipelineBuilder::getPipeline(std::move(builder)); - PullingAsyncPipelineExecutor executor(pipeline); - Block this_block; - - while (executor.pull(this_block)) - new_mergeable_blocks->push_back(this_block); - - if (new_mergeable_blocks->empty()) - return; - - { - std::lock_guard lock(live_view.mutex); - - mergeable_blocks = live_view.getMergeableBlocks(); - mergeable_blocks->blocks->push_back(new_mergeable_blocks); - from = live_view.blocksToPipes(mergeable_blocks->blocks, mergeable_blocks->sample_block); - } - } - - auto pipeline = live_view.completeQuery(std::move(from)); - pipeline.addChain(Chain(std::move(output))); - pipeline.setSinks([&](const Block & cur_header, Pipe::StreamType) - { - return std::make_shared(cur_header); - }); - - auto executor = pipeline.execute(); - executor->execute(pipeline.getNumThreads()); + return {std::move(select_query_node), std::move(inner_query_node), std::move(dependent_table_node)}; } +QueryTreeNodePtr cloneQueryTreeAndReplaceTableExpression(const QueryTreeNodePtr & query_tree, + const QueryTreeNodePtr table_expression_to_replace, + const QueryTreeNodePtr & replacement_table_expression) +{ + std::unordered_map replacement_map; + replacement_map.emplace(table_expression_to_replace.get(), replacement_table_expression); + return query_tree->cloneAndReplace(replacement_map); +} + +} StorageLiveView::StorageLiveView( const StorageID & table_id_, @@ -296,16 +233,10 @@ StorageLiveView::StorageLiveView( if (!query.select) throw Exception(ErrorCodes::INCORRECT_QUERY, "SELECT query is not specified for {}", getName()); - /// Default value, if only table name exist in the query - if (query.select->list_of_selects->children.size() != 1) - throw Exception(ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_LIVE_VIEW, "UNION is not supported for LIVE VIEW"); + auto select_query_clone = query.select->clone(); + select_query_description = buildSelectQueryDescription(select_query_clone, getContext()); - inner_query = query.select->list_of_selects->children.at(0); - - auto inner_query_tmp = inner_query->clone(); - select_table_id = extractDependentTable(inner_query_tmp, getContext(), table_id_.table_name, inner_subquery); - - DatabaseCatalog::instance().addViewDependency(select_table_id, table_id_); + DatabaseCatalog::instance().addViewDependency(select_query_description.select_table_id, table_id_); if (query.live_view_periodic_refresh) { @@ -321,115 +252,16 @@ StorageLiveView::StorageLiveView( periodic_refresh_task->deactivate(); } -Block StorageLiveView::getHeader() const +StorageLiveView::~StorageLiveView() { - std::lock_guard lock(sample_block_lock); - - if (!sample_block) - { - sample_block = InterpreterSelectQuery(inner_query->clone(), live_view_context, SelectQueryOptions(QueryProcessingStage::Complete)).getSampleBlock(); - sample_block.insert({DataTypeUInt64().createColumnConst( - sample_block.rows(), 0)->convertToFullColumnIfConst(), - std::make_shared(), - "_version"}); - /// convert all columns to full columns - /// in case some of them are constant - for (size_t i = 0; i < sample_block.columns(); ++i) - { - sample_block.safeGetByPosition(i).column = sample_block.safeGetByPosition(i).column->convertToFullColumnIfConst(); - } - } - return sample_block; + shutdown(); } -StoragePtr StorageLiveView::getParentStorage() const +NamesAndTypesList StorageLiveView::getVirtuals() const { - return DatabaseCatalog::instance().getTable(select_table_id, getContext()); -} - -ASTPtr StorageLiveView::getInnerBlocksQuery() -{ - std::lock_guard lock(sample_block_lock); - if (!inner_blocks_query) - { - inner_blocks_query = inner_query->clone(); - /// Rewrite inner query with right aliases for JOIN. - /// It cannot be done in constructor or startup() because InterpreterSelectQuery may access table, - /// which is not loaded yet during server startup, so we do it lazily - InterpreterSelectQuery(inner_blocks_query, live_view_context, SelectQueryOptions().modify().analyze()); // NOLINT - auto table_id = getStorageID(); - extractDependentTable(inner_blocks_query, getContext(), table_id.table_name, inner_subquery); - } - return inner_blocks_query->clone(); -} - -bool StorageLiveView::getNewBlocks() -{ - SipHash hash; - UInt128 key; - BlocksPtr new_blocks = std::make_shared(); - BlocksMetadataPtr new_blocks_metadata = std::make_shared(); - - /// can't set mergeable_blocks here or anywhere else outside the writeIntoLiveView function - /// as there could be a race condition when the new block has been inserted into - /// the source table by the PushingToViews chain and this method - /// called before writeIntoLiveView function is called which can lead to - /// the same block added twice to the mergeable_blocks leading to - /// inserted data to be duplicated - auto new_mergeable_blocks = collectMergeableBlocks(live_view_context); - Pipes from = blocksToPipes(new_mergeable_blocks->blocks, new_mergeable_blocks->sample_block); - auto builder = completeQuery(std::move(from)); - auto pipeline = QueryPipelineBuilder::getPipeline(std::move(builder)); - - PullingAsyncPipelineExecutor executor(pipeline); - Block block; - while (executor.pull(block)) - { - if (block.rows() == 0) - continue; - - /// calculate hash before virtual column is added - block.updateHash(hash); - /// add result version meta column - block.insert({DataTypeUInt64().createColumnConst( - block.rows(), getBlocksVersion() + 1)->convertToFullColumnIfConst(), - std::make_shared(), - "_version"}); - new_blocks->push_back(block); - } - - hash.get128(key); - - /// Update blocks only if hash keys do not match - /// NOTE: hash could be different for the same result - /// if blocks are not in the same order - bool updated = false; - { - if (getBlocksHashKey() != getHexUIntLowercase(key)) - { - if (new_blocks->empty()) - { - new_blocks->push_back(getHeader()); - } - new_blocks_metadata->hash = getHexUIntLowercase(key); - new_blocks_metadata->version = getBlocksVersion() + 1; - new_blocks_metadata->time = std::chrono::system_clock::now(); - - (*blocks_ptr) = new_blocks; - (*blocks_metadata_ptr) = new_blocks_metadata; - - updated = true; - } - else - { - new_blocks_metadata->hash = getBlocksHashKey(); - new_blocks_metadata->version = getBlocksVersion(); - new_blocks_metadata->time = std::chrono::system_clock::now(); - - (*blocks_metadata_ptr) = new_blocks_metadata; - } - } - return updated; + return NamesAndTypesList{ + NameAndTypePair("_version", std::make_shared()) + }; } void StorageLiveView::checkTableCanBeDropped() const @@ -443,6 +275,15 @@ void StorageLiveView::checkTableCanBeDropped() const } } +void StorageLiveView::drop() +{ + auto table_id = getStorageID(); + DatabaseCatalog::instance().removeViewDependency(select_query_description.select_table_id, table_id); + + std::lock_guard lock(mutex); + condition.notify_all(); +} + void StorageLiveView::startup() { if (is_periodically_refreshed) @@ -456,72 +297,7 @@ void StorageLiveView::shutdown() if (is_periodically_refreshed) periodic_refresh_task->deactivate(); - DatabaseCatalog::instance().removeViewDependency(select_table_id, getStorageID()); -} - -StorageLiveView::~StorageLiveView() -{ - shutdown(); -} - -void StorageLiveView::drop() -{ - auto table_id = getStorageID(); - DatabaseCatalog::instance().removeViewDependency(select_table_id, table_id); - - std::lock_guard lock(mutex); - condition.notify_all(); -} - -void StorageLiveView::scheduleNextPeriodicRefresh() -{ - Seconds current_time = std::chrono::duration_cast (std::chrono::system_clock::now().time_since_epoch()); - Seconds blocks_time = std::chrono::duration_cast (getBlocksTime().time_since_epoch()); - - if ((current_time - periodic_live_view_refresh) >= blocks_time) - { - refresh(false); - blocks_time = std::chrono::duration_cast (getBlocksTime().time_since_epoch()); - } - current_time = std::chrono::duration_cast (std::chrono::system_clock::now().time_since_epoch()); - - auto next_refresh_time = blocks_time + periodic_live_view_refresh; - - if (current_time >= next_refresh_time) - periodic_refresh_task->scheduleAfter(0); - else - { - auto schedule_time = std::chrono::duration_cast (next_refresh_time - current_time); - periodic_refresh_task->scheduleAfter(static_cast(schedule_time.count())); - } -} - -void StorageLiveView::periodicRefreshTaskFunc() -{ - LOG_TRACE(log, "periodic refresh task"); - - std::lock_guard lock(mutex); - - if (hasActiveUsers()) - scheduleNextPeriodicRefresh(); -} - -void StorageLiveView::refresh(bool grab_lock) -{ - // Lock is already acquired exclusively from InterperterAlterQuery.cpp InterpreterAlterQuery::execute() method. - // So, reacquiring lock is not needed and will result in an exception. - - if (grab_lock) - { - std::lock_guard lock(mutex); - if (getNewBlocks()) - condition.notify_all(); - } - else - { - if (getNewBlocks()) - condition.notify_all(); - } + DatabaseCatalog::instance().removeViewDependency(select_query_description.select_table_id, getStorageID()); } Pipe StorageLiveView::read( @@ -536,15 +312,16 @@ Pipe StorageLiveView::read( std::lock_guard lock(mutex); if (!(*blocks_ptr)) - refresh(false); - + { + refreshImpl(lock); + } else if (is_periodically_refreshed) { Seconds current_time = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()); - Seconds blocks_time = std::chrono::duration_cast(getBlocksTime().time_since_epoch()); + Seconds blocks_time = std::chrono::duration_cast(getBlocksTime(lock).time_since_epoch()); if ((current_time - periodic_live_view_refresh) >= blocks_time) - refresh(false); + refreshImpl(lock); } return Pipe(std::make_shared(*blocks_ptr, getHeader())); @@ -585,21 +362,423 @@ Pipe StorageLiveView::watch( std::lock_guard lock(mutex); if (!(*blocks_ptr)) - refresh(false); + refreshImpl(lock); if (is_periodically_refreshed) - scheduleNextPeriodicRefresh(); + scheduleNextPeriodicRefresh(lock); } processed_stage = QueryProcessingStage::Complete; return reader; } -NamesAndTypesList StorageLiveView::getVirtuals() const +void StorageLiveView::writeBlock(const Block & block, ContextPtr local_context) { - return NamesAndTypesList{ - NameAndTypePair("_version", std::make_shared()) + auto output = std::make_shared(*this); + + /// Check if live view has any readers if not + /// just reset blocks to empty and do nothing else + /// When first reader comes the blocks will be read. + { + std::lock_guard lock(mutex); + if (!hasActiveUsers(lock)) + { + reset(lock); + return; + } + } + + bool is_block_processed = false; + Pipes from; + BlocksPtr new_mergeable_blocks = std::make_shared(); + + { + std::lock_guard lock(mutex); + if (!mergeable_blocks || mergeable_blocks->blocks->size() >= local_context->getGlobalContext()->getSettingsRef().max_live_view_insert_blocks_before_refresh) + { + mergeable_blocks = collectMergeableBlocks(local_context, lock); + from = blocksToPipes(mergeable_blocks->blocks, mergeable_blocks->sample_block); + is_block_processed = true; + } + } + + if (!is_block_processed) + { + Pipes pipes; + pipes.emplace_back(std::make_shared(block)); + + auto creator = [&](const StorageID & blocks_id_global) + { + auto parent_metadata = getDependentTableStorage()->getInMemoryMetadataPtr(); + return StorageBlocks::createStorage( + blocks_id_global, parent_metadata->getColumns(), + std::move(pipes), QueryProcessingStage::FetchColumns); + }; + TemporaryTableHolder blocks_storage(local_context, creator); + + QueryPipelineBuilder builder; + + if (local_context->getSettingsRef().allow_experimental_analyzer) + { + auto select_description = buildSelectQueryTreeDescription(select_query_description.inner_query, local_context); + if (select_description.dependent_table_node) + { + auto storage = blocks_storage.getTable(); + auto storage_snapshot = storage->getStorageSnapshot(storage->getInMemoryMetadataPtr(), local_context); + auto replacement_table_expression = std::make_shared(std::move(storage), + TableLockHolder{}, + std::move(storage_snapshot)); + + select_description.inner_query_node = cloneQueryTreeAndReplaceTableExpression(select_description.inner_query_node, + select_description.dependent_table_node, + replacement_table_expression); + } + + InterpreterSelectQueryAnalyzer interpreter(select_description.inner_query_node, + local_context, + SelectQueryOptions(QueryProcessingStage::WithMergeableState)); + builder = interpreter.buildQueryPipeline(); + } + else + { + InterpreterSelectQuery interpreter(select_query_description.inner_query, + local_context, + blocks_storage.getTable(), + blocks_storage.getTable()->getInMemoryMetadataPtr(), + QueryProcessingStage::WithMergeableState); + builder = interpreter.buildQueryPipeline(); + } + + builder.addSimpleTransform([&](const Block & cur_header) + { + return std::make_shared(cur_header); + }); + + auto pipeline = QueryPipelineBuilder::getPipeline(std::move(builder)); + PullingAsyncPipelineExecutor executor(pipeline); + Block this_block; + + while (executor.pull(this_block)) + new_mergeable_blocks->push_back(this_block); + + if (new_mergeable_blocks->empty()) + return; + + { + std::lock_guard lock(mutex); + mergeable_blocks->blocks->push_back(new_mergeable_blocks); + from = blocksToPipes(mergeable_blocks->blocks, mergeable_blocks->sample_block); + } + } + + auto pipeline = completeQuery(std::move(from)); + pipeline.addChain(Chain(std::move(output))); + pipeline.setSinks([&](const Block & cur_header, Pipe::StreamType) + { + return std::make_shared(cur_header); + }); + + auto executor = pipeline.execute(); + executor->execute(pipeline.getNumThreads()); +} + +void StorageLiveView::refresh() +{ + std::lock_guard lock(mutex); + refreshImpl(lock); +} + +void StorageLiveView::refreshImpl(std::lock_guard & lock) +{ + if (getNewBlocks(lock)) + condition.notify_all(); +} + +Block StorageLiveView::getHeader() const +{ + std::lock_guard lock(sample_block_lock); + + if (!sample_block) + { + if (live_view_context->getSettingsRef().allow_experimental_analyzer) + { + sample_block = InterpreterSelectQueryAnalyzer::getSampleBlock(select_query_description.select_query, + live_view_context, + SelectQueryOptions(QueryProcessingStage::Complete)); + } + else + { + auto & select_with_union_query = select_query_description.select_query->as(); + auto select_query = select_with_union_query.list_of_selects->children.at(0)->clone(); + sample_block = InterpreterSelectQuery(select_query, + live_view_context, + SelectQueryOptions(QueryProcessingStage::Complete)).getSampleBlock(); + } + + sample_block.insert({DataTypeUInt64().createColumnConst( + sample_block.rows(), 0)->convertToFullColumnIfConst(), + std::make_shared(), + "_version"}); + + /// convert all columns to full columns + /// in case some of them are constant + for (size_t i = 0; i < sample_block.columns(); ++i) + { + sample_block.safeGetByPosition(i).column = sample_block.safeGetByPosition(i).column->convertToFullColumnIfConst(); + } + } + + return sample_block; +} + +StoragePtr StorageLiveView::getDependentTableStorage() const +{ + return DatabaseCatalog::instance().getTable(select_query_description.select_table_id, getContext()); +} + +ASTPtr StorageLiveView::getInnerBlocksQuery() +{ + std::lock_guard lock(sample_block_lock); + if (!inner_blocks_query) + { + auto & select_with_union_query = select_query_description.select_query->as(); + auto blocks_query = select_with_union_query.list_of_selects->children.at(0)->clone(); + + if (!live_view_context->getSettingsRef().allow_experimental_analyzer) + { + /// Rewrite inner query with right aliases for JOIN. + /// It cannot be done in constructor or startup() because InterpreterSelectQuery may access table, + /// which is not loaded yet during server startup, so we do it lazily + InterpreterSelectQuery(blocks_query, live_view_context, SelectQueryOptions().modify().analyze()); // NOLINT + } + + auto table_id = getStorageID(); + std::string replace_table_name = table_id.table_name + "_blocks"; + inner_blocks_query = buildSelectQueryDescription(blocks_query, getContext(), replace_table_name).select_query; + } + + return inner_blocks_query->clone(); +} + +MergeableBlocksPtr StorageLiveView::collectMergeableBlocks(ContextPtr local_context, std::lock_guard &) const +{ + MergeableBlocksPtr new_mergeable_blocks = std::make_shared(); + BlocksPtrs new_blocks = std::make_shared>(); + BlocksPtr base_blocks = std::make_shared(); + + QueryPipelineBuilder builder; + + if (local_context->getSettingsRef().allow_experimental_analyzer) + { + InterpreterSelectQueryAnalyzer interpreter(select_query_description.inner_query, + local_context, + SelectQueryOptions(QueryProcessingStage::WithMergeableState)); + builder = interpreter.buildQueryPipeline(); + } + else + { + InterpreterSelectQuery interpreter(select_query_description.inner_query->clone(), + local_context, + SelectQueryOptions(QueryProcessingStage::WithMergeableState), Names()); + builder = interpreter.buildQueryPipeline(); + } + + builder.addSimpleTransform([&](const Block & cur_header) + { + return std::make_shared(cur_header); + }); + + new_mergeable_blocks->sample_block = builder.getHeader(); + + auto pipeline = QueryPipelineBuilder::getPipeline(std::move(builder)); + PullingAsyncPipelineExecutor executor(pipeline); + Block this_block; + + while (executor.pull(this_block)) + base_blocks->push_back(this_block); + + new_blocks->push_back(base_blocks); + new_mergeable_blocks->blocks = new_blocks; + + return new_mergeable_blocks; +} + +/// Complete query using input streams from mergeable blocks +QueryPipelineBuilder StorageLiveView::completeQuery(Pipes pipes) +{ + auto block_context = Context::createCopy(getContext()); + block_context->makeQueryContext(); + + QueryPlanResourceHolder resource_holder; + resource_holder.interpreter_context.push_back(block_context); + + auto creator = [&](const StorageID & blocks_id_global) + { + auto parent_table_metadata = getDependentTableStorage()->getInMemoryMetadataPtr(); + return StorageBlocks::createStorage( + blocks_id_global, parent_table_metadata->getColumns(), + std::move(pipes), QueryProcessingStage::WithMergeableState); }; + + TemporaryTableHolder blocks_storage_table_holder(getContext(), creator); + + QueryPipelineBuilder builder; + + if (block_context->getSettingsRef().allow_experimental_analyzer) + { + auto select_description = buildSelectQueryTreeDescription(select_query_description.select_query, block_context); + + if (select_description.dependent_table_node) + { + auto storage = blocks_storage_table_holder.getTable(); + auto storage_snapshot = storage->getStorageSnapshot(storage->getInMemoryMetadataPtr(), block_context); + auto replacement_table_expression = std::make_shared(std::move(storage), + TableLockHolder{}, + std::move(storage_snapshot)); + + select_description.select_query_node = cloneQueryTreeAndReplaceTableExpression(select_description.select_query_node, + select_description.dependent_table_node, + replacement_table_expression); + } + + InterpreterSelectQueryAnalyzer interpreter(select_description.select_query_node, + block_context, + SelectQueryOptions(QueryProcessingStage::Complete)); + builder = interpreter.buildQueryPipeline(); + } + else + { + auto inner_blocks_query = getInnerBlocksQuery(); + block_context->addExternalTable(getBlocksTableName(), std::move(blocks_storage_table_holder)); + InterpreterSelectQuery interpreter(inner_blocks_query, + block_context, + StoragePtr(), + nullptr, + SelectQueryOptions(QueryProcessingStage::Complete)); + builder = interpreter.buildQueryPipeline(); + } + + builder.addSimpleTransform([&](const Block & cur_header) + { + return std::make_shared(cur_header); + }); + + /// Squashing is needed here because the view query can generate a lot of blocks + /// even when only one block is inserted into the parent table (e.g. if the query is a GROUP BY + /// and two-level aggregation is triggered). + builder.addSimpleTransform([&](const Block & cur_header) + { + return std::make_shared( + cur_header, + getContext()->getSettingsRef().min_insert_block_size_rows, + getContext()->getSettingsRef().min_insert_block_size_bytes); + }); + + builder.addResources(std::move(resource_holder)); + + return builder; +} + +bool StorageLiveView::getNewBlocks(std::lock_guard & lock) +{ + SipHash hash; + UInt128 key; + BlocksPtr new_blocks = std::make_shared(); + BlocksMetadataPtr new_blocks_metadata = std::make_shared(); + + /// can't set mergeable_blocks here or anywhere else outside the writeIntoLiveView function + /// as there could be a race condition when the new block has been inserted into + /// the source table by the PushingToViews chain and this method + /// called before writeIntoLiveView function is called which can lead to + /// the same block added twice to the mergeable_blocks leading to + /// inserted data to be duplicated + auto new_mergeable_blocks = collectMergeableBlocks(live_view_context, lock); + Pipes from = blocksToPipes(new_mergeable_blocks->blocks, new_mergeable_blocks->sample_block); + auto builder = completeQuery(std::move(from)); + auto pipeline = QueryPipelineBuilder::getPipeline(std::move(builder)); + + PullingAsyncPipelineExecutor executor(pipeline); + Block block; + while (executor.pull(block)) + { + if (block.rows() == 0) + continue; + + /// calculate hash before virtual column is added + block.updateHash(hash); + /// add result version meta column + block.insert({DataTypeUInt64().createColumnConst( + block.rows(), getBlocksVersion(lock) + 1)->convertToFullColumnIfConst(), + std::make_shared(), + "_version"}); + new_blocks->push_back(block); + } + + hash.get128(key); + + /// Update blocks only if hash keys do not match + /// NOTE: hash could be different for the same result + /// if blocks are not in the same order + bool updated = false; + { + if (getBlocksHashKey(lock) != getHexUIntLowercase(key)) + { + if (new_blocks->empty()) + { + new_blocks->push_back(getHeader()); + } + new_blocks_metadata->hash = getHexUIntLowercase(key); + new_blocks_metadata->version = getBlocksVersion(lock) + 1; + new_blocks_metadata->time = std::chrono::system_clock::now(); + + (*blocks_ptr) = new_blocks; + (*blocks_metadata_ptr) = new_blocks_metadata; + + updated = true; + } + else + { + new_blocks_metadata->hash = getBlocksHashKey(lock); + new_blocks_metadata->version = getBlocksVersion(lock); + new_blocks_metadata->time = std::chrono::system_clock::now(); + + (*blocks_metadata_ptr) = new_blocks_metadata; + } + } + return updated; +} + +void StorageLiveView::periodicRefreshTaskFunc() +{ + LOG_TRACE(log, "periodic refresh task"); + + std::lock_guard lock(mutex); + + if (hasActiveUsers(lock)) + scheduleNextPeriodicRefresh(lock); +} + +void StorageLiveView::scheduleNextPeriodicRefresh(std::lock_guard & lock) +{ + Seconds current_time = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()); + Seconds blocks_time = std::chrono::duration_cast(getBlocksTime(lock).time_since_epoch()); + + if ((current_time - periodic_live_view_refresh) >= blocks_time) + { + refreshImpl(lock); + blocks_time = std::chrono::duration_cast(getBlocksTime(lock).time_since_epoch()); + } + current_time = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()); + + auto next_refresh_time = blocks_time + periodic_live_view_refresh; + + if (current_time >= next_refresh_time) + periodic_refresh_task->scheduleAfter(0); + else + { + auto schedule_time = std::chrono::duration_cast (next_refresh_time - current_time); + periodic_refresh_task->scheduleAfter(static_cast(schedule_time.count())); + } } void registerStorageLiveView(StorageFactory & factory) diff --git a/src/Storages/LiveView/StorageLiveView.h b/src/Storages/LiveView/StorageLiveView.h index 831d9115708..fbb1dbe027c 100644 --- a/src/Storages/LiveView/StorageLiveView.h +++ b/src/Storages/LiveView/StorageLiveView.h @@ -63,78 +63,25 @@ public: const String & comment); ~StorageLiveView() override; - String getName() const override { return "LiveView"; } - bool isView() const override { return true; } - String getBlocksTableName() const - { - return getStorageID().table_name + "_blocks"; - } - StoragePtr getParentStorage() const; - ASTPtr getInnerQuery() const { return inner_query->clone(); } - ASTPtr getInnerSubQuery() const - { - if (inner_subquery) - return inner_subquery->clone(); - return nullptr; - } - ASTPtr getInnerBlocksQuery(); + String getName() const override { return "LiveView"; } + + bool isView() const override { return true; } /// It is passed inside the query and solved at its level. bool supportsSampling() const override { return true; } + bool supportsFinal() const override { return true; } NamesAndTypesList getVirtuals() const override; - /// Check we have any active readers - /// must be called with mutex locked - bool hasActiveUsers() - { - return active_ptr.use_count() > 1; - } - - /// Get blocks hash - /// must be called with mutex locked - String getBlocksHashKey() - { - if (*blocks_metadata_ptr) - return (*blocks_metadata_ptr)->hash; - return {}; - } - /// Get blocks version - /// must be called with mutex locked - UInt64 getBlocksVersion() - { - if (*blocks_metadata_ptr) - return (*blocks_metadata_ptr)->version; - return 0; - } - - /// Get blocks time - /// must be called with mutex locked - Time getBlocksTime() - { - if (*blocks_metadata_ptr) - return (*blocks_metadata_ptr)->time; - return {}; - } - - /// Reset blocks - /// must be called with mutex locked - void reset() - { - (*blocks_ptr).reset(); - if (*blocks_metadata_ptr) - (*blocks_metadata_ptr)->hash.clear(); - mergeable_blocks.reset(); - } - void checkTableCanBeDropped() const override; - void drop() override; - void startup() override; - void shutdown() override; - void refresh(bool grab_lock = true); + void drop() override; + + void startup() override; + + void shutdown() override; Pipe read( const Names & column_names, @@ -153,36 +100,88 @@ public: size_t max_block_size, size_t num_streams) override; - std::shared_ptr getBlocksPtr() { return blocks_ptr; } - MergeableBlocksPtr getMergeableBlocks() { return mergeable_blocks; } + ASTPtr getInnerQuery() const { return select_query_description.select_query->clone(); } - /// Collect mergeable blocks and their sample. Must be called holding mutex - MergeableBlocksPtr collectMergeableBlocks(ContextPtr context); - /// Complete query using input streams from mergeable blocks - QueryPipelineBuilder completeQuery(Pipes pipes); + /// Get blocks hash + /// must be called with mutex locked + String getBlocksHashKey(std::lock_guard &) + { + if (*blocks_metadata_ptr) + return (*blocks_metadata_ptr)->hash; + return {}; + } + /// Get blocks version + /// must be called with mutex locked + UInt64 getBlocksVersion(std::lock_guard &) + { + if (*blocks_metadata_ptr) + return (*blocks_metadata_ptr)->version; + return 0; + } - void setMergeableBlocks(MergeableBlocksPtr blocks) { mergeable_blocks = blocks; } - std::shared_ptr getActivePtr() { return active_ptr; } + void writeBlock(const Block & block, ContextPtr context); - /// Read new data blocks that store query result - bool getNewBlocks(); + void refresh(); + +private: + void refreshImpl(std::lock_guard & lock); + + String getBlocksTableName() const + { + return getStorageID().table_name + "_blocks"; + } Block getHeader() const; - /// convert blocks to input streams - static Pipes blocksToPipes(BlocksPtrs blocks, Block & sample_block); + StoragePtr getDependentTableStorage() const; - static void writeIntoLiveView( - StorageLiveView & live_view, - const Block & block, - ContextPtr context); + ASTPtr getInnerBlocksQuery(); + + /// Check we have any active readers + /// must be called with mutex locked + bool hasActiveUsers(std::lock_guard &) const + { + return active_ptr.use_count() > 1; + } + + /// Get blocks time + /// must be called with mutex locked + Time getBlocksTime(std::lock_guard &) + { + if (*blocks_metadata_ptr) + return (*blocks_metadata_ptr)->time; + return {}; + } + + /// Reset blocks + /// must be called with mutex locked + void reset(std::lock_guard &) + { + (*blocks_ptr).reset(); + if (*blocks_metadata_ptr) + (*blocks_metadata_ptr)->hash.clear(); + mergeable_blocks.reset(); + } + + /// Collect mergeable blocks and their sample. Must be called holding mutex + MergeableBlocksPtr collectMergeableBlocks(ContextPtr context, std::lock_guard & lock) const; + + /// Complete query using input streams from mergeable blocks + QueryPipelineBuilder completeQuery(Pipes pipes); + + /// Read new data blocks that store query result + bool getNewBlocks(std::lock_guard & lock); + + void periodicRefreshTaskFunc(); + + /// Must be called with mutex locked + void scheduleNextPeriodicRefresh(std::lock_guard & lock); + + SelectQueryDescription select_query_description; + + /// Query over the mergeable blocks to produce final result + ASTPtr inner_blocks_query; -private: - /// TODO move to common struct SelectQueryDescription - StorageID select_table_id = StorageID::createEmpty(); /// Will be initialized in constructor - ASTPtr inner_query; /// stored query : SELECT * FROM ( SELECT a FROM A) - ASTPtr inner_subquery; /// stored query's innermost subquery if any - ASTPtr inner_blocks_query; /// query over the mergeable blocks to produce final result ContextMutablePtr live_view_context; Poco::Logger * log; @@ -212,10 +211,6 @@ private: /// Periodic refresh task used when [PERIODIC] REFRESH is specified in create statement BackgroundSchedulePool::TaskHolder periodic_refresh_task; - void periodicRefreshTaskFunc(); - - /// Must be called with mutex locked - void scheduleNextPeriodicRefresh(); }; } diff --git a/src/Storages/SelectQueryDescription.h b/src/Storages/SelectQueryDescription.h index 28a0a186a07..6c608924508 100644 --- a/src/Storages/SelectQueryDescription.h +++ b/src/Storages/SelectQueryDescription.h @@ -23,6 +23,8 @@ struct SelectQueryDescription SelectQueryDescription() = default; SelectQueryDescription(const SelectQueryDescription & other); SelectQueryDescription & operator=(const SelectQueryDescription & other); + SelectQueryDescription(SelectQueryDescription && other) noexcept = default; + SelectQueryDescription & operator=(SelectQueryDescription && other) noexcept = default; }; } diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index e7468ae9d1e..619b1362bff 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -717,26 +717,6 @@ void StorageDistributed::read( /// This is a bug, it is possible only when there is no shards to query, and this is handled earlier. if (!query_plan.isInitialized()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Pipeline is not initialized"); - - if (local_context->getSettingsRef().allow_experimental_analyzer) - { - Planner planner(query_info.query_tree, - SelectQueryOptions(processed_stage), - query_info.planner_context, - PlannerConfiguration{.only_analyze = true}); - planner.buildQueryPlanIfNeeded(); - - auto expected_header = planner.getQueryPlan().getCurrentDataStream().header; - - auto rename_actions_dag = ActionsDAG::makeConvertingActions( - query_plan.getCurrentDataStream().header.getColumnsWithTypeAndName(), - expected_header.getColumnsWithTypeAndName(), - ActionsDAG::MatchColumnsMode::Position, - true /*ignore_constant_values*/); - auto rename_step = std::make_unique(query_plan.getCurrentDataStream(), std::move(rename_actions_dag)); - rename_step->setStepDescription("Change remote column names to local column names"); - query_plan.addStep(std::move(rename_step)); - } } From f73c49b0823be8b41cbaf49668ffafb20c0028d0 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Thu, 2 Feb 2023 19:43:29 +0100 Subject: [PATCH 300/717] Fixed style check --- src/Analyzer/QueryNode.cpp | 5 +++++ src/Storages/LiveView/StorageLiveView.cpp | 1 + 2 files changed, 6 insertions(+) diff --git a/src/Analyzer/QueryNode.cpp b/src/Analyzer/QueryNode.cpp index be773582745..06f72d32253 100644 --- a/src/Analyzer/QueryNode.cpp +++ b/src/Analyzer/QueryNode.cpp @@ -23,6 +23,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + QueryNode::QueryNode(ContextMutablePtr context_, SettingsChanges settings_changes_) : IQueryTreeNode(children_size) , context(std::move(context_)) diff --git a/src/Storages/LiveView/StorageLiveView.cpp b/src/Storages/LiveView/StorageLiveView.cpp index ba499412b33..83aa927d584 100644 --- a/src/Storages/LiveView/StorageLiveView.cpp +++ b/src/Storages/LiveView/StorageLiveView.cpp @@ -59,6 +59,7 @@ namespace ErrorCodes extern const int TABLE_WAS_NOT_DROPPED; extern const int QUERY_IS_NOT_SUPPORTED_IN_LIVE_VIEW; extern const int SUPPORT_IS_DISABLED; + extern const int UNSUPPORTED_METHOD; } namespace From 077555bd5de93a3fef20d9a2cacb257d03b46d3b Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Thu, 2 Feb 2023 19:44:54 +0100 Subject: [PATCH 301/717] Analyzer support build pushing to views chain --- .../Transforms/buildPushingToViewsChain.cpp | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/Processors/Transforms/buildPushingToViewsChain.cpp b/src/Processors/Transforms/buildPushingToViewsChain.cpp index ade056629db..0dcafd275eb 100644 --- a/src/Processors/Transforms/buildPushingToViewsChain.cpp +++ b/src/Processors/Transforms/buildPushingToViewsChain.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -326,9 +327,13 @@ Chain buildPushingToViewsChain( query = view_metadata_snapshot->getSelectQuery().inner_query; target_name = inner_table_id.getFullTableName(); + Block header; + /// Get list of columns we get from select query. - auto header = InterpreterSelectQuery(query, select_context, SelectQueryOptions().analyze()) - .getSampleBlock(); + if (select_context->getSettingsRef().allow_experimental_analyzer) + header = InterpreterSelectQueryAnalyzer::getSampleBlock(query, select_context); + else + header = InterpreterSelectQuery(query, select_context, SelectQueryOptions().analyze()).getSampleBlock(); /// Insert only columns returned by select. Names insert_columns; @@ -477,8 +482,18 @@ static QueryPipeline process(Block block, ViewRuntimeData & view, const ViewsDat std::move(block), views_data.source_storage->getVirtuals())); - InterpreterSelectQuery select(view.query, local_context, SelectQueryOptions()); - auto pipeline = select.buildQueryPipeline(); + QueryPipelineBuilder pipeline; + + if (local_context->getSettingsRef().allow_experimental_analyzer) + { + InterpreterSelectQueryAnalyzer interpreter(view.query, local_context, SelectQueryOptions()); + pipeline = interpreter.buildQueryPipeline(); + } + else + { InterpreterSelectQuery interpreter(view.query, local_context, SelectQueryOptions()); + pipeline = interpreter.buildQueryPipeline(); + } + pipeline.resize(1); pipeline.dropTotalsAndExtremes(); @@ -635,7 +650,7 @@ PushingToLiveViewSink::PushingToLiveViewSink(const Block & header, StorageLiveVi void PushingToLiveViewSink::consume(Chunk chunk) { Progress local_progress(chunk.getNumRows(), chunk.bytes(), 0); - StorageLiveView::writeIntoLiveView(live_view, getHeader().cloneWithColumns(chunk.detachColumns()), context); + live_view.writeBlock(getHeader().cloneWithColumns(chunk.detachColumns()), context); if (auto process = context->getProcessListElement()) process->updateProgressIn(local_progress); From af38660cf55d44a63c87761320adf5fce51a299e Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sat, 4 Feb 2023 16:51:43 +0100 Subject: [PATCH 302/717] Analyzer support ARRAY JOIN with Nested --- src/Analyzer/AggregationUtils.cpp | 35 +- src/Analyzer/Passes/QueryAnalysisPass.cpp | 418 +++++++++++------- src/Analyzer/Utils.cpp | 56 ++- src/Analyzer/Utils.h | 9 + src/Functions/nested.cpp | 171 +++++++ src/Planner/CollectColumnIdentifiers.cpp | 4 +- src/Planner/Planner.cpp | 7 + src/Planner/PlannerJoinTree.cpp | 58 ++- src/Planner/PlannerJoins.cpp | 19 +- src/Planner/Utils.cpp | 24 + src/Planner/Utils.h | 3 + .../00855_join_with_array_join.reference | 4 + .../00855_join_with_array_join.sql | 6 +- ...77_analyzer_array_join_with_join.reference | 24 +- 14 files changed, 601 insertions(+), 237 deletions(-) create mode 100644 src/Functions/nested.cpp diff --git a/src/Analyzer/AggregationUtils.cpp b/src/Analyzer/AggregationUtils.cpp index a73df87f9c2..e2d4ada4f4b 100644 --- a/src/Analyzer/AggregationUtils.cpp +++ b/src/Analyzer/AggregationUtils.cpp @@ -2,6 +2,7 @@ #include #include +#include namespace DB { @@ -74,41 +75,9 @@ void assertNoAggregateFunctionNodes(const QueryTreeNodePtr & node, const String visitor.visit(node); } -namespace -{ - -class ValidateGroupingFunctionNodesVisitor : public ConstInDepthQueryTreeVisitor -{ -public: - explicit ValidateGroupingFunctionNodesVisitor(String assert_no_grouping_function_place_message_) - : assert_no_grouping_function_place_message(std::move(assert_no_grouping_function_place_message_)) - {} - - void visitImpl(const QueryTreeNodePtr & node) - { - auto * function_node = node->as(); - if (function_node && function_node->getFunctionName() == "grouping") - throw Exception(ErrorCodes::ILLEGAL_AGGREGATION, - "GROUPING function {} is found {} in query", - function_node->formatASTForErrorMessage(), - assert_no_grouping_function_place_message); - } - - static bool needChildVisit(const QueryTreeNodePtr &, const QueryTreeNodePtr & child_node) - { - return !(child_node->getNodeType() == QueryTreeNodeType::QUERY || child_node->getNodeType() == QueryTreeNodeType::UNION); - } - -private: - String assert_no_grouping_function_place_message; -}; - -} - void assertNoGroupingFunction(const QueryTreeNodePtr & node, const String & assert_no_grouping_function_place_message) { - ValidateGroupingFunctionNodesVisitor visitor(assert_no_grouping_function_place_message); - visitor.visit(node); + assertNoFunction(node, "grouping", ErrorCodes::ILLEGAL_AGGREGATION, "GROUPING", assert_no_grouping_function_place_message); } } diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index eec3e8646c2..d366a017642 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -108,6 +109,7 @@ namespace ErrorCodes extern const int ALIAS_REQUIRED; extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int ILLEGAL_PREWHERE; } /** Query analyzer implementation overview. Please check documentation in QueryAnalysisPass.h before. @@ -1179,7 +1181,13 @@ private: QueryTreeNodePtr tryResolveIdentifierFromTableColumns(const IdentifierLookup & identifier_lookup, IdentifierResolveScope & scope); - static bool tryBindIdentifierToTableExpression(const IdentifierLookup & identifier_lookup, const QueryTreeNodePtr & table_expression_node, const IdentifierResolveScope & scope); + static bool tryBindIdentifierToTableExpression(const IdentifierLookup & identifier_lookup, + const QueryTreeNodePtr & table_expression_node, + const IdentifierResolveScope & scope); + + static bool tryBindIdentifierToTableExpressions(const IdentifierLookup & identifier_lookup, + const QueryTreeNodePtr & table_expression_node, + const IdentifierResolveScope & scope); QueryTreeNodePtr tryResolveIdentifierFromTableExpression(const IdentifierLookup & identifier_lookup, const QueryTreeNodePtr & table_expression_node, @@ -1189,6 +1197,10 @@ private: const QueryTreeNodePtr & table_expression_node, IdentifierResolveScope & scope); + QueryTreeNodePtr tryResolveExpressionFromArrayJoinExpressions(const QueryTreeNodePtr & resolved_expression, + const QueryTreeNodePtr & table_expression_node, + IdentifierResolveScope & scope); + QueryTreeNodePtr tryResolveIdentifierFromArrayJoin(const IdentifierLookup & identifier_lookup, const QueryTreeNodePtr & table_expression_node, IdentifierResolveScope & scope); @@ -2595,6 +2607,25 @@ bool QueryAnalyzer::tryBindIdentifierToTableExpression(const IdentifierLookup & return false; } +bool QueryAnalyzer::tryBindIdentifierToTableExpressions(const IdentifierLookup & identifier_lookup, + const QueryTreeNodePtr & table_expression_node_to_ignore, + const IdentifierResolveScope & scope) +{ + bool can_bind_identifier_to_table_expression = false; + + for (const auto & [table_expression_node, _] : scope.table_expression_node_to_data) + { + if (table_expression_node.get() == table_expression_node_to_ignore.get()) + continue; + + can_bind_identifier_to_table_expression = tryBindIdentifierToTableExpression(identifier_lookup, table_expression_node, scope); + if (can_bind_identifier_to_table_expression) + break; + } + + return can_bind_identifier_to_table_expression; +} + QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromTableExpression(const IdentifierLookup & identifier_lookup, const QueryTreeNodePtr & table_expression_node, IdentifierResolveScope & scope) @@ -2637,8 +2668,8 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromTableExpression(const Id auto resolve_identifier_from_storage_or_throw = [&](size_t identifier_column_qualifier_parts) -> QueryTreeNodePtr { - auto identifier_view = IdentifierView(identifier); - identifier_view.popFirst(identifier_column_qualifier_parts); + auto identifier_without_column_qualifier = identifier; + identifier_without_column_qualifier.popFirst(identifier_column_qualifier_parts); /** Compound identifier cannot be resolved directly from storage if storage is not table. * @@ -2649,42 +2680,90 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromTableExpression(const Id * Here there is no column with name test_subquery.compound_expression.value, and additional wrap in tuple element is required. */ - ColumnNodePtr result_column; - bool compound_identifier = identifier_view.getPartsSize() > 1; + QueryTreeNodePtr result_expression; bool match_full_identifier = false; - auto it = table_expression_data.column_name_to_column_node.find(std::string(identifier_view.getFullName())); + auto it = table_expression_data.column_name_to_column_node.find(identifier_without_column_qualifier.getFullName()); if (it != table_expression_data.column_name_to_column_node.end()) { match_full_identifier = true; - result_column = it->second; + result_expression = it->second; } else { - it = table_expression_data.column_name_to_column_node.find(std::string(identifier_view.at(0))); - + it = table_expression_data.column_name_to_column_node.find(identifier_without_column_qualifier.at(0)); if (it != table_expression_data.column_name_to_column_node.end()) - result_column = it->second; + result_expression = it->second; } - QueryTreeNodePtr result_expression = result_column; bool clone_is_needed = true; String table_expression_source = table_expression_data.table_expression_description; if (!table_expression_data.table_expression_name.empty()) table_expression_source += " with name " + table_expression_data.table_expression_name; - if (result_column && !match_full_identifier && compound_identifier) + if (result_expression && !match_full_identifier && identifier_without_column_qualifier.isCompound()) { size_t identifier_bind_size = identifier_column_qualifier_parts + 1; result_expression = tryResolveIdentifierFromCompoundExpression(identifier_lookup.identifier, identifier_bind_size, - result_column, + result_expression, table_expression_source, scope); clone_is_needed = false; } + if (!result_expression) + { + QueryTreeNodes nested_column_nodes; + DataTypes nested_types; + Array nested_names_array; + + for (auto & [column_name, _] : table_expression_data.column_names_and_types) + { + Identifier column_name_identifier_without_last_part(column_name); + auto column_name_identifier_last_part = column_name_identifier_without_last_part.getParts().back(); + column_name_identifier_without_last_part.popLast(); + + if (identifier_without_column_qualifier.getFullName() != column_name_identifier_without_last_part.getFullName()) + continue; + + auto column_node_it = table_expression_data.column_name_to_column_node.find(column_name); + if (column_node_it == table_expression_data.column_name_to_column_node.end()) + continue; + + const auto & column_node = column_node_it->second; + const auto & column_type = column_node->getColumnType(); + const auto * column_type_array = typeid_cast(column_type.get()); + if (!column_type_array) + continue; + + nested_column_nodes.push_back(column_node); + nested_types.push_back(column_type_array->getNestedType()); + nested_names_array.push_back(Field(std::move(column_name_identifier_last_part))); + } + + if (!nested_types.empty()) + { + auto nested_function_node = std::make_shared("nested"); + auto & nested_function_node_arguments = nested_function_node->getArguments().getNodes(); + + auto nested_function_names_array_type = std::make_shared(std::make_shared()); + auto nested_function_names_constant_node = std::make_shared(std::move(nested_names_array), + std::move(nested_function_names_array_type)); + nested_function_node_arguments.push_back(std::move(nested_function_names_constant_node)); + nested_function_node_arguments.insert(nested_function_node_arguments.end(), + nested_column_nodes.begin(), + nested_column_nodes.end()); + + auto nested_function = FunctionFactory::instance().get(nested_function_node->getFunctionName(), scope.context); + nested_function_node->resolveAsFunction(nested_function->build(nested_function_node->getArgumentColumns())); + + clone_is_needed = false; + result_expression = std::move(nested_function_node); + } + } + if (!result_expression) { std::unordered_set valid_identifiers; @@ -2715,32 +2794,9 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromTableExpression(const Id if (qualified_identifier_with_removed_part.empty()) break; - IdentifierLookup bind_to_aliases_identifier_lookup = {qualified_identifier_with_removed_part, IdentifierLookupContext::EXPRESSION}; - if (tryBindIdentifierToAliases(bind_to_aliases_identifier_lookup, scope)) - break; - - bool can_remove_qualificator = true; - - if (table_expression_data.should_qualify_columns) - { - for (auto & table_expression_to_check_data : scope.table_expression_node_to_data) - { - const auto & table_expression_to_check = table_expression_to_check_data.first; - if (table_expression_to_check.get() == table_expression_node.get()) - continue; - - IdentifierLookup column_identifier_lookup{qualified_identifier_with_removed_part, IdentifierLookupContext::EXPRESSION}; - bool can_bind_identifier_to_table_expression = tryBindIdentifierToTableExpression(column_identifier_lookup, table_expression_to_check, scope); - - if (can_bind_identifier_to_table_expression) - { - can_remove_qualificator = false; - break; - } - } - } - - if (!can_remove_qualificator) + IdentifierLookup column_identifier_lookup = {qualified_identifier_with_removed_part, IdentifierLookupContext::EXPRESSION}; + if (tryBindIdentifierToAliases(column_identifier_lookup, scope) || + tryBindIdentifierToTableExpressions(column_identifier_lookup, table_expression_node, scope)) break; qualified_identifier = std::move(qualified_identifier_with_removed_part); @@ -2917,6 +2973,79 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromJoin(const IdentifierLoo return resolved_identifier; } +QueryTreeNodePtr QueryAnalyzer::tryResolveExpressionFromArrayJoinExpressions(const QueryTreeNodePtr & resolved_expression, + const QueryTreeNodePtr & table_expression_node, + IdentifierResolveScope & scope) +{ + const auto & array_join_node = table_expression_node->as(); + const auto & array_join_column_expressions_list = array_join_node.getJoinExpressions(); + const auto & array_join_column_expressions_nodes = array_join_column_expressions_list.getNodes(); + + QueryTreeNodePtr array_join_resolved_expression; + + /** Special case when qualified or unqualified identifier point to array join expression without alias. + * + * CREATE TABLE test_table (id UInt64, value String, value_array Array(UInt8)) ENGINE=TinyLog; + * SELECT id, value, value_array, test_table.value_array, default.test_table.value_array FROM test_table ARRAY JOIN value_array; + * + * value_array, test_table.value_array, default.test_table.value_array must be resolved into array join expression. + */ + for (const auto & array_join_column_expression : array_join_column_expressions_nodes) + { + auto & array_join_column_expression_typed = array_join_column_expression->as(); + if (array_join_column_expression_typed.hasAlias()) + continue; + + auto & array_join_column_inner_expression = array_join_column_expression_typed.getExpressionOrThrow(); + auto * array_join_column_inner_expression_function = array_join_column_inner_expression->as(); + + if (array_join_column_inner_expression_function && + array_join_column_inner_expression_function->getFunctionName() == "nested" && + array_join_column_inner_expression_function->getArguments().getNodes().size() > 1 && + isTuple(array_join_column_expression_typed.getResultType())) + { + const auto & nested_function_arguments = array_join_column_inner_expression_function->getArguments().getNodes(); + size_t nested_function_arguments_size = nested_function_arguments.size(); + + const auto & nested_keys_names_constant_node = nested_function_arguments[0]->as(); + const auto & nested_keys_names = nested_keys_names_constant_node.getValue().get(); + size_t nested_keys_names_size = nested_keys_names.size(); + + if (nested_keys_names_size == nested_function_arguments_size - 1) + { + for (size_t i = 1; i < nested_function_arguments_size; ++i) + { + if (!nested_function_arguments[i]->isEqual(*resolved_expression)) + continue; + + auto array_join_column = std::make_shared(array_join_column_expression_typed.getColumn(), + array_join_column_expression_typed.getColumnSource()); + + const auto & nested_key_name = nested_keys_names[i - 1].get(); + Identifier nested_identifier = Identifier(nested_key_name); + auto tuple_element_function = wrapExpressionNodeInTupleElement(array_join_column, nested_identifier); + resolveFunction(tuple_element_function, scope); + + array_join_resolved_expression = std::move(tuple_element_function); + break; + } + } + } + + if (array_join_resolved_expression) + break; + + if (array_join_column_inner_expression->isEqual(*resolved_expression)) + { + array_join_resolved_expression = std::make_shared(array_join_column_expression_typed.getColumn(), + array_join_column_expression_typed.getColumnSource()); + break; + } + } + + return array_join_resolved_expression; +} + QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromArrayJoin(const IdentifierLookup & identifier_lookup, const QueryTreeNodePtr & table_expression_node, IdentifierResolveScope & scope) @@ -2932,43 +3061,27 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromArrayJoin(const Identifi /** Allow JOIN with USING with ARRAY JOIN. * - * SELECT * FROM test_table_1 AS t1 ARRAY JOIN [1,2,3] AS id INNER JOIN test_table_2 AS t2 ON t1.id = t2.id - * SELECT * FROM test_table_1 AS t1 ARRAY JOIN t1.id AS id INNER JOIN test_table_2 AS t2 ON t1.id = t2.id + * SELECT * FROM test_table_1 AS t1 ARRAY JOIN [1,2,3] AS id INNER JOIN test_table_2 AS t2 USING (id); + * SELECT * FROM test_table_1 AS t1 ARRAY JOIN t1.id AS id INNER JOIN test_table_2 AS t2 USING (id); */ for (const auto & array_join_column_expression : array_join_column_expressions_nodes) { auto & array_join_column_expression_typed = array_join_column_expression->as(); - if (identifier_lookup.identifier.isShort() && - array_join_column_expression_typed.getAlias() == identifier_lookup.identifier.getFullName()) - return array_join_column_expression; + if (array_join_column_expression_typed.getAlias() == identifier_lookup.identifier.getFullName()) + { + auto array_join_column = std::make_shared(array_join_column_expression_typed.getColumn(), + array_join_column_expression_typed.getColumnSource()); + return array_join_column; + } } if (!resolved_identifier) return nullptr; - /** Special case when qualified or unqualified identifier point to array join expression without alias. - * - * CREATE TABLE test_table (id UInt64, value String, value_array Array(UInt8)) ENGINE=TinyLog; - * SELECT id, value, value_array, test_table.value_array, default.test_table.value_array FROM test_table ARRAY JOIN value_array; - * - * value_array, test_table.value_array, default.test_table.value_array must be resolved into array join expression. - */ - for (const auto & array_join_column_expression : array_join_column_expressions_nodes) - { - auto & array_join_column_expression_typed = array_join_column_expression->as(); - - if (array_join_column_expression_typed.hasAlias()) - continue; - - auto & array_join_column_inner_expression = array_join_column_expression_typed.getExpressionOrThrow(); - if (array_join_column_inner_expression.get() == resolved_identifier.get() || - array_join_column_inner_expression->isEqual(*resolved_identifier)) - { - resolved_identifier = array_join_column_expression; - break; - } - } + auto array_join_resolved_expression = tryResolveExpressionFromArrayJoinExpressions(resolved_identifier, table_expression_node, scope); + if (array_join_resolved_expression) + resolved_identifier = std::move(array_join_resolved_expression); return resolved_identifier; } @@ -3313,26 +3426,13 @@ void QueryAnalyzer::qualifyColumnNodesWithProjectionNames(const QueryTreeNodes & /// Iterate over additional column qualifications and apply them if needed for (size_t i = 0; i < additional_column_qualification_parts_size; ++i) { - bool need_to_qualify = false; auto identifier_to_check = Identifier(column_qualified_identifier_parts); - IdentifierLookup lookup{identifier_to_check, IdentifierLookupContext::EXPRESSION}; + IdentifierLookup identifier_lookup{identifier_to_check, IdentifierLookupContext::EXPRESSION}; + bool need_to_qualify = table_expression_data.should_qualify_columns; + if (need_to_qualify) + need_to_qualify = tryBindIdentifierToTableExpressions(identifier_lookup, table_expression_node, scope); - if (table_expression_data.should_qualify_columns) - { - for (const auto & scope_table_expression_data : scope.table_expression_node_to_data) - { - if (scope_table_expression_data.first.get() == table_expression_node.get()) - continue; - - if (tryBindIdentifierToTableExpression(lookup, scope_table_expression_data.first, scope)) - { - need_to_qualify = true; - break; - } - } - } - - if (tryBindIdentifierToAliases(lookup, scope)) + if (tryBindIdentifierToAliases(identifier_lookup, scope)) need_to_qualify = true; if (need_to_qualify) @@ -3575,55 +3675,35 @@ QueryAnalyzer::QueryTreeNodesWithNames QueryAnalyzer::resolveUnqualifiedMatcher( auto table_expressions_stack = buildTableExpressionsStack(nearest_query_scope_query_node->getJoinTree()); std::vector table_expressions_column_nodes_with_names_stack; - std::unordered_set left_table_expression_column_names_to_skip; - std::unordered_set right_table_expression_column_names_to_skip; + std::unordered_set table_expression_column_names_to_skip; for (auto & table_expression : table_expressions_stack) { + bool table_expression_in_resolve_process = scope.table_expressions_in_resolve_process.contains(table_expression.get()); + if (auto * array_join_node = table_expression->as()) { if (table_expressions_column_nodes_with_names_stack.empty()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected at least 1 table expressions on stack before ARRAY JOIN processing"); + if (table_expression_in_resolve_process) + continue; + auto & table_expression_column_nodes_with_names = table_expressions_column_nodes_with_names_stack.back(); - const auto & array_join_column_list = array_join_node->getJoinExpressions(); - const auto & array_join_column_nodes = array_join_column_list.getNodes(); - - /** Special case with ARRAY JOIN column without alias. - * - * CREATE TABLE test_table (id UInt64, value String, value_array Array(UInt8)) ENGINE=TinyLog; - * SELECT * FROM test_table ARRAY JOIN value_array; - * - * In matched columns `value_array` must be resolved into array join column. - */ - for (const auto & array_join_column_node : array_join_column_nodes) + for (auto & [table_expression_column_node, _] : table_expression_column_nodes_with_names) { - if (array_join_column_node->hasAlias()) - continue; - - auto array_join_column_inner_expression = array_join_column_node->as().getExpressionOrThrow(); - if (array_join_column_inner_expression->getNodeType() != QueryTreeNodeType::COLUMN) - continue; - - for (auto & table_expressions_column_node_with_name : table_expression_column_nodes_with_names) - { - auto & table_expression_column_node = table_expressions_column_node_with_name.first; - - if (table_expression_column_node.get() == array_join_column_inner_expression.get() || - table_expression_column_node->isEqual(*array_join_column_inner_expression)) - { - table_expression_column_node = array_join_column_node; - } - } + auto array_join_resolved_expression = tryResolveExpressionFromArrayJoinExpressions(table_expression_column_node, + table_expression, + scope); + if (array_join_resolved_expression) + table_expression_column_node = std::move(array_join_resolved_expression); } continue; } - bool table_expression_in_resolve_process = scope.table_expressions_in_resolve_process.contains(table_expression.get()); - auto * join_node = table_expression->as(); if (join_node) @@ -3640,8 +3720,7 @@ QueryAnalyzer::QueryTreeNodesWithNames QueryAnalyzer::resolveUnqualifiedMatcher( auto left_table_expression_columns = std::move(table_expressions_column_nodes_with_names_stack.back()); table_expressions_column_nodes_with_names_stack.pop_back(); - left_table_expression_column_names_to_skip.clear(); - right_table_expression_column_names_to_skip.clear(); + table_expression_column_names_to_skip.clear(); QueryTreeNodesWithNames matched_expression_nodes_with_column_names; @@ -3656,13 +3735,13 @@ QueryAnalyzer::QueryTreeNodesWithNames QueryAnalyzer::resolveUnqualifiedMatcher( for (auto & join_using_node : join_using_list.getNodes()) { - auto & column_node = join_using_node->as(); - const auto & using_column_name = column_node.getColumnName(); + auto & join_using_column_node = join_using_node->as(); + const auto & join_using_column_name = join_using_column_node.getColumnName(); - if (!matcher_node_typed.isMatchingColumn(using_column_name)) + if (!matcher_node_typed.isMatchingColumn(join_using_column_name)) continue; - const auto & join_using_column_nodes_list = column_node.getExpressionOrThrow()->as(); + const auto & join_using_column_nodes_list = join_using_column_node.getExpressionOrThrow()->as(); const auto & join_using_column_nodes = join_using_column_nodes_list.getNodes(); QueryTreeNodePtr matched_column_node; @@ -3672,40 +3751,17 @@ QueryAnalyzer::QueryTreeNodesWithNames QueryAnalyzer::resolveUnqualifiedMatcher( else matched_column_node = join_using_column_nodes.at(0); - /** It is possible that in USING there is JOIN with array joined column. - * SELECT * FROM (SELECT [0] AS value) AS t1 ARRAY JOIN value AS id INNER JOIN test_table USING (id); - * In such example match `value` column from t1, and all columns from test_table except `id`. - * - * SELECT * FROM (SELECT [0] AS id) AS t1 ARRAY JOIN id INNER JOIN test_table USING (id); - * In such example, match `id` column from ARRAY JOIN, and all columns from test_table except `id`. - * - * SELECT * FROM (SELECT [0] AS id) AS t1 ARRAY JOIN id AS id INNER JOIN test_table USING (id); - * In such example match `id` column from t1, and all columns from test_table except `id`. - * - * SELECT * FROM (SELECT [0] AS id) AS t1 ARRAY JOIN [1] AS id INNER JOIN test_table USING (id); - * In such example match `id` column from t1, and all columns from test_table except `id`. - */ - auto matched_column_source = matched_column_node->as().getColumnSource(); + matched_column_node = matched_column_node->clone(); + matched_column_node->as().setColumnType(join_using_column_node.getResultType()); - if (matched_column_source->getNodeType() == QueryTreeNodeType::ARRAY_JOIN && matched_column_node->hasAlias()) - { - if (isRight(join_node->getKind())) - left_table_expression_column_names_to_skip.insert(using_column_name); - else - right_table_expression_column_names_to_skip.insert(using_column_name); - } - else - { - left_table_expression_column_names_to_skip.insert(using_column_name); - right_table_expression_column_names_to_skip.insert(using_column_name); - matched_expression_nodes_with_column_names.emplace_back(std::move(matched_column_node), using_column_name); - } + table_expression_column_names_to_skip.insert(join_using_column_name); + matched_expression_nodes_with_column_names.emplace_back(std::move(matched_column_node), join_using_column_name); } } for (auto && left_table_column_with_name : left_table_expression_columns) { - if (left_table_expression_column_names_to_skip.contains(left_table_column_with_name.second)) + if (table_expression_column_names_to_skip.contains(left_table_column_with_name.second)) continue; matched_expression_nodes_with_column_names.push_back(std::move(left_table_column_with_name)); @@ -3713,7 +3769,7 @@ QueryAnalyzer::QueryTreeNodesWithNames QueryAnalyzer::resolveUnqualifiedMatcher( for (auto && right_table_column_with_name : right_table_expression_columns) { - if (right_table_expression_column_names_to_skip.contains(right_table_column_with_name.second)) + if (table_expression_column_names_to_skip.contains(right_table_column_with_name.second)) continue; matched_expression_nodes_with_column_names.push_back(std::move(right_table_column_with_name)); @@ -5854,6 +5910,10 @@ void QueryAnalyzer::resolveArrayJoin(QueryTreeNodePtr & array_join_node, Identif auto & array_join_nodes = array_join_node_typed.getJoinExpressions().getNodes(); size_t array_join_nodes_size = array_join_nodes.size(); + if (array_join_nodes_size == 0) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "ARRAY JOIN requires at least single expression"); + std::vector array_join_column_expressions; array_join_column_expressions.reserve(array_join_nodes_size); @@ -5870,18 +5930,28 @@ void QueryAnalyzer::resolveArrayJoin(QueryTreeNodePtr & array_join_node, Identif /// Add array join expression into scope expressions_visitor.visit(array_join_expression); + std::string identifier_full_name; + + if (auto * identifier_node = array_join_expression->as()) + identifier_full_name = identifier_node->getIdentifier().getFullName(); + resolveExpressionNode(array_join_expression, scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); auto result_type = array_join_expression->getResultType(); + bool is_array_type = isArray(result_type); + bool is_map_type = isMap(result_type); - if (!isArray(result_type)) + if (!is_array_type && !is_map_type) throw Exception(ErrorCodes::TYPE_MISMATCH, - "ARRAY JOIN {} requires expression {} with Array type. Actual {}. In scope {}", + "ARRAY JOIN {} requires expression {} with Array or Map type. Actual {}. In scope {}", array_join_node_typed.formatASTForErrorMessage(), array_join_expression->formatASTForErrorMessage(), result_type->getName(), scope.scope_node->formatASTForErrorMessage()); + if (is_map_type) + result_type = assert_cast(*result_type).getNestedType(); + result_type = assert_cast(*result_type).getNestedType(); String array_join_column_name; @@ -5894,6 +5964,10 @@ void QueryAnalyzer::resolveArrayJoin(QueryTreeNodePtr & array_join_node, Identif { array_join_column_name = array_join_expression_inner_column->getColumnName(); } + else if (!identifier_full_name.empty()) + { + array_join_column_name = identifier_full_name; + } else { array_join_column_name = "__array_join_expression_" + std::to_string(array_join_expressions_counter); @@ -5925,11 +5999,17 @@ void QueryAnalyzer::resolveArrayJoin(QueryTreeNodePtr & array_join_node, Identif */ for (size_t i = 0; i < array_join_nodes_size; ++i) { - auto & array_join_expression = array_join_nodes[i]; - array_join_expression = std::move(array_join_column_expressions[i]); - auto it = scope.alias_name_to_expression_node.find(array_join_expression->getAlias()); + auto & array_join_column_expression = array_join_nodes[i]; + array_join_column_expression = std::move(array_join_column_expressions[i]); + + auto it = scope.alias_name_to_expression_node.find(array_join_column_expression->getAlias()); if (it != scope.alias_name_to_expression_node.end()) - it->second = array_join_nodes[i]; + { + auto & array_join_column_expression_typed = array_join_column_expression->as(); + auto array_join_column = std::make_shared(array_join_column_expression_typed.getColumn(), + array_join_column_expression_typed.getColumnSource()); + it->second = std::move(array_join_column); + } } } @@ -5984,6 +6064,13 @@ void QueryAnalyzer::resolveJoin(QueryTreeNodePtr & join_node, IdentifierResolveS identifier_full_name, scope.scope_node->formatASTForErrorMessage()); + if (result_left_table_expression->getNodeType() != QueryTreeNodeType::COLUMN) + throw Exception(ErrorCodes::UNSUPPORTED_METHOD, + "JOIN {} using identifier '{}' must be resolved into column node from left table expression. In scope {}", + join_node_typed.formatASTForErrorMessage(), + identifier_full_name, + scope.scope_node->formatASTForErrorMessage()); + auto result_right_table_expression = tryResolveIdentifierFromJoinTreeNode(identifier_lookup, join_node_typed.getRightTableExpression(), scope); if (!result_right_table_expression) throw Exception(ErrorCodes::UNKNOWN_IDENTIFIER, @@ -5992,6 +6079,13 @@ void QueryAnalyzer::resolveJoin(QueryTreeNodePtr & join_node, IdentifierResolveS identifier_full_name, scope.scope_node->formatASTForErrorMessage()); + if (result_right_table_expression->getNodeType() != QueryTreeNodeType::COLUMN) + throw Exception(ErrorCodes::UNSUPPORTED_METHOD, + "JOIN {} using identifier '{}' must be resolved into column node from right table expression. In scope {}", + join_node_typed.formatASTForErrorMessage(), + identifier_full_name, + scope.scope_node->formatASTForErrorMessage()); + auto expression_types = DataTypes{result_left_table_expression->getResultType(), result_right_table_expression->getResultType()}; DataTypePtr common_type = tryGetLeastSupertype(expression_types); @@ -6164,8 +6258,7 @@ public: bool needChildVisit(const QueryTreeNodePtr &, const QueryTreeNodePtr & child_node) { - auto * child_function_node = child_node->as(); - if (child_function_node) + if (auto * child_function_node = child_node->as()) { if (child_function_node->isAggregateFunction()) return false; @@ -6499,6 +6592,13 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier if (query_node_typed.isGroupByAll()) expandGroupByAll(query_node_typed); + if (query_node_typed.hasPrewhere()) + assertNoFunction(query_node_typed.getPrewhere(), + "arrayJoin", + ErrorCodes::ILLEGAL_PREWHERE, + "ARRAY JOIN", + "in PREWHERE"); + /** Validate aggregates * * 1. Check that there are no aggregate functions and GROUPING function in JOIN TREE, WHERE, PREWHERE, in another aggregate functions. diff --git a/src/Analyzer/Utils.cpp b/src/Analyzer/Utils.cpp index 790bfe0369a..5778930c0f5 100644 --- a/src/Analyzer/Utils.cpp +++ b/src/Analyzer/Utils.cpp @@ -10,10 +10,12 @@ #include +#include #include +#include +#include #include #include -#include #include #include #include @@ -374,4 +376,56 @@ bool nestedIdentifierCanBeResolved(const DataTypePtr & compound_type, Identifier return true; } +namespace +{ + +class ValidateFunctionNodesVisitor : public ConstInDepthQueryTreeVisitor +{ +public: + explicit ValidateFunctionNodesVisitor(std::string_view function_name_, + int exception_code_, + std::string_view exception_function_name_, + std::string_view exception_place_message_) + : function_name(function_name_) + , exception_code(exception_code_) + , exception_function_name(exception_function_name_) + , exception_place_message(exception_place_message_) + {} + + void visitImpl(const QueryTreeNodePtr & node) + { + auto * function_node = node->as(); + if (function_node && function_node->getFunctionName() == function_name) + throw Exception(exception_code, + "{} function {} is found {} in query", + exception_function_name, + function_node->formatASTForErrorMessage(), + exception_place_message); + } + + static bool needChildVisit(const QueryTreeNodePtr &, const QueryTreeNodePtr & child_node) + { + auto child_node_type = child_node->getNodeType(); + return !(child_node_type == QueryTreeNodeType::QUERY || child_node_type == QueryTreeNodeType::UNION); + } + +private: + std::string_view function_name; + int exception_code = 0; + std::string_view exception_function_name; + std::string_view exception_place_message; +}; + +} + +void assertNoFunction(const QueryTreeNodePtr & node, + std::string_view function_name, + int exception_code, + std::string_view exception_function_name, + std::string_view exception_place_message) +{ + ValidateFunctionNodesVisitor visitor(function_name, exception_code, exception_function_name, exception_place_message); + visitor.visit(node); +} + } diff --git a/src/Analyzer/Utils.h b/src/Analyzer/Utils.h index 19800e88d2b..2996d084c38 100644 --- a/src/Analyzer/Utils.h +++ b/src/Analyzer/Utils.h @@ -42,4 +42,13 @@ QueryTreeNodes buildTableExpressionsStack(const QueryTreeNodePtr & join_tree_nod */ bool nestedIdentifierCanBeResolved(const DataTypePtr & compound_type, IdentifierView nested_identifier); +/** Assert that there are no function with specified function name in node children. + * Do not visit subqueries. + */ +void assertNoFunction(const QueryTreeNodePtr & node, + std::string_view function_name, + int exception_code, + std::string_view exception_function_name, + std::string_view exception_place_message); + } diff --git a/src/Functions/nested.cpp b/src/Functions/nested.cpp new file mode 100644 index 00000000000..93a440633c4 --- /dev/null +++ b/src/Functions/nested.cpp @@ -0,0 +1,171 @@ +#include + +#include + +#include +#include +#include + +#include +#include +#include + +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int SIZES_OF_ARRAYS_DOESNT_MATCH; +} + +namespace +{ + +class FunctionNested : public IFunction +{ +public: + static constexpr auto name = "nested"; + + static FunctionPtr create(ContextPtr) + { + return std::make_shared(); + } + + String getName() const override + { + return name; + } + + bool isVariadic() const override { return true; } + + size_t getNumberOfArguments() const override + { + return 0; + } + + bool useDefaultImplementationForConstants() const override + { + return true; + } + + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override + { + return {0}; + } + + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + size_t arguments_size = arguments.size(); + if (arguments_size < 2) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Number of arguments for function {} doesn't match: passed {}, should be at least 2", + getName(), + arguments_size); + + Names nested_names = extractNestedNames(arguments[0].column); + if (nested_names.empty()) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "First argument for function {} must be constant column with array of strings", + getName()); + + if (nested_names.size() != arguments_size - 1) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Size of nested names array for function {} does not match arrays arguments size. Actual {}. Expected {}", + getName(), + nested_names.size(), + arguments_size - 1); + + DataTypes nested_types; + nested_types.reserve(arguments_size); + + for (size_t i = 1; i < arguments_size; ++i) + { + const auto & argument = arguments[i]; + const auto * argument_type_array = typeid_cast(argument.type.get()); + + if (!argument_type_array) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Argument {} for function {} must be array. Actual {}", + i + 1, + getName(), + argument.type->getName()); + + nested_types.push_back(argument_type_array->getNestedType()); + } + + return std::make_shared(std::make_shared(nested_types, nested_names)); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t) const override + { + size_t arguments_size = arguments.size(); + + const auto * lhs_array = assert_cast(arguments.at(1).column.get()); + + Columns data_columns; + data_columns.reserve(arguments_size); + data_columns.push_back(lhs_array->getDataPtr()); + + for (size_t i = 2; i < arguments_size; ++i) + { + const auto * rhs_array = assert_cast(arguments[i].column.get()); + + if (!lhs_array->hasEqualOffsets(*rhs_array)) + throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH, + "The argument 1 and argument {} of function {} have different array offsets", + i + 1, + getName()); + + data_columns.push_back(rhs_array->getDataPtr()); + } + + auto tuple_column = ColumnTuple::create(std::move(data_columns)); + auto array_column = ColumnArray::create(std::move(tuple_column), lhs_array->getOffsetsPtr()); + + return array_column; + } +private: + static Names extractNestedNames(const ColumnPtr & column) + { + const auto * const_column = typeid_cast(column.get()); + if (!const_column) + return {}; + + Field nested_names_field; + const_column->get(0, nested_names_field); + + if (nested_names_field.getType() != Field::Types::Array) + return {}; + + const auto & nested_names_array = nested_names_field.get(); + + Names nested_names; + nested_names.reserve(nested_names_array.size()); + + for (const auto & nested_name_field : nested_names_array) + { + if (nested_name_field.getType() != Field::Types::String) + return {}; + + nested_names.push_back(nested_name_field.get()); + } + + return nested_names; + } +}; + +} + +REGISTER_FUNCTION(Nsted) +{ + factory.registerFunction(); +} + +} diff --git a/src/Planner/CollectColumnIdentifiers.cpp b/src/Planner/CollectColumnIdentifiers.cpp index f7cdf196ad1..b1c45fd7a13 100644 --- a/src/Planner/CollectColumnIdentifiers.cpp +++ b/src/Planner/CollectColumnIdentifiers.cpp @@ -11,7 +11,7 @@ namespace DB namespace { -class CollectTopLevelColumnIdentifiersVisitor : public InDepthQueryTreeVisitor +class CollectTopLevelColumnIdentifiersVisitor : public ConstInDepthQueryTreeVisitor { public: @@ -20,7 +20,7 @@ public: , planner_context(planner_context_) {} - static bool needChildVisit(VisitQueryTreeNodeType &, VisitQueryTreeNodeType & child) + static bool needChildVisit(const QueryTreeNodePtr &, const QueryTreeNodePtr & child) { const auto & node_type = child->getNodeType(); return node_type != QueryTreeNodeType::TABLE diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp index 703de0e9119..ced76200f79 100644 --- a/src/Planner/Planner.cpp +++ b/src/Planner/Planner.cpp @@ -1132,6 +1132,13 @@ void Planner::buildPlanForQueryNode() query_node.getPrewhere() = {}; } + if (query_node.hasWhere()) + { + auto condition_constant = tryExtractConstantFromConditionNode(query_node.getWhere()); + if (condition_constant.has_value() && *condition_constant) + query_node.getWhere() = {}; + } + SelectQueryInfo select_query_info; select_query_info.original_query = queryNodeToSelectQuery(query_tree); select_query_info.query = select_query_info.original_query; diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index 505f0e8e3d0..03b81d1852b 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -778,8 +778,8 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_ } auto drop_unused_columns_after_join_actions_dag = std::make_shared(result_plan.getCurrentDataStream().header.getColumnsWithTypeAndName()); - ActionsDAG::NodeRawConstPtrs updated_outputs; - std::unordered_set updated_outputs_names; + ActionsDAG::NodeRawConstPtrs drop_unused_columns_after_join_actions_dag_updated_outputs; + std::unordered_set drop_unused_columns_after_join_actions_dag_updated_outputs_names; std::optional first_skipped_column_node_index; auto & drop_unused_columns_after_join_actions_dag_outputs = drop_unused_columns_after_join_actions_dag->getOutputs(); @@ -790,7 +790,7 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_ const auto & output = drop_unused_columns_after_join_actions_dag_outputs[i]; const auto & global_planner_context = planner_context->getGlobalPlannerContext(); - if (updated_outputs_names.contains(output->result_name) + if (drop_unused_columns_after_join_actions_dag_updated_outputs_names.contains(output->result_name) || !global_planner_context->hasColumnIdentifier(output->result_name)) continue; @@ -801,18 +801,18 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_ continue; } - updated_outputs.push_back(output); - updated_outputs_names.insert(output->result_name); + drop_unused_columns_after_join_actions_dag_updated_outputs.push_back(output); + drop_unused_columns_after_join_actions_dag_updated_outputs_names.insert(output->result_name); } /** It is expected that JOIN TREE query plan will contain at least 1 column, even if there are no columns in outer scope. * * Example: SELECT count() FROM test_table_1 AS t1, test_table_2 AS t2; */ - if (updated_outputs.empty() && first_skipped_column_node_index) - updated_outputs.push_back(drop_unused_columns_after_join_actions_dag_outputs[*first_skipped_column_node_index]); + if (drop_unused_columns_after_join_actions_dag_updated_outputs.empty() && first_skipped_column_node_index) + drop_unused_columns_after_join_actions_dag_updated_outputs.push_back(drop_unused_columns_after_join_actions_dag_outputs[*first_skipped_column_node_index]); - drop_unused_columns_after_join_actions_dag_outputs = std::move(updated_outputs); + drop_unused_columns_after_join_actions_dag_outputs = std::move(drop_unused_columns_after_join_actions_dag_updated_outputs); auto drop_unused_columns_after_join_transform_step = std::make_unique(result_plan.getCurrentDataStream(), std::move(drop_unused_columns_after_join_actions_dag)); drop_unused_columns_after_join_transform_step->setStepDescription("DROP unused columns after JOIN"); @@ -823,6 +823,7 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_ JoinTreeQueryPlan buildQueryPlanForArrayJoinNode(const QueryTreeNodePtr & array_join_table_expression, JoinTreeQueryPlan join_tree_query_plan, + const ColumnIdentifierSet & outer_scope_columns, PlannerContextPtr & planner_context) { auto & array_join_node = array_join_table_expression->as(); @@ -837,6 +838,7 @@ JoinTreeQueryPlan buildQueryPlanForArrayJoinNode(const QueryTreeNodePtr & array_ ActionsDAGPtr array_join_action_dag = std::make_shared(plan_output_columns); PlannerActionsVisitor actions_visitor(planner_context); + std::unordered_set array_join_expressions_output_nodes; NameSet array_join_column_names; for (auto & array_join_expression : array_join_node.getJoinExpressions().getNodes()) @@ -846,10 +848,12 @@ JoinTreeQueryPlan buildQueryPlanForArrayJoinNode(const QueryTreeNodePtr & array_ auto & array_join_expression_column = array_join_expression->as(); auto expression_dag_index_nodes = actions_visitor.visit(array_join_action_dag, array_join_expression_column.getExpressionOrThrow()); + for (auto & expression_dag_index_node : expression_dag_index_nodes) { const auto * array_join_column_node = &array_join_action_dag->addAlias(*expression_dag_index_node, array_join_column_identifier); array_join_action_dag->getOutputs().push_back(array_join_column_node); + array_join_expressions_output_nodes.insert(array_join_column_node->result_name); } } @@ -858,6 +862,35 @@ JoinTreeQueryPlan buildQueryPlanForArrayJoinNode(const QueryTreeNodePtr & array_ array_join_actions->setStepDescription("ARRAY JOIN actions"); plan.addStep(std::move(array_join_actions)); + auto drop_unused_columns_before_array_join_actions_dag = std::make_shared(plan.getCurrentDataStream().header.getColumnsWithTypeAndName()); + ActionsDAG::NodeRawConstPtrs drop_unused_columns_before_array_join_actions_dag_updated_outputs; + std::unordered_set drop_unused_columns_before_array_join_actions_dag_updated_outputs_names; + + auto & drop_unused_columns_before_array_join_actions_dag_outputs = drop_unused_columns_before_array_join_actions_dag->getOutputs(); + size_t drop_unused_columns_before_array_join_actions_dag_outputs_size = drop_unused_columns_before_array_join_actions_dag_outputs.size(); + + for (size_t i = 0; i < drop_unused_columns_before_array_join_actions_dag_outputs_size; ++i) + { + const auto & output = drop_unused_columns_before_array_join_actions_dag_outputs[i]; + + if (drop_unused_columns_before_array_join_actions_dag_updated_outputs_names.contains(output->result_name)) + continue; + + if (!array_join_expressions_output_nodes.contains(output->result_name) && + !outer_scope_columns.contains(output->result_name)) + continue; + + drop_unused_columns_before_array_join_actions_dag_updated_outputs.push_back(output); + drop_unused_columns_before_array_join_actions_dag_updated_outputs_names.insert(output->result_name); + } + + drop_unused_columns_before_array_join_actions_dag_outputs = std::move(drop_unused_columns_before_array_join_actions_dag_updated_outputs); + + auto drop_unused_columns_before_array_join_transform_step = std::make_unique(plan.getCurrentDataStream(), + std::move(drop_unused_columns_before_array_join_actions_dag)); + drop_unused_columns_before_array_join_transform_step->setStepDescription("DROP unused columns before ARRAY JOIN"); + plan.addStep(std::move(drop_unused_columns_before_array_join_transform_step)); + auto array_join_action = std::make_shared(array_join_column_names, array_join_node.isLeft(), planner_context->getQueryContext()); auto array_join_step = std::make_unique(plan.getCurrentDataStream(), std::move(array_join_action)); array_join_step->setStepDescription("ARRAY JOIN"); @@ -885,9 +918,13 @@ JoinTreeQueryPlan buildJoinTreeQueryPlan(const QueryTreeNodePtr & query_node, for (Int64 i = static_cast(table_expressions_stack_size) - 1; i >= 0; --i) { table_expressions_outer_scope_columns[i] = current_outer_scope_columns; + auto & table_expression = table_expressions_stack[i]; + auto table_expression_type = table_expression->getNodeType(); - if (table_expressions_stack[i]->getNodeType() == QueryTreeNodeType::JOIN) - collectTopLevelColumnIdentifiers(table_expressions_stack[i], planner_context, current_outer_scope_columns); + if (table_expression_type == QueryTreeNodeType::JOIN) + collectTopLevelColumnIdentifiers(table_expression, planner_context, current_outer_scope_columns); + else if (table_expression_type == QueryTreeNodeType::ARRAY_JOIN) + collectTopLevelColumnIdentifiers(table_expression, planner_context, current_outer_scope_columns); } std::vector query_plans_stack; @@ -906,6 +943,7 @@ JoinTreeQueryPlan buildJoinTreeQueryPlan(const QueryTreeNodePtr & query_node, auto query_plan = std::move(query_plans_stack.back()); query_plans_stack.back() = buildQueryPlanForArrayJoinNode(table_expression, std::move(query_plan), + table_expressions_outer_scope_columns[i], planner_context); } else if (auto * join_node = table_expression->as()) diff --git a/src/Planner/PlannerJoins.cpp b/src/Planner/PlannerJoins.cpp index 8d87ebd466b..e1c137ddfb8 100644 --- a/src/Planner/PlannerJoins.cpp +++ b/src/Planner/PlannerJoins.cpp @@ -37,6 +37,7 @@ #include #include +#include namespace DB { @@ -513,23 +514,7 @@ std::optional tryExtractConstantFromJoinNode(const QueryTreeNodePtr & join if (!join_node_typed.getJoinExpression()) return {}; - const auto * constant_node = join_node_typed.getJoinExpression()->as(); - if (!constant_node) - return {}; - - const auto & value = constant_node->getValue(); - auto constant_type = constant_node->getResultType(); - constant_type = removeNullable(removeLowCardinality(constant_type)); - - auto which_constant_type = WhichDataType(constant_type); - if (!which_constant_type.isUInt8() && !which_constant_type.isNothing()) - return {}; - - if (value.isNull()) - return false; - - UInt8 predicate_value = value.safeGet(); - return predicate_value > 0; + return tryExtractConstantFromConditionNode(join_node_typed.getJoinExpression()); } namespace diff --git a/src/Planner/Utils.cpp b/src/Planner/Utils.cpp index 0eac0881609..6df024fee4c 100644 --- a/src/Planner/Utils.cpp +++ b/src/Planner/Utils.cpp @@ -4,6 +4,9 @@ #include #include +#include +#include + #include #include @@ -333,4 +336,25 @@ QueryTreeNodePtr mergeConditionNodes(const QueryTreeNodes & condition_nodes, con return function_node; } +std::optional tryExtractConstantFromConditionNode(const QueryTreeNodePtr & condition_node) +{ + const auto * constant_node = condition_node->as(); + if (!constant_node) + return {}; + + const auto & value = constant_node->getValue(); + auto constant_type = constant_node->getResultType(); + constant_type = removeNullable(removeLowCardinality(constant_type)); + + auto which_constant_type = WhichDataType(constant_type); + if (!which_constant_type.isUInt8() && !which_constant_type.isNothing()) + return {}; + + if (value.isNull()) + return false; + + UInt8 predicate_value = value.safeGet(); + return predicate_value > 0; +} + } diff --git a/src/Planner/Utils.h b/src/Planner/Utils.h index f8b3f7a96d2..e304ae13eb5 100644 --- a/src/Planner/Utils.h +++ b/src/Planner/Utils.h @@ -63,4 +63,7 @@ bool queryHasWithTotalsInAnySubqueryInJoinTree(const QueryTreeNodePtr & query_no /// Returns `and` function node that has condition nodes as its arguments QueryTreeNodePtr mergeConditionNodes(const QueryTreeNodes & condition_nodes, const ContextPtr & context); +/// Try extract boolean constant from condition node +std::optional tryExtractConstantFromConditionNode(const QueryTreeNodePtr & condition_node); + } diff --git a/tests/queries/0_stateless/00855_join_with_array_join.reference b/tests/queries/0_stateless/00855_join_with_array_join.reference index fe9bd148afd..6fddaf24d13 100644 --- a/tests/queries/0_stateless/00855_join_with_array_join.reference +++ b/tests/queries/0_stateless/00855_join_with_array_join.reference @@ -6,11 +6,15 @@ 6 0 7 1 0 8 1 0 +101 0 +102 0 - 1 0 2 2 a2 1 0 2 2 a2 +1 +2 a2 0 0 0 0 0 0 diff --git a/tests/queries/0_stateless/00855_join_with_array_join.sql b/tests/queries/0_stateless/00855_join_with_array_join.sql index c1ea0bbb429..c278ff0738a 100644 --- a/tests/queries/0_stateless/00855_join_with_array_join.sql +++ b/tests/queries/0_stateless/00855_join_with_array_join.sql @@ -1,4 +1,5 @@ SET joined_subquery_requires_alias = 0; +SET allow_experimental_analyzer = 1; SELECT ax, c FROM (SELECT [1,2] ax, 0 c) ARRAY JOIN ax JOIN (SELECT 0 c) USING (c); SELECT ax, c FROM (SELECT [3,4] ax, 0 c) JOIN (SELECT 0 c) USING (c) ARRAY JOIN ax; @@ -9,7 +10,7 @@ SELECT ax, c, d FROM (SELECT [7,8] ax, 1 c, 0 d) s1 JOIN system.one s2 ON s1.c = SELECT ax, c FROM (SELECT [101,102] ax, 0 c) s1 JOIN system.one s2 ON s1.c = s2.dummy JOIN system.one s3 ON s1.c = s3.dummy -ARRAY JOIN ax; -- { serverError 48 } +ARRAY JOIN ax; SELECT '-'; @@ -28,8 +29,7 @@ INSERT INTO d VALUES (2, 'a2'), (3, 'a3'); SELECT d_ids, id, name FROM f LEFT ARRAY JOIN d_ids LEFT JOIN d ON d.id = d_ids ORDER BY id; SELECT did, id, name FROM f LEFT ARRAY JOIN d_ids as did LEFT JOIN d ON d.id = did ORDER BY id; --- name clash, doesn't work yet -SELECT id, name FROM f LEFT ARRAY JOIN d_ids as id LEFT JOIN d ON d.id = id ORDER BY id; -- { serverError 403 } +SELECT id, name FROM f LEFT ARRAY JOIN d_ids as id LEFT JOIN d ON d.id = id ORDER BY id; SELECT * FROM ( SELECT [dummy, dummy] AS dummy FROM system.one ) AS x ARRAY JOIN dummy JOIN system.one AS y ON x.dummy == y.dummy; diff --git a/tests/queries/0_stateless/02477_analyzer_array_join_with_join.reference b/tests/queries/0_stateless/02477_analyzer_array_join_with_join.reference index f7084c4617c..5c07313a46b 100644 --- a/tests/queries/0_stateless/02477_analyzer_array_join_with_join.reference +++ b/tests/queries/0_stateless/02477_analyzer_array_join_with_join.reference @@ -50,7 +50,7 @@ SELECT * FROM (SELECT [1] AS id) AS subquery_1 ARRAY JOIN id INNER JOIN (SELECT SELECT '--'; -- SELECT * FROM (SELECT [5] AS id) AS subquery_1 ARRAY JOIN [1,2,3] AS id INNER JOIN (SELECT 1 AS id) AS subquery_2 USING (id); -[5] +1 SELECT '--'; -- SELECT * FROM (SELECT [0] AS id) AS subquery ARRAY JOIN id INNER JOIN test_table USING (id); @@ -61,68 +61,68 @@ SELECT * FROM (SELECT [1] AS id) AS subquery ARRAY JOIN id INNER JOIN test_table SELECT '--'; -- SELECT * FROM (SELECT [0] AS id) AS subquery ARRAY JOIN id AS id INNER JOIN test_table USING (id); -[0] Value_0 [1,2,3] +0 Value_0 [1,2,3] SELECT '--'; -- SELECT * FROM (SELECT [1] AS id) AS subquery ARRAY JOIN id AS id INNER JOIN test_table USING (id); SELECT '--'; -- SELECT *, id FROM (SELECT [0] AS id) AS subquery ARRAY JOIN id AS id INNER JOIN test_table USING (id); -[0] Value_0 [1,2,3] 0 +0 Value_0 [1,2,3] 0 SELECT '--'; -- SELECT *, id FROM (SELECT [1] AS id) AS subquery ARRAY JOIN id AS id INNER JOIN test_table USING (id); SELECT '--'; -- SELECT * FROM (SELECT [0] AS value) AS subquery ARRAY JOIN value AS id INNER JOIN test_table USING (id); -[0] Value_0 [1,2,3] +0 [0] Value_0 [1,2,3] SELECT '--'; -- SELECT * FROM (SELECT [1] AS value) AS subquery ARRAY JOIN value AS id INNER JOIN test_table USING (id); SELECT '--'; -- SELECT *, id FROM (SELECT [0] AS value) AS subquery ARRAY JOIN value AS id INNER JOIN test_table USING (id); -[0] Value_0 [1,2,3] 0 +0 [0] Value_0 [1,2,3] 0 SELECT '--'; -- SELECT *, id FROM (SELECT [1] AS value) AS subquery ARRAY JOIN value AS id INNER JOIN test_table USING (id); SELECT '--'; -- SELECT * FROM (SELECT [0] AS id) AS subquery ARRAY JOIN [0] AS id INNER JOIN test_table USING (id); -[0] Value_0 [1,2,3] +0 Value_0 [1,2,3] SELECT '--'; -- SELECT * FROM (SELECT [0] AS id) AS subquery ARRAY JOIN [1] AS id INNER JOIN test_table USING (id); SELECT '--'; -- SELECT *, id FROM (SELECT [0] AS id) AS subquery ARRAY JOIN [0] AS id INNER JOIN test_table USING (id); -[0] Value_0 [1,2,3] 0 +0 Value_0 [1,2,3] 0 SELECT '--'; -- SELECT *, id FROM (SELECT [0] AS id) AS subquery ARRAY JOIN [1] AS id INNER JOIN test_table USING (id); SELECT '--'; -- SELECT * FROM (SELECT [5] AS id) AS subquery ARRAY JOIN [0] AS id INNER JOIN test_table USING (id); -[5] Value_0 [1,2,3] +0 Value_0 [1,2,3] SELECT '--'; -- SELECT * FROM (SELECT [5] AS id) AS subquery ARRAY JOIN [1] AS id INNER JOIN test_table USING (id); SELECT '--'; -- SELECT *, id FROM (SELECT [5] AS id) AS subquery ARRAY JOIN [0] AS id INNER JOIN test_table USING (id); -[5] Value_0 [1,2,3] 0 +0 Value_0 [1,2,3] 0 SELECT '--'; -- SELECT *, id FROM (SELECT [5] AS id) AS subquery ARRAY JOIN [1] AS id INNER JOIN test_table USING (id); SELECT '--'; -- SELECT * FROM (SELECT [5] AS id_array) AS subquery ARRAY JOIN id_array, [0] AS id INNER JOIN test_table USING (id); -5 Value_0 [1,2,3] +0 5 Value_0 [1,2,3] SELECT '--'; -- SELECT * FROM (SELECT [[0]] AS id) AS subquery ARRAY JOIN id AS id_nested_array ARRAY JOIN id_nested_array AS id INNER JOIN test_table USING (id); -[[0]] Value_0 [1,2,3] +0 Value_0 [1,2,3] SELECT '--'; -- SELECT *, id FROM (SELECT [[0]] AS id) AS subquery ARRAY JOIN id AS id_nested_array ARRAY JOIN id_nested_array AS id INNER JOIN test_table USING (id); -[[0]] Value_0 [1,2,3] 0 +0 Value_0 [1,2,3] 0 From f21486c95bb700acb9bb206b4470482b41a1f1ab Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sat, 4 Feb 2023 18:39:41 +0100 Subject: [PATCH 303/717] Analyzer validate that there are no window functions in GROUP BY --- src/Analyzer/FunctionNode.h | 19 +++++++++++++++++-- .../OptimizeGroupByFunctionKeysPass.cpp | 5 +++-- src/Analyzer/Passes/QueryAnalysisPass.cpp | 4 ++++ ...ize_group_by_function_keys_crash.reference | 0 ..._optimize_group_by_function_keys_crash.sql | 1 + 5 files changed, 25 insertions(+), 4 deletions(-) create mode 100644 tests/queries/0_stateless/02552_analyzer_optimize_group_by_function_keys_crash.reference create mode 100644 tests/queries/0_stateless/02552_analyzer_optimize_group_by_function_keys_crash.sql diff --git a/src/Analyzer/FunctionNode.h b/src/Analyzer/FunctionNode.h index 41751ec3f09..d9c8aa74ba3 100644 --- a/src/Analyzer/FunctionNode.h +++ b/src/Analyzer/FunctionNode.h @@ -85,7 +85,10 @@ public: /// Get arguments node QueryTreeNodePtr & getArgumentsNode() { return children[arguments_child_index]; } + /// Get argument types const DataTypes & getArgumentTypes() const; + + /// Get argument columns ColumnsWithTypeAndName getArgumentColumns() const; /// Returns true if function node has window, false otherwise @@ -104,8 +107,8 @@ public: */ QueryTreeNodePtr & getWindowNode() { return children[window_child_index]; } - /** Get non aggregate function. - * If function is not resolved nullptr returned. + /** Get ordinary function. + * If function is not resolved or is resolved as non ordinary function nullptr is returned. */ FunctionBasePtr getFunction() const { @@ -114,6 +117,18 @@ public: return std::static_pointer_cast(function); } + /** Get ordinary function. + * If function is not resolved or is resolved as non ordinary function exception is thrown. + */ + FunctionBasePtr getFunctionOrThrow() const + { + if (kind != FunctionKind::ORDINARY) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Function node with name '{}' is not resolved as ordinary function"); + + return std::static_pointer_cast(function); + } + /** Get aggregate function. * If function is not resolved nullptr returned. * If function is resolved as non aggregate function nullptr returned. diff --git a/src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.cpp b/src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.cpp index 611ff3a87c7..b313765ee31 100644 --- a/src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.cpp +++ b/src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.cpp @@ -62,7 +62,7 @@ private: std::vector candidates; auto & function_arguments = function->getArguments().getNodes(); - bool is_deterministic = function->getFunction()->isDeterministicInScopeOfQuery(); + bool is_deterministic = function->getFunctionOrThrow()->isDeterministicInScopeOfQuery(); for (auto it = function_arguments.rbegin(); it != function_arguments.rend(); ++it) candidates.push_back({ *it, is_deterministic }); @@ -86,7 +86,8 @@ private: if (!found) { - bool is_deterministic_function = parents_are_only_deterministic && function->getFunction()->isDeterministicInScopeOfQuery(); + bool is_deterministic_function = parents_are_only_deterministic && + function->getFunctionOrThrow()->isDeterministicInScopeOfQuery(); for (auto it = arguments.rbegin(); it != arguments.rend(); ++it) candidates.push_back({ *it, is_deterministic_function }); } diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index d366a017642..c6b83ac3e14 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -6644,7 +6644,11 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier collectWindowFunctionNodes(query_node, window_function_nodes); if (query_node_typed.hasGroupBy()) + { assertNoAggregateFunctionNodes(query_node_typed.getGroupByNode(), "in GROUP BY"); + assertNoGroupingFunction(query_node_typed.getGroupByNode(), "in GROUP BY"); + assertNoWindowFunctionNodes(query_node_typed.getGroupByNode(), "in GROUP BY"); + } for (auto & aggregate_function_node : aggregate_function_nodes) { diff --git a/tests/queries/0_stateless/02552_analyzer_optimize_group_by_function_keys_crash.reference b/tests/queries/0_stateless/02552_analyzer_optimize_group_by_function_keys_crash.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02552_analyzer_optimize_group_by_function_keys_crash.sql b/tests/queries/0_stateless/02552_analyzer_optimize_group_by_function_keys_crash.sql new file mode 100644 index 00000000000..f0cc3623d4f --- /dev/null +++ b/tests/queries/0_stateless/02552_analyzer_optimize_group_by_function_keys_crash.sql @@ -0,0 +1 @@ +SELECT NULL GROUP BY tuple('0.0000000007'), count(NULL) OVER (ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) -- { serverError 184 }; From 3a624e6beff85f65719f96074b185a4aa8833d4a Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sun, 5 Feb 2023 13:19:21 +0100 Subject: [PATCH 304/717] Analyzer ColumnNode AST conversion fix --- src/Analyzer/ColumnNode.cpp | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/Analyzer/ColumnNode.cpp b/src/Analyzer/ColumnNode.cpp index a5f7dc9786e..affc9c67eda 100644 --- a/src/Analyzer/ColumnNode.cpp +++ b/src/Analyzer/ColumnNode.cpp @@ -93,7 +93,7 @@ QueryTreeNodePtr ColumnNode::cloneImpl() const ASTPtr ColumnNode::toASTImpl() const { - std::vector additional_column_qualification_parts; + std::vector column_identifier_parts; auto column_source = getColumnSourceOrNull(); if (column_source) @@ -106,23 +106,19 @@ ASTPtr ColumnNode::toASTImpl() const { if (column_source->hasAlias()) { - additional_column_qualification_parts = {column_source->getAlias()}; + column_identifier_parts = {column_source->getAlias()}; } else if (auto * table_node = column_source->as()) { const auto & table_storage_id = table_node->getStorageID(); - additional_column_qualification_parts = {table_storage_id.getDatabaseName(), table_storage_id.getTableName()}; + column_identifier_parts = {table_storage_id.getDatabaseName(), table_storage_id.getTableName()}; } } } - auto column_name_identifier = Identifier(column.name); - const auto & column_name_identifier_parts = column_name_identifier.getParts(); - additional_column_qualification_parts.insert(additional_column_qualification_parts.end(), - column_name_identifier_parts.begin(), - column_name_identifier_parts.end()); + column_identifier_parts.push_back(column.name); - return std::make_shared(std::move(additional_column_qualification_parts)); + return std::make_shared(std::move(column_identifier_parts)); } } From a090a8449da2d4912be2b38cbba4479ceb14e47f Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sun, 5 Feb 2023 13:20:10 +0100 Subject: [PATCH 305/717] Analyzer distributed read fix --- src/Storages/StorageDistributed.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 619b1362bff..f23ed8f3d0e 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -679,10 +679,9 @@ void StorageDistributed::read( } const auto & modified_query_ast = ClusterProxy::rewriteSelectQuery( - local_context, query_info.query, + local_context, query_ast, remote_database, remote_table, remote_table_function_ptr); - /// Return directly (with correct header) if no shard to query. if (query_info.getCluster()->getShardsInfo().empty()) { From afb3eb3c31798c68d27ee09d9f7cdbc95649665f Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sun, 5 Feb 2023 13:21:00 +0100 Subject: [PATCH 306/717] Planner JOIN planning fix --- src/Planner/PlannerJoinTree.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index 03b81d1852b..b37fd4a6dca 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -680,7 +680,8 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_ for (auto & column_from_joined_table : columns_from_joined_table) { - if (planner_context->getGlobalPlannerContext()->hasColumnIdentifier(column_from_joined_table.name)) + if (planner_context->getGlobalPlannerContext()->hasColumnIdentifier(column_from_joined_table.name) && + outer_scope_columns.contains(column_from_joined_table.name)) table_join->addJoinedColumn(column_from_joined_table); } From c0e513efebec790d031d68964f43579fdcc1ff3a Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sun, 5 Feb 2023 13:21:58 +0100 Subject: [PATCH 307/717] Fixed style check --- src/Analyzer/FunctionNode.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Analyzer/FunctionNode.h b/src/Analyzer/FunctionNode.h index d9c8aa74ba3..a3212b3fb07 100644 --- a/src/Analyzer/FunctionNode.h +++ b/src/Analyzer/FunctionNode.h @@ -16,6 +16,7 @@ namespace DB namespace ErrorCodes { extern const int UNSUPPORTED_METHOD; + extern const int LOGICAL_ERROR; } class IFunctionOverloadResolver; From 0cba5848ac024d0493a0b877886e5b1af3a45473 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sun, 5 Feb 2023 14:27:06 +0100 Subject: [PATCH 308/717] Fixed tests --- src/Functions/nested.cpp | 2 +- ...01323_redundant_functions_in_order_by.reference | 14 +++++++------- .../01323_redundant_functions_in_order_by.sql | 2 ++ .../01674_where_prewhere_array_crash.sql | 6 ++++-- .../02374_analyzer_join_using.reference | 8 ++------ .../02382_analyzer_matcher_join_using.reference | 4 ++-- .../02514_analyzer_drop_join_on.reference | 5 ----- ...lyzer_optimize_group_by_function_keys_crash.sql | 2 ++ 8 files changed, 20 insertions(+), 23 deletions(-) diff --git a/src/Functions/nested.cpp b/src/Functions/nested.cpp index 93a440633c4..f3012fde282 100644 --- a/src/Functions/nested.cpp +++ b/src/Functions/nested.cpp @@ -163,7 +163,7 @@ private: } -REGISTER_FUNCTION(Nsted) +REGISTER_FUNCTION(Nested) { factory.registerFunction(); } diff --git a/tests/queries/0_stateless/01323_redundant_functions_in_order_by.reference b/tests/queries/0_stateless/01323_redundant_functions_in_order_by.reference index c69f8bb2c46..91a96eb68a3 100644 --- a/tests/queries/0_stateless/01323_redundant_functions_in_order_by.reference +++ b/tests/queries/0_stateless/01323_redundant_functions_in_order_by.reference @@ -190,16 +190,16 @@ QUERY id: 0 COLUMN id: 18, column_name: key, result_type: UInt64, source_id: 8 EXPRESSION LIST id: 19, nodes: 2 - COLUMN id: 2, column_name: key, result_type: UInt64, source_id: 3 - COLUMN id: 20, column_name: key, result_type: UInt64, source_id: 5 + COLUMN id: 20, column_name: key, result_type: UInt64, source_id: 3 + COLUMN id: 21, column_name: key, result_type: UInt64, source_id: 5 ORDER BY - LIST id: 21, nodes: 2 - SORT id: 22, sort_direction: ASCENDING, with_fill: 0 + LIST id: 22, nodes: 2 + SORT id: 23, sort_direction: ASCENDING, with_fill: 0 EXPRESSION - COLUMN id: 23, column_name: key, result_type: UInt64, source_id: 3 - SORT id: 24, sort_direction: ASCENDING, with_fill: 0 + COLUMN id: 24, column_name: key, result_type: UInt64, source_id: 3 + SORT id: 25, sort_direction: ASCENDING, with_fill: 0 EXPRESSION - COLUMN id: 25, column_name: key, result_type: UInt64, source_id: 5 + COLUMN id: 26, column_name: key, result_type: UInt64, source_id: 5 SELECT key, a diff --git a/tests/queries/0_stateless/01323_redundant_functions_in_order_by.sql b/tests/queries/0_stateless/01323_redundant_functions_in_order_by.sql index 5cdc4164d56..338c1345052 100644 --- a/tests/queries/0_stateless/01323_redundant_functions_in_order_by.sql +++ b/tests/queries/0_stateless/01323_redundant_functions_in_order_by.sql @@ -1,3 +1,5 @@ +SET single_join_prefer_left_table = 0; + DROP TABLE IF EXISTS test; CREATE TABLE test (key UInt64, a UInt8, b String, c Float64) ENGINE = MergeTree() ORDER BY key; diff --git a/tests/queries/0_stateless/01674_where_prewhere_array_crash.sql b/tests/queries/0_stateless/01674_where_prewhere_array_crash.sql index d6eef000b36..7a2466c70d7 100644 --- a/tests/queries/0_stateless/01674_where_prewhere_array_crash.sql +++ b/tests/queries/0_stateless/01674_where_prewhere_array_crash.sql @@ -1,5 +1,7 @@ +SET allow_experimental_analyzer = 1; + drop table if exists tab; create table tab (x UInt64, `arr.a` Array(UInt64), `arr.b` Array(UInt64)) engine = MergeTree order by x; -select x from tab array join arr prewhere x != 0 where arr; -- { serverError 47; } -select x from tab array join arr prewhere arr where x != 0; -- { serverError 47; } +select x from tab array join arr prewhere x != 0 where arr; -- { serverError 43 } +select x from tab array join arr prewhere arr where x != 0; -- { serverError 43 } drop table if exists tab; diff --git a/tests/queries/0_stateless/02374_analyzer_join_using.reference b/tests/queries/0_stateless/02374_analyzer_join_using.reference index 62750c33f89..622476942d0 100644 --- a/tests/queries/0_stateless/02374_analyzer_join_using.reference +++ b/tests/queries/0_stateless/02374_analyzer_join_using.reference @@ -364,13 +364,11 @@ SELECT t1.value AS t1_value, toTypeName(t1_value), t2.value AS t2_value, toTypeN FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 USING (id) INNER JOIN test_table_join_3 AS t3 USING(id); Join_1_Value_0 String Join_2_Value_0 String Join_3_Value_0 String Join_1_Value_1 String Join_2_Value_1 String Join_3_Value_1 String - String Join_2_Value_3 String Join_3_Value_0 String SELECT '--'; -- SELECT 1 FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 USING (id) INNER JOIN test_table_join_3 AS t3 USING(id); 1 1 -1 SELECT id FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON t1.id = t2.id INNER JOIN test_table_join_3 AS t3 USING (id); -- { serverError 207 } SELECT 'First JOIN FULL second JOIN LEFT'; First JOIN FULL second JOIN LEFT @@ -388,7 +386,7 @@ FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 USING (id) LEFT J Join_1_Value_0 String Join_2_Value_0 String Join_3_Value_0 String Join_1_Value_1 String Join_2_Value_1 String Join_3_Value_1 String Join_1_Value_2 String String String - String Join_2_Value_3 String Join_3_Value_0 String + String Join_2_Value_3 String String SELECT '--'; -- SELECT 1 FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 USING (id) LEFT JOIN test_table_join_3 AS t3 USING(id); @@ -412,7 +410,6 @@ SELECT t1.value AS t1_value, toTypeName(t1_value), t2.value AS t2_value, toTypeN FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 USING (id) RIGHT JOIN test_table_join_3 AS t3 USING(id); Join_1_Value_0 String Join_2_Value_0 String Join_3_Value_0 String Join_1_Value_1 String Join_2_Value_1 String Join_3_Value_1 String - String Join_2_Value_3 String Join_3_Value_0 String String String Join_3_Value_4 String SELECT '--'; -- @@ -420,7 +417,6 @@ SELECT 1 FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 USING (i 1 1 1 -1 SELECT id FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON t1.id = t2.id RIGHT JOIN test_table_join_3 AS t3 USING (id); -- { serverError 207 } SELECT 'First JOIN FULL second JOIN FULL'; First JOIN FULL second JOIN FULL @@ -439,7 +435,7 @@ FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 USING (id) FULL J Join_1_Value_0 String Join_2_Value_0 String Join_3_Value_0 String Join_1_Value_1 String Join_2_Value_1 String Join_3_Value_1 String Join_1_Value_2 String String String - String Join_2_Value_3 String Join_3_Value_0 String + String Join_2_Value_3 String String String String Join_3_Value_4 String SELECT '--'; -- diff --git a/tests/queries/0_stateless/02382_analyzer_matcher_join_using.reference b/tests/queries/0_stateless/02382_analyzer_matcher_join_using.reference index f2199aad4c8..f3c57eb2889 100644 --- a/tests/queries/0_stateless/02382_analyzer_matcher_join_using.reference +++ b/tests/queries/0_stateless/02382_analyzer_matcher_join_using.reference @@ -19,10 +19,10 @@ SELECT * FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 USING ( SELECT '--'; -- SELECT * FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 USING (id) ORDER BY id, t1.value; -0 Join_2_Value_3 0 Join_1_Value_0 Join_2_Value_0 1 Join_1_Value_1 Join_2_Value_1 2 Join_1_Value_2 +3 Join_2_Value_3 SELECT '--'; -- SELECT * FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 USING (id) INNER JOIN test_table_join_3 AS t3 USING (id) ORDER BY id, t1.value; @@ -42,6 +42,6 @@ SELECT * FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 USING ( SELECT '--'; -- SELECT * FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 USING (id) FULL JOIN test_table_join_3 AS t3 USING (id) ORDER BY id, t1.value; -0 Join_3_Value_4 0 Join_1_Value_0 Join_2_Value_0 Join_3_Value_0 1 Join_1_Value_1 Join_2_Value_1 Join_3_Value_1 +4 Join_3_Value_4 diff --git a/tests/queries/0_stateless/02514_analyzer_drop_join_on.reference b/tests/queries/0_stateless/02514_analyzer_drop_join_on.reference index abd49790ced..0f6fa7e2e66 100644 --- a/tests/queries/0_stateless/02514_analyzer_drop_join_on.reference +++ b/tests/queries/0_stateless/02514_analyzer_drop_join_on.reference @@ -13,7 +13,6 @@ Header: count() UInt64 Join (JOIN FillRightFirst) Header: default.a.a2_4 String default.c.c1_2 UInt64 - default.d.d1_3 UInt64 Expression ((JOIN actions + DROP unused columns after JOIN)) Header: default.a.a2_4 String default.c.c1_2 UInt64 @@ -56,21 +55,18 @@ Header: a2 String Header: default.a.k_2 UInt64 default.a.a2_0 String default.d.d2_1 String - default.d.k_5 UInt64 Expression (DROP unused columns after JOIN) Header: default.a.k_2 UInt64 default.a.a2_0 String Join (JOIN FillRightFirst) Header: default.a.k_2 UInt64 default.a.a2_0 String - default.c.k_4 UInt64 Expression (DROP unused columns after JOIN) Header: default.a.k_2 UInt64 default.a.a2_0 String Join (JOIN FillRightFirst) Header: default.a.k_2 UInt64 default.a.a2_0 String - default.b.k_3 UInt64 Expression (Change column names to column identifiers) Header: default.a.k_2 UInt64 default.a.a2_0 String @@ -111,7 +107,6 @@ Header: bx String b.bx_0 String default.c.c2_5 String default.c.c1_3 UInt64 - d.d1_4 UInt64 Filter (( + (JOIN actions + DROP unused columns after JOIN))) Header: default.a.a2_6 String b.bx_0 String diff --git a/tests/queries/0_stateless/02552_analyzer_optimize_group_by_function_keys_crash.sql b/tests/queries/0_stateless/02552_analyzer_optimize_group_by_function_keys_crash.sql index f0cc3623d4f..5fca43ace92 100644 --- a/tests/queries/0_stateless/02552_analyzer_optimize_group_by_function_keys_crash.sql +++ b/tests/queries/0_stateless/02552_analyzer_optimize_group_by_function_keys_crash.sql @@ -1 +1,3 @@ +SET allow_experimental_analyzer = 1; + SELECT NULL GROUP BY tuple('0.0000000007'), count(NULL) OVER (ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) -- { serverError 184 }; From 9b7cd64093a79ff51b28a912501ce028618f88a5 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sun, 5 Feb 2023 14:30:04 +0100 Subject: [PATCH 309/717] Analyzer join_use_nulls fix --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index c6b83ac3e14..18da324fea9 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -2960,14 +2960,21 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromJoin(const IdentifierLoo bool join_use_nulls = scope.context->getSettingsRef().join_use_nulls; - if (join_use_nulls - && (isFull(join_kind) || - (isLeft(join_kind) && resolved_side && *resolved_side == JoinTableSide::Right) || - (isRight(join_kind) && resolved_side && *resolved_side == JoinTableSide::Left))) + if (join_use_nulls && + resolved_identifier->getNodeType() == QueryTreeNodeType::COLUMN && + (isFull(join_kind) || + (isLeft(join_kind) && resolved_side && *resolved_side == JoinTableSide::Right) || + (isRight(join_kind) && resolved_side && *resolved_side == JoinTableSide::Left))) { resolved_identifier = resolved_identifier->clone(); auto & resolved_column = resolved_identifier->as(); - resolved_column.setColumnType(makeNullable(resolved_column.getColumnType())); + const auto & resolved_column_type = resolved_column.getColumnType(); + const auto & resolved_column_name = resolved_column.getColumnName(); + + auto to_nullable_function_resolver = FunctionFactory::instance().get("toNullable", scope.context); + auto to_nullable_function_arguments = {ColumnWithTypeAndName(nullptr, resolved_column_type, resolved_column_name)}; + auto to_nullable_function = to_nullable_function_resolver->build(to_nullable_function_arguments); + resolved_column.setColumnType(to_nullable_function->getResultType()); } return resolved_identifier; From 84065fb13f9be676d70a31571fd1ebc05715879e Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 8 Feb 2023 15:41:24 +0100 Subject: [PATCH 310/717] Analyzer added distributed table functions support --- src/Analyzer/AggregationUtils.cpp | 39 ++++++- src/Analyzer/AggregationUtils.h | 9 +- src/Analyzer/ColumnNode.cpp | 2 +- src/Analyzer/IQueryTreeNode.cpp | 34 +++--- src/Analyzer/LambdaNode.cpp | 2 + src/Analyzer/Passes/QueryAnalysisPass.cpp | 45 ++++---- src/Analyzer/Utils.cpp | 56 +++++++++- src/Analyzer/Utils.h | 9 +- src/Analyzer/WindowFunctionsUtils.cpp | 34 +++++- src/Analyzer/WindowFunctionsUtils.h | 5 + .../ClusterProxy/SelectStreamFactory.cpp | 30 +++--- src/Planner/CollectColumnIdentifiers.cpp | 19 ++-- src/Planner/CollectSets.cpp | 8 +- src/Planner/Planner.cpp | 22 ++-- src/Planner/PlannerExpressionAnalysis.cpp | 61 ++++++++++- src/Planner/PlannerExpressionAnalysis.h | 4 +- src/Storages/StorageDistributed.cpp | 101 +++++++++++++++--- src/Storages/StorageTableFunction.h | 3 - ...g_without_actual_aggregation_bug.reference | 1 - ..._having_without_actual_aggregation_bug.sql | 7 +- 20 files changed, 388 insertions(+), 103 deletions(-) diff --git a/src/Analyzer/AggregationUtils.cpp b/src/Analyzer/AggregationUtils.cpp index e2d4ada4f4b..fd35d0cea84 100644 --- a/src/Analyzer/AggregationUtils.cpp +++ b/src/Analyzer/AggregationUtils.cpp @@ -26,8 +26,15 @@ public: : assert_no_aggregates_place_message(std::move(assert_no_aggregates_place_message_)) {} + explicit CollectAggregateFunctionNodesVisitor(bool only_check_) + : only_check(only_check_) + {} + void visitImpl(const QueryTreeNodePtr & node) { + if (only_check && has_aggregate_functions) + return; + auto * function_node = node->as(); if (!function_node || !function_node->isAggregateFunction()) return; @@ -40,16 +47,32 @@ public: if (aggregate_function_nodes) aggregate_function_nodes->push_back(node); + + has_aggregate_functions = true; } - static bool needChildVisit(const QueryTreeNodePtr &, const QueryTreeNodePtr & child_node) + bool needChildVisit(const QueryTreeNodePtr & parent_node, const QueryTreeNodePtr & child_node) { - return !(child_node->getNodeType() == QueryTreeNodeType::QUERY || child_node->getNodeType() == QueryTreeNodeType::UNION); + if (only_check && has_aggregate_functions) + return false; + + if (parent_node->getNodeType() == QueryTreeNodeType::CONSTANT) + return false; + + auto child_node_type = child_node->getNodeType(); + return !(child_node_type == QueryTreeNodeType::QUERY || child_node_type == QueryTreeNodeType::UNION); + } + + bool hasAggregateFunctions() const + { + return has_aggregate_functions; } private: String assert_no_aggregates_place_message; QueryTreeNodes * aggregate_function_nodes = nullptr; + bool only_check = false; + bool has_aggregate_functions = false; }; } @@ -69,15 +92,23 @@ void collectAggregateFunctionNodes(const QueryTreeNodePtr & node, QueryTreeNodes visitor.visit(node); } +bool hasAggregateFunctionNodes(const QueryTreeNodePtr & node) +{ + CollectAggregateFunctionNodesVisitor visitor(true /*only_check*/); + visitor.visit(node); + + return visitor.hasAggregateFunctions(); +} + void assertNoAggregateFunctionNodes(const QueryTreeNodePtr & node, const String & assert_no_aggregates_place_message) { CollectAggregateFunctionNodesVisitor visitor(assert_no_aggregates_place_message); visitor.visit(node); } -void assertNoGroupingFunction(const QueryTreeNodePtr & node, const String & assert_no_grouping_function_place_message) +void assertNoGroupingFunctionNodes(const QueryTreeNodePtr & node, const String & assert_no_grouping_function_place_message) { - assertNoFunction(node, "grouping", ErrorCodes::ILLEGAL_AGGREGATION, "GROUPING", assert_no_grouping_function_place_message); + assertNoFunctionNodes(node, "grouping", ErrorCodes::ILLEGAL_AGGREGATION, "GROUPING", assert_no_grouping_function_place_message); } } diff --git a/src/Analyzer/AggregationUtils.h b/src/Analyzer/AggregationUtils.h index c2e53e55c04..a2119f716a8 100644 --- a/src/Analyzer/AggregationUtils.h +++ b/src/Analyzer/AggregationUtils.h @@ -15,14 +15,19 @@ QueryTreeNodes collectAggregateFunctionNodes(const QueryTreeNodePtr & node); */ void collectAggregateFunctionNodes(const QueryTreeNodePtr & node, QueryTreeNodes & result); +/** Returns true if there are aggregate function nodes in node children, false otherwise. + * Do not visit subqueries. + */ +bool hasAggregateFunctionNodes(const QueryTreeNodePtr & node); + /** Assert that there are no aggregate function nodes in node children. * Do not visit subqueries. */ void assertNoAggregateFunctionNodes(const QueryTreeNodePtr & node, const String & assert_no_aggregates_place_message); -/** Assert that there are no GROUPING functions in node children. +/** Assert that there are no GROUPING function nodes in node children. * Do not visit subqueries. */ -void assertNoGroupingFunction(const QueryTreeNodePtr & node, const String & assert_no_grouping_function_place_message); +void assertNoGroupingFunctionNodes(const QueryTreeNodePtr & node, const String & assert_no_grouping_function_place_message); } diff --git a/src/Analyzer/ColumnNode.cpp b/src/Analyzer/ColumnNode.cpp index affc9c67eda..b9939df37bb 100644 --- a/src/Analyzer/ColumnNode.cpp +++ b/src/Analyzer/ColumnNode.cpp @@ -88,7 +88,7 @@ void ColumnNode::updateTreeHashImpl(HashState & hash_state) const QueryTreeNodePtr ColumnNode::cloneImpl() const { - return std::make_shared(column, getColumnSource()); + return std::make_shared(column, getSourceWeakPointer()); } ASTPtr ColumnNode::toASTImpl() const diff --git a/src/Analyzer/IQueryTreeNode.cpp b/src/Analyzer/IQueryTreeNode.cpp index 2b6fd6d8fa6..640c93be415 100644 --- a/src/Analyzer/IQueryTreeNode.cpp +++ b/src/Analyzer/IQueryTreeNode.cpp @@ -166,24 +166,27 @@ IQueryTreeNode::Hash IQueryTreeNode::getTreeHash() const { HashState hash_state; - std::unordered_map node_to_identifier; + std::unordered_map weak_node_to_identifier; - std::vector nodes_to_process; - nodes_to_process.push_back(this); + std::vector> nodes_to_process; + nodes_to_process.emplace_back(this, false); while (!nodes_to_process.empty()) { - const auto * node_to_process = nodes_to_process.back(); + const auto [node_to_process, is_weak_node] = nodes_to_process.back(); nodes_to_process.pop_back(); - auto node_identifier_it = node_to_identifier.find(node_to_process); - if (node_identifier_it != node_to_identifier.end()) + if (is_weak_node) { - hash_state.update(node_identifier_it->second); - continue; - } + auto node_identifier_it = weak_node_to_identifier.find(node_to_process); + if (node_identifier_it != weak_node_to_identifier.end()) + { + hash_state.update(node_identifier_it->second); + continue; + } - node_to_identifier.emplace(node_to_process, node_to_identifier.size()); + weak_node_to_identifier.emplace(node_to_process, weak_node_to_identifier.size()); + } hash_state.update(static_cast(node_to_process->getNodeType())); if (!node_to_process->alias.empty()) @@ -201,7 +204,7 @@ IQueryTreeNode::Hash IQueryTreeNode::getTreeHash() const if (!node_to_process_child) continue; - nodes_to_process.push_back(node_to_process_child.get()); + nodes_to_process.emplace_back(node_to_process_child.get(), false); } hash_state.update(node_to_process->weak_pointers.size()); @@ -212,7 +215,7 @@ IQueryTreeNode::Hash IQueryTreeNode::getTreeHash() const if (!strong_pointer) continue; - nodes_to_process.push_back(strong_pointer.get()); + nodes_to_process.emplace_back(strong_pointer.get(), true); } } @@ -254,12 +257,15 @@ QueryTreeNodePtr IQueryTreeNode::cloneAndReplace(const ReplacementMap & replacem auto node_clone = it != replacement_map.end() ? it->second : node_to_clone->cloneImpl(); *place_for_cloned_node = node_clone; + old_pointer_to_new_pointer.emplace(node_to_clone, node_clone); + + if (it != replacement_map.end()) + continue; + node_clone->setAlias(node_to_clone->alias); node_clone->children = node_to_clone->children; node_clone->weak_pointers = node_to_clone->weak_pointers; - old_pointer_to_new_pointer.emplace(node_to_clone, node_clone); - for (auto & child : node_clone->children) { if (!child) diff --git a/src/Analyzer/LambdaNode.cpp b/src/Analyzer/LambdaNode.cpp index 809f73072d2..b60b40878ec 100644 --- a/src/Analyzer/LambdaNode.cpp +++ b/src/Analyzer/LambdaNode.cpp @@ -82,6 +82,8 @@ ASTPtr LambdaNode::toASTImpl() const lambda_function_ast->children.push_back(std::move(lambda_function_arguments_ast)); lambda_function_ast->arguments = lambda_function_ast->children.back(); + lambda_function_ast->is_lambda_function = true; + return lambda_function_ast; } diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 18da324fea9..6d9208428d8 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -1058,13 +1058,12 @@ public: [[fallthrough]]; case QueryTreeNodeType::LIST: { - if (!table_expression) - throw Exception(ErrorCodes::LOGICAL_ERROR, - "For expression analysis table expression must not be empty"); - - scope.expression_join_tree_node = table_expression; - validateTableExpressionModifiers(scope.expression_join_tree_node, scope); - initializeTableExpressionData(scope.expression_join_tree_node, scope); + if (table_expression) + { + scope.expression_join_tree_node = table_expression; + validateTableExpressionModifiers(scope.expression_join_tree_node, scope); + initializeTableExpressionData(scope.expression_join_tree_node, scope); + } if (node_type == QueryTreeNodeType::LIST) resolveExpressionNodeList(node, scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); @@ -4910,7 +4909,10 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi column = function_base->getConstantResultForNonConstArguments(argument_columns, result_type); } - if (column && isColumnConst(*column)) + /** Do not perform constant folding if there are aggregate or arrayJoin functions inside function. + * Example: SELECT toTypeName(sum(number)) FROM numbers(10); + */ + if (column && isColumnConst(*column) && (!hasAggregateFunctionNodes(node) && !hasFunctionNode(node, "arrayJoin"))) { /// Replace function node with result constant node Field column_constant_value; @@ -6255,7 +6257,10 @@ public: else if (auto * table_node = column_node_source->as()) column_name = table_node->getStorageID().getFullTableName(); - column_name += '.' + column_node->getColumnName(); + if (!column_name.empty()) + column_name += '.'; + + column_name += column_node->getColumnName(); throw Exception(ErrorCodes::NOT_AN_AGGREGATE, "Column {} is not under aggregate function and not in GROUP BY. In scope {}", @@ -6272,7 +6277,7 @@ public: for (const auto & group_by_key_node : group_by_keys_nodes) { - if (child_node->isEqual(*group_by_key_node)) + if (child_node->isEqual(*group_by_key_node, {.compare_aliases = false})) return false; } } @@ -6600,7 +6605,7 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier expandGroupByAll(query_node_typed); if (query_node_typed.hasPrewhere()) - assertNoFunction(query_node_typed.getPrewhere(), + assertNoFunctionNodes(query_node_typed.getPrewhere(), "arrayJoin", ErrorCodes::ILLEGAL_PREWHERE, "ARRAY JOIN", @@ -6620,21 +6625,21 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier if (!join_tree_is_subquery) { assertNoAggregateFunctionNodes(query_node_typed.getJoinTree(), "in JOIN TREE"); - assertNoGroupingFunction(query_node_typed.getJoinTree(), "in JOIN TREE"); + assertNoGroupingFunctionNodes(query_node_typed.getJoinTree(), "in JOIN TREE"); assertNoWindowFunctionNodes(query_node_typed.getJoinTree(), "in JOIN TREE"); } if (query_node_typed.hasWhere()) { assertNoAggregateFunctionNodes(query_node_typed.getWhere(), "in WHERE"); - assertNoGroupingFunction(query_node_typed.getWhere(), "in WHERE"); + assertNoGroupingFunctionNodes(query_node_typed.getWhere(), "in WHERE"); assertNoWindowFunctionNodes(query_node_typed.getWhere(), "in WHERE"); } if (query_node_typed.hasPrewhere()) { assertNoAggregateFunctionNodes(query_node_typed.getPrewhere(), "in PREWHERE"); - assertNoGroupingFunction(query_node_typed.getPrewhere(), "in PREWHERE"); + assertNoGroupingFunctionNodes(query_node_typed.getPrewhere(), "in PREWHERE"); assertNoWindowFunctionNodes(query_node_typed.getPrewhere(), "in PREWHERE"); } @@ -6653,7 +6658,7 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier if (query_node_typed.hasGroupBy()) { assertNoAggregateFunctionNodes(query_node_typed.getGroupByNode(), "in GROUP BY"); - assertNoGroupingFunction(query_node_typed.getGroupByNode(), "in GROUP BY"); + assertNoGroupingFunctionNodes(query_node_typed.getGroupByNode(), "in GROUP BY"); assertNoWindowFunctionNodes(query_node_typed.getGroupByNode(), "in GROUP BY"); } @@ -6662,7 +6667,7 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier auto & aggregate_function_node_typed = aggregate_function_node->as(); assertNoAggregateFunctionNodes(aggregate_function_node_typed.getArgumentsNode(), "inside another aggregate function"); - assertNoGroupingFunction(aggregate_function_node_typed.getArgumentsNode(), "inside another aggregate function"); + assertNoGroupingFunctionNodes(aggregate_function_node_typed.getArgumentsNode(), "inside another aggregate function"); assertNoWindowFunctionNodes(aggregate_function_node_typed.getArgumentsNode(), "inside an aggregate function"); } @@ -6703,12 +6708,12 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier if (query_node_typed.getGroupBy().getNodes().empty()) { if (query_node_typed.hasHaving()) - assertNoGroupingFunction(query_node_typed.getHaving(), "in HAVING without GROUP BY"); + assertNoGroupingFunctionNodes(query_node_typed.getHaving(), "in HAVING without GROUP BY"); if (query_node_typed.hasOrderBy()) - assertNoGroupingFunction(query_node_typed.getOrderByNode(), "in ORDER BY without GROUP BY"); + assertNoGroupingFunctionNodes(query_node_typed.getOrderByNode(), "in ORDER BY without GROUP BY"); - assertNoGroupingFunction(query_node_typed.getProjectionNode(), "in SELECT without GROUP BY"); + assertNoGroupingFunctionNodes(query_node_typed.getProjectionNode(), "in SELECT without GROUP BY"); } bool has_aggregation = !query_node_typed.getGroupBy().getNodes().empty() || !aggregate_function_nodes.empty(); @@ -6726,7 +6731,7 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier validate_group_by_columns_visitor.visit(query_node_typed.getProjectionNode()); } - if (!has_aggregation && (query_node_typed.isGroupByWithGroupingSets() || is_rollup_or_cube)) + if (!has_aggregation && (query_node_typed.isGroupByWithGroupingSets() || is_rollup_or_cube || query_node_typed.isGroupByWithTotals())) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "WITH TOTALS, ROLLUP, CUBE or GROUPING SETS are not supported without aggregation"); /** WITH section can be safely removed, because WITH section only can provide aliases to query expressions diff --git a/src/Analyzer/Utils.cpp b/src/Analyzer/Utils.cpp index 5778930c0f5..8433bc5a90c 100644 --- a/src/Analyzer/Utils.cpp +++ b/src/Analyzer/Utils.cpp @@ -418,7 +418,7 @@ private: } -void assertNoFunction(const QueryTreeNodePtr & node, +void assertNoFunctionNodes(const QueryTreeNodePtr & node, std::string_view function_name, int exception_code, std::string_view exception_function_name, @@ -428,4 +428,58 @@ void assertNoFunction(const QueryTreeNodePtr & node, visitor.visit(node); } +namespace +{ + +class CheckFunctionExistsVisitor : public ConstInDepthQueryTreeVisitor +{ +public: + explicit CheckFunctionExistsVisitor(std::string_view function_name_) + : function_name(function_name_) + {} + + void visitImpl(const QueryTreeNodePtr & node) + { + if (has_function) + return; + + auto * function_node = node->as(); + if (!function_node) + return; + + has_function = function_node->getFunctionName() == function_name; + } + + bool needChildVisit(const QueryTreeNodePtr & parent_node, const QueryTreeNodePtr & child_node) + { + if (has_function) + return false; + + if (parent_node->getNodeType() == QueryTreeNodeType::CONSTANT) + return false; + + auto child_node_type = child_node->getNodeType(); + return !(child_node_type == QueryTreeNodeType::QUERY || child_node_type == QueryTreeNodeType::UNION); + } + + bool hasFunction() const + { + return has_function; + } + +private: + std::string_view function_name; + bool has_function = false; +}; + +} + +bool hasFunctionNode(const QueryTreeNodePtr & node, std::string_view function_name) +{ + CheckFunctionExistsVisitor visitor(function_name); + visitor.visit(node); + + return visitor.hasFunction(); +} + } diff --git a/src/Analyzer/Utils.h b/src/Analyzer/Utils.h index 2996d084c38..d8e3a35c3f5 100644 --- a/src/Analyzer/Utils.h +++ b/src/Analyzer/Utils.h @@ -42,13 +42,18 @@ QueryTreeNodes buildTableExpressionsStack(const QueryTreeNodePtr & join_tree_nod */ bool nestedIdentifierCanBeResolved(const DataTypePtr & compound_type, IdentifierView nested_identifier); -/** Assert that there are no function with specified function name in node children. +/** Assert that there are no function nodes with specified function name in node children. * Do not visit subqueries. */ -void assertNoFunction(const QueryTreeNodePtr & node, +void assertNoFunctionNodes(const QueryTreeNodePtr & node, std::string_view function_name, int exception_code, std::string_view exception_function_name, std::string_view exception_place_message); +/** Returns true if there is function node with specified function name in node children, false otherwise. + * Do not visit subqueries. + */ +bool hasFunctionNode(const QueryTreeNodePtr & node, std::string_view function_name); + } diff --git a/src/Analyzer/WindowFunctionsUtils.cpp b/src/Analyzer/WindowFunctionsUtils.cpp index fb411f2418c..200e0d098a9 100644 --- a/src/Analyzer/WindowFunctionsUtils.cpp +++ b/src/Analyzer/WindowFunctionsUtils.cpp @@ -26,8 +26,15 @@ public: : assert_no_window_functions_place_message(std::move(assert_no_window_functions_place_message_)) {} + explicit CollectWindowFunctionNodeVisitor(bool only_check_) + : only_check(only_check_) + {} + void visitImpl(const QueryTreeNodePtr & node) { + if (only_check && has_window_functions) + return; + auto * function_node = node->as(); if (!function_node || !function_node->isWindowFunction()) return; @@ -40,16 +47,31 @@ public: if (window_function_nodes) window_function_nodes->push_back(node); + + has_window_functions = true; } - static bool needChildVisit(const QueryTreeNodePtr &, const QueryTreeNodePtr & child_node) + bool needChildVisit(const QueryTreeNodePtr & parent_node, const QueryTreeNodePtr & child_node) { - return !(child_node->getNodeType() == QueryTreeNodeType::QUERY || child_node->getNodeType() == QueryTreeNodeType::UNION); + if (only_check && has_window_functions) + return false; + + if (parent_node->getNodeType() == QueryTreeNodeType::CONSTANT) + return false; + + auto child_node_type = child_node->getNodeType(); + return !(child_node_type == QueryTreeNodeType::QUERY || child_node_type == QueryTreeNodeType::UNION); } + bool hasWindowFunctions() const + { + return has_window_functions; + } private: QueryTreeNodes * window_function_nodes = nullptr; String assert_no_window_functions_place_message; + bool only_check = false; + bool has_window_functions = false; }; } @@ -63,6 +85,14 @@ QueryTreeNodes collectWindowFunctionNodes(const QueryTreeNodePtr & node) return window_function_nodes; } +bool hasWindowFunctionNodes(const QueryTreeNodePtr & node) +{ + CollectWindowFunctionNodeVisitor visitor(true /*only_check*/); + visitor.visit(node); + + return visitor.hasWindowFunctions(); +} + void collectWindowFunctionNodes(const QueryTreeNodePtr & node, QueryTreeNodes & result) { CollectWindowFunctionNodeVisitor visitor(&result); diff --git a/src/Analyzer/WindowFunctionsUtils.h b/src/Analyzer/WindowFunctionsUtils.h index b6ff5f22f93..7363e2cf6be 100644 --- a/src/Analyzer/WindowFunctionsUtils.h +++ b/src/Analyzer/WindowFunctionsUtils.h @@ -15,6 +15,11 @@ QueryTreeNodes collectWindowFunctionNodes(const QueryTreeNodePtr & node); */ void collectWindowFunctionNodes(const QueryTreeNodePtr & node, QueryTreeNodes & result); +/** Returns true if there are window function nodes in node children, false otherwise. + * Do not visit subqueries. + */ +bool hasWindowFunctionNodes(const QueryTreeNodePtr & node); + /** Assert that there are no window function nodes in node children. * Do not visit subqueries. */ diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp index 30038ae45b9..0cf3f360994 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp @@ -1,6 +1,5 @@ #include #include -#include #include #include #include @@ -57,20 +56,23 @@ ASTPtr rewriteSelectQuery( // are written into the query context and will be sent by the query pipeline. select_query.setExpression(ASTSelectQuery::Expression::SETTINGS, {}); - if (table_function_ptr) - select_query.addTableFunction(table_function_ptr); - else - select_query.replaceDatabaseAndTable(remote_database, remote_table); - - /// Restore long column names (cause our short names are ambiguous). - /// TODO: aliased table functions & CREATE TABLE AS table function cases - if (!table_function_ptr) + if (!context->getSettingsRef().allow_experimental_analyzer) { - RestoreQualifiedNamesVisitor::Data data; - data.distributed_table = DatabaseAndTableWithAlias(*getTableExpression(query->as(), 0)); - data.remote_table.database = remote_database; - data.remote_table.table = remote_table; - RestoreQualifiedNamesVisitor(data).visit(modified_query_ast); + if (table_function_ptr) + select_query.addTableFunction(table_function_ptr); + else + select_query.replaceDatabaseAndTable(remote_database, remote_table); + + /// Restore long column names (cause our short names are ambiguous). + /// TODO: aliased table functions & CREATE TABLE AS table function cases + if (!table_function_ptr) + { + RestoreQualifiedNamesVisitor::Data data; + data.distributed_table = DatabaseAndTableWithAlias(*getTableExpression(query->as(), 0)); + data.remote_table.database = remote_database; + data.remote_table.table = remote_table; + RestoreQualifiedNamesVisitor(data).visit(modified_query_ast); + } } /// To make local JOIN works, default database should be added to table names. diff --git a/src/Planner/CollectColumnIdentifiers.cpp b/src/Planner/CollectColumnIdentifiers.cpp index b1c45fd7a13..ee9b1bf06be 100644 --- a/src/Planner/CollectColumnIdentifiers.cpp +++ b/src/Planner/CollectColumnIdentifiers.cpp @@ -20,15 +20,18 @@ public: , planner_context(planner_context_) {} - static bool needChildVisit(const QueryTreeNodePtr &, const QueryTreeNodePtr & child) + static bool needChildVisit(const QueryTreeNodePtr & parent_node, const QueryTreeNodePtr & child) { - const auto & node_type = child->getNodeType(); - return node_type != QueryTreeNodeType::TABLE - && node_type != QueryTreeNodeType::TABLE_FUNCTION - && node_type != QueryTreeNodeType::QUERY - && node_type != QueryTreeNodeType::UNION - && node_type != QueryTreeNodeType::JOIN - && node_type != QueryTreeNodeType::ARRAY_JOIN; + if (parent_node->getNodeType() == QueryTreeNodeType::CONSTANT) + return false; + + auto child_node_type = child->getNodeType(); + return child_node_type != QueryTreeNodeType::TABLE + && child_node_type != QueryTreeNodeType::TABLE_FUNCTION + && child_node_type != QueryTreeNodeType::QUERY + && child_node_type != QueryTreeNodeType::UNION + && child_node_type != QueryTreeNodeType::JOIN + && child_node_type != QueryTreeNodeType::ARRAY_JOIN; } void visitImpl(const QueryTreeNodePtr & node) diff --git a/src/Planner/CollectSets.cpp b/src/Planner/CollectSets.cpp index e63b3ef078d..2bc8eb8f69d 100644 --- a/src/Planner/CollectSets.cpp +++ b/src/Planner/CollectSets.cpp @@ -81,9 +81,13 @@ public: } } - static bool needChildVisit(const QueryTreeNodePtr &, const QueryTreeNodePtr & child_node) + static bool needChildVisit(const QueryTreeNodePtr & parent_node, const QueryTreeNodePtr & child_node) { - return !(child_node->getNodeType() == QueryTreeNodeType::QUERY || child_node->getNodeType() == QueryTreeNodeType::UNION); + if (parent_node->getNodeType() == QueryTreeNodeType::CONSTANT) + return false; + + auto child_node_type = child_node->getNodeType(); + return !(child_node_type == QueryTreeNodeType::QUERY || child_node_type == QueryTreeNodeType::UNION); } private: diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp index ced76200f79..84fb861c9c3 100644 --- a/src/Planner/Planner.cpp +++ b/src/Planner/Planner.cpp @@ -198,6 +198,7 @@ public: && settings.group_by_overflow_mode == OverflowMode::ANY && settings.totals_mode != TotalsMode::AFTER_HAVING_EXCLUSIVE; aggregate_final = query_processing_info.getToStage() > QueryProcessingStage::WithMergeableState && !query_node.isGroupByWithTotals() && !query_node.isGroupByWithRollup() && !query_node.isGroupByWithCube(); + aggregate_with_grouping_set = query_node.isGroupByWithRollup() || query_node.isGroupByWithCube() || query_node.isGroupByWithGroupingSets(); aggregation_should_produce_results_in_order_of_bucket_number = query_processing_info.getToStage() == QueryProcessingStage::WithMergeableState && settings.distributed_aggregation_memory_efficient; @@ -221,6 +222,7 @@ public: bool aggregate_overflow_row = false; bool aggregate_final = false; + bool aggregate_with_grouping_set = false; bool aggregation_should_produce_results_in_order_of_bucket_number = false; bool query_has_array_join_in_join_tree = false; bool query_has_with_totals_in_any_subquery_in_join_tree = false; @@ -394,7 +396,8 @@ void addMergingAggregatedStep(QueryPlan & query_plan, query_plan.getCurrentDataStream(), params, query_analysis_result.aggregate_final, - settings.distributed_aggregation_memory_efficient && is_remote_storage, + /// Grouping sets don't work with distributed_aggregation_memory_efficient enabled (#43989) + settings.distributed_aggregation_memory_efficient && is_remote_storage && !query_analysis_result.aggregate_with_grouping_set, settings.max_threads, settings.aggregation_memory_efficient_merge_threads, query_analysis_result.aggregation_should_produce_results_in_order_of_bucket_number, @@ -1150,11 +1153,9 @@ void Planner::buildPlanForQueryNode() current_storage_limits.push_back(select_query_info.local_storage_limits); select_query_info.storage_limits = std::make_shared(current_storage_limits); select_query_info.has_order_by = query_node.hasOrderBy(); - auto aggregate_function_nodes = collectAggregateFunctionNodes(query_tree); - auto window_function_nodes = collectWindowFunctionNodes(query_tree); - select_query_info.has_window = !window_function_nodes.empty(); - select_query_info.has_aggregates = !aggregate_function_nodes.empty(); - select_query_info.need_aggregate = query_node.hasGroupBy() || !aggregate_function_nodes.empty(); + select_query_info.has_window = hasWindowFunctionNodes(query_tree); + select_query_info.has_aggregates = hasAggregateFunctionNodes(query_tree); + select_query_info.need_aggregate = query_node.hasGroupBy() || select_query_info.has_aggregates; if (!select_query_info.need_aggregate && query_node.hasHaving()) { @@ -1205,8 +1206,15 @@ void Planner::buildPlanForQueryNode() return; PlannerQueryProcessingInfo query_processing_info(from_stage, select_query_options.to_stage); + + if (!query_processing_info.isFirstStage() && !query_processing_info.isSecondStage() && !query_processing_info.isIntermediateStage()) + return; + QueryAnalysisResult query_analysis_result(query_tree, query_processing_info, planner_context); - auto expression_analysis_result = buildExpressionAnalysisResult(query_tree, query_plan.getCurrentDataStream().header.getColumnsWithTypeAndName(), planner_context); + auto expression_analysis_result = buildExpressionAnalysisResult(query_tree, + query_plan.getCurrentDataStream().header.getColumnsWithTypeAndName(), + planner_context, + query_processing_info); std::vector result_actions_to_execute; diff --git a/src/Planner/PlannerExpressionAnalysis.cpp b/src/Planner/PlannerExpressionAnalysis.cpp index b1017c99c3e..b83675bf86a 100644 --- a/src/Planner/PlannerExpressionAnalysis.cpp +++ b/src/Planner/PlannerExpressionAnalysis.cpp @@ -346,7 +346,7 @@ SortAnalysisResult analyzeSort(const QueryNode & query_node, const PlannerContextPtr & planner_context, ActionsChain & actions_chain) { - const auto *chain_available_output_columns = actions_chain.getLastStepAvailableOutputColumnsOrNull(); + const auto * chain_available_output_columns = actions_chain.getLastStepAvailableOutputColumnsOrNull(); const auto & order_by_input = chain_available_output_columns ? *chain_available_output_columns : join_tree_input_columns; ActionsDAGPtr before_sort_actions = std::make_shared(order_by_input); @@ -388,16 +388,28 @@ SortAnalysisResult analyzeSort(const QueryNode & query_node, LimitByAnalysisResult analyzeLimitBy(const QueryNode & query_node, const ColumnsWithTypeAndName & join_tree_input_columns, const PlannerContextPtr & planner_context, + const NameSet & required_output_nodes_names, ActionsChain & actions_chain) { const auto * chain_available_output_columns = actions_chain.getLastStepAvailableOutputColumnsOrNull(); const auto & limit_by_input = chain_available_output_columns ? *chain_available_output_columns : join_tree_input_columns; auto before_limit_by_actions = buildActionsDAGFromExpressionNode(query_node.getLimitByNode(), limit_by_input, planner_context); + NameSet limit_by_column_names_set; Names limit_by_column_names; limit_by_column_names.reserve(before_limit_by_actions->getOutputs().size()); for (auto & output_node : before_limit_by_actions->getOutputs()) + { + limit_by_column_names_set.insert(output_node->result_name); limit_by_column_names.push_back(output_node->result_name); + } + + for (const auto & node : before_limit_by_actions->getNodes()) + { + if (required_output_nodes_names.contains(node.result_name) && + !limit_by_column_names_set.contains(node.result_name)) + before_limit_by_actions->getOutputs().push_back(&node); + } auto actions_step_before_limit_by = std::make_unique(before_limit_by_actions); actions_chain.addStep(std::move(actions_step_before_limit_by)); @@ -409,7 +421,8 @@ LimitByAnalysisResult analyzeLimitBy(const QueryNode & query_node, PlannerExpressionsAnalysisResult buildExpressionAnalysisResult(const QueryTreeNodePtr & query_tree, const ColumnsWithTypeAndName & join_tree_input_columns, - const PlannerContextPtr & planner_context) + const PlannerContextPtr & planner_context, + const PlannerQueryProcessingInfo & planner_query_processing_info) { auto & query_node = query_tree->as(); @@ -445,10 +458,50 @@ PlannerExpressionsAnalysisResult buildExpressionAnalysisResult(const QueryTreeNo std::optional limit_by_analysis_result_optional; if (query_node.hasLimitBy()) - limit_by_analysis_result_optional = analyzeLimitBy(query_node, join_tree_input_columns, planner_context, actions_chain); + { + /** If we process only first stage of query and there is ORDER BY, we must preserve ORDER BY output columns + * and put them into LIMIT BY output columns, to prevent removing of unused expressions during chain finalize. + * + * Example: SELECT 1 FROM remote('127.0.0.{2,3}', system.one) ORDER BY dummy LIMIT 1 BY 1; + * In this example, LIMIT BY actions does not need `dummy` column, but we must preserve it, because + * otherwise coordinator does not find it in block. + */ + NameSet required_output_nodes_names; + if (sort_analysis_result_optional.has_value() && !planner_query_processing_info.isSecondStage()) + { + const auto & before_order_by_actions = sort_analysis_result_optional->before_order_by_actions; + for (const auto & output_node : before_order_by_actions->getOutputs()) + required_output_nodes_names.insert(output_node->result_name); + } + + limit_by_analysis_result_optional = analyzeLimitBy(query_node, + join_tree_input_columns, + planner_context, + required_output_nodes_names, + actions_chain); + } const auto * chain_available_output_columns = actions_chain.getLastStepAvailableOutputColumnsOrNull(); - const auto & project_names_input = chain_available_output_columns ? *chain_available_output_columns : join_tree_input_columns; + auto project_names_input = chain_available_output_columns ? *chain_available_output_columns : join_tree_input_columns; + + /** If there is DISTINCT we must preserve non constant projection output columns + * in project names actions, to prevent removing of unused expressions during chain finalize. + * + * Example: SELECT DISTINCT id, 1 AS value FROM test_table ORDER BY id; + */ + if (query_node.isDistinct()) + { + std::unordered_set projection_column_names; + for (auto & [column_name, _] : projection_analysis_result.projection_column_names_with_display_aliases) + projection_column_names.insert(column_name); + + for (auto & column : project_names_input) + { + if (projection_column_names.contains(column.name)) + column.column = nullptr; + } + } + auto project_names_actions = std::make_shared(project_names_input); project_names_actions->project(projection_analysis_result.projection_column_names_with_display_aliases); actions_chain.addStep(std::make_unique(project_names_actions)); diff --git a/src/Planner/PlannerExpressionAnalysis.h b/src/Planner/PlannerExpressionAnalysis.h index d697641964c..b6d6ed96e9d 100644 --- a/src/Planner/PlannerExpressionAnalysis.h +++ b/src/Planner/PlannerExpressionAnalysis.h @@ -10,6 +10,7 @@ #include #include #include +#include namespace DB { @@ -170,6 +171,7 @@ private: /// Build expression analysis result for query tree, join tree input columns and planner context PlannerExpressionsAnalysisResult buildExpressionAnalysisResult(const QueryTreeNodePtr & query_tree, const ColumnsWithTypeAndName & join_tree_input_columns, - const PlannerContextPtr & planner_context); + const PlannerContextPtr & planner_context, + const PlannerQueryProcessingInfo & planner_query_processing_info); } diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index f23ed8f3d0e..37aeaa650b0 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -38,7 +38,11 @@ #include #include +#include #include +#include +#include +#include #include #include @@ -618,19 +622,84 @@ StorageSnapshotPtr StorageDistributed::getStorageSnapshotForQuery( namespace { -QueryTreeNodePtr buildQueryTreeDistributedTableReplacedWithLocalTable(const SelectQueryInfo & query_info, StorageID remote_storage_id) +class StorageDistributedLocal : public IStorage +{ +public: + StorageDistributedLocal(const StorageID & table_id_, const ColumnsDescription & columns_) + : IStorage(table_id_) + { + StorageInMemoryMetadata storage_metadata; + storage_metadata.setColumns(columns_); + setInMemoryMetadata(storage_metadata); + } + + std::string getName() const override { return "StorageDistributedLocal"; } + + bool supportsSampling() const override { return true; } + bool supportsFinal() const override { return true; } + bool supportsPrewhere() const override { return true; } + bool supportsSubcolumns() const override { return true; } + bool supportsDynamicSubcolumns() const override { return true; } + bool canMoveConditionsToPrewhere() const override { return false; } + + QueryProcessingStage::Enum + getQueryProcessingStage(ContextPtr, QueryProcessingStage::Enum, const StorageSnapshotPtr &, SelectQueryInfo &) const override + { + throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "StorageDistributedLocal does not support getQueryProcessingStage method"); + } + + Pipe read(const Names & /*column_names*/, + const StorageSnapshotPtr & /*storage_snapshot*/, + SelectQueryInfo & /*query_info*/, + ContextPtr /*context*/, + QueryProcessingStage::Enum /*processed_stage*/, + size_t /*max_block_size*/, + size_t /*num_streams*/) override + { + throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "StorageDistributedLocal does not support read method"); + } +}; + +QueryTreeNodePtr buildQueryTreeDistributedTableReplacedWithLocalTable(const SelectQueryInfo & query_info, + const StorageSnapshotPtr & distributed_storage_snapshot, + const StorageID & remote_storage_id, + const ASTPtr & remote_table_function) { const auto & query_context = query_info.planner_context->getQueryContext(); - auto resolved_remote_storage_id = query_context->resolveStorageID(remote_storage_id); - auto storage = DatabaseCatalog::instance().tryGetTable(resolved_remote_storage_id, query_context); - if (!storage) - throw Exception(ErrorCodes::UNSUPPORTED_METHOD, - "Distributed local table {} does not exists on coordinator", - remote_storage_id.getFullTableName()); - auto storage_lock = storage->lockForShare(query_context->getInitialQueryId(), query_context->getSettingsRef().lock_acquire_timeout); - auto storage_snapshot = storage->getStorageSnapshot(storage->getInMemoryMetadataPtr(), query_context); - auto replacement_table_expression = std::make_shared(std::move(storage), std::move(storage_lock), std::move(storage_snapshot)); + QueryTreeNodePtr replacement_table_expression; + + if (remote_table_function) + { + auto remote_table_function_query_tree = buildQueryTree(remote_table_function, query_context); + auto & remote_table_function_node = remote_table_function_query_tree->as(); + + auto table_function_node = std::make_shared(remote_table_function_node.getFunctionName()); + table_function_node->getArgumentsNode() = remote_table_function_node.getArgumentsNode(); + + QueryAnalysisPass query_analysis_pass; + query_analysis_pass.run(table_function_node->getArgumentsNode(), query_context); + + auto remote_table_function_to_resolve = table_function_node->toAST(); + TableFunctionPtr table_function_ptr = TableFunctionFactory::instance().get(remote_table_function_to_resolve, query_context); + auto table_function_storage = table_function_ptr->execute(remote_table_function_to_resolve, query_context, table_function_ptr->getName()); + + table_function_node->resolve(std::move(table_function_ptr), std::move(table_function_storage), query_context); + replacement_table_expression = std::move(table_function_node); + } + else + { + auto resolved_remote_storage_id = query_context->resolveStorageID(remote_storage_id); + auto storage = DatabaseCatalog::instance().tryGetTable(resolved_remote_storage_id, query_context); + if (!storage) + storage = std::make_shared(resolved_remote_storage_id, distributed_storage_snapshot->metadata->getColumns()); + + auto storage_lock = storage->lockForShare(query_context->getInitialQueryId(), query_context->getSettingsRef().lock_acquire_timeout); + auto storage_snapshot = storage->getStorageSnapshot(storage->getInMemoryMetadataPtr(), query_context); + replacement_table_expression = std::make_shared(std::move(storage), std::move(storage_lock), std::move(storage_snapshot)); + } + + replacement_table_expression->setAlias(query_info.table_expression->getAlias()); std::unordered_map replacement_map; replacement_map.emplace(query_info.table_expression.get(), std::move(replacement_table_expression)); @@ -650,7 +719,6 @@ void StorageDistributed::read( const size_t /*max_block_size*/, const size_t /*num_streams*/) { - const auto * select_query = query_info.query->as(); if (select_query->final() && local_context->getSettingsRef().allow_experimental_parallel_reading_from_replicas) throw Exception(ErrorCodes::ILLEGAL_FINAL, "Final modifier is not allowed together with parallel reading from replicas feature"); @@ -660,8 +728,15 @@ void StorageDistributed::read( if (local_context->getSettingsRef().allow_experimental_analyzer) { - StorageID remote_storage_id{remote_database, remote_table}; - auto query_tree_with_replaced_distributed_table = buildQueryTreeDistributedTableReplacedWithLocalTable(query_info, remote_storage_id); + StorageID remote_storage_id = StorageID::createEmpty(); + if (!remote_table_function_ptr) + remote_storage_id = StorageID{remote_database, remote_table}; + + auto query_tree_with_replaced_distributed_table = buildQueryTreeDistributedTableReplacedWithLocalTable(query_info, + storage_snapshot, + remote_storage_id, + remote_table_function_ptr); + query_ast = queryNodeToSelectQuery(query_tree_with_replaced_distributed_table); Planner planner(query_tree_with_replaced_distributed_table, diff --git a/src/Storages/StorageTableFunction.h b/src/Storages/StorageTableFunction.h index 9ba7497fbf2..ccec087a8d9 100644 --- a/src/Storages/StorageTableFunction.h +++ b/src/Storages/StorageTableFunction.h @@ -103,9 +103,6 @@ public: size_t max_block_size, size_t num_streams) override { - String cnames; - for (const auto & c : column_names) - cnames += c + " "; auto storage = getNested(); auto nested_snapshot = storage->getStorageSnapshot(storage->getInMemoryMetadataPtr(), context); storage->read(query_plan, column_names, nested_snapshot, query_info, context, diff --git a/tests/queries/0_stateless/02497_having_without_actual_aggregation_bug.reference b/tests/queries/0_stateless/02497_having_without_actual_aggregation_bug.reference index 3a52d32f5de..32d7afafb33 100644 --- a/tests/queries/0_stateless/02497_having_without_actual_aggregation_bug.reference +++ b/tests/queries/0_stateless/02497_having_without_actual_aggregation_bug.reference @@ -1,3 +1,2 @@ 9 100 -10 diff --git a/tests/queries/0_stateless/02497_having_without_actual_aggregation_bug.sql b/tests/queries/0_stateless/02497_having_without_actual_aggregation_bug.sql index 5ddd6119bbe..e38de101a22 100644 --- a/tests/queries/0_stateless/02497_having_without_actual_aggregation_bug.sql +++ b/tests/queries/0_stateless/02497_having_without_actual_aggregation_bug.sql @@ -1,9 +1,8 @@ +SET allow_experimental_analyzer = 1; + select number from numbers_mt(10) having number >= 9; select count() from numbers_mt(100) having count() > 1; select queryID() as t from numbers(10) with totals having t = initialQueryID(); -- { serverError 48 } - --- this query works despite 'with total' doesn't make any sense due to this logic: --- https://github.com/ClickHouse/ClickHouse/blob/master/src/Interpreters/InterpreterSelectQuery.cpp#L608-L610 -select count() from (select queryID() as t from remote('127.0.0.{1..3}', numbers(10)) with totals having t = initialQueryID()) settings prefer_localhost_replica = 1; +select count() from (select queryID() as t from remote('127.0.0.{1..3}', numbers(10)) with totals having t = initialQueryID()) settings prefer_localhost_replica = 1; -- { serverError 48 } From c29f3c0f99094b3d8c6fd1adf0610347f3e9f385 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 8 Feb 2023 19:30:06 +0100 Subject: [PATCH 311/717] Analyzer fix GROUP BY columns validation --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 41 +++++++++++++---------- src/Planner/Planner.cpp | 4 --- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 6d9208428d8..15ef30568f0 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -6209,6 +6209,9 @@ public: query_tree_node_type == QueryTreeNodeType::INTERPOLATE) return; + if (nodeIsAggregateFunctionOrInGroupByKeys(node)) + return; + auto * function_node = node->as(); if (function_node && function_node->getFunctionName() == "grouping") { @@ -6244,12 +6247,6 @@ public: if (column_node_source->getNodeType() == QueryTreeNodeType::LAMBDA) return; - for (const auto & group_by_key_node : group_by_keys_nodes) - { - if (node->isEqual(*group_by_key_node)) - return; - } - std::string column_name; if (column_node_source->hasAlias()) @@ -6268,24 +6265,32 @@ public: scope.scope_node->formatASTForErrorMessage()); } - bool needChildVisit(const QueryTreeNodePtr &, const QueryTreeNodePtr & child_node) + bool needChildVisit(const QueryTreeNodePtr & parent_node, const QueryTreeNodePtr & child_node) { - if (auto * child_function_node = child_node->as()) - { - if (child_function_node->isAggregateFunction()) - return false; + if (nodeIsAggregateFunctionOrInGroupByKeys(parent_node)) + return false; - for (const auto & group_by_key_node : group_by_keys_nodes) - { - if (child_node->isEqual(*group_by_key_node, {.compare_aliases = false})) - return false; - } - } + if (parent_node->getNodeType() == QueryTreeNodeType::CONSTANT) + return false; - return !(child_node->getNodeType() == QueryTreeNodeType::QUERY || child_node->getNodeType() == QueryTreeNodeType::UNION); + auto child_node_type = child_node->getNodeType(); + return !(child_node_type == QueryTreeNodeType::QUERY || child_node_type == QueryTreeNodeType::UNION); } private: + bool nodeIsAggregateFunctionOrInGroupByKeys(const QueryTreeNodePtr & node) const + { + if (auto * function_node = node->as()) + if (function_node->isAggregateFunction()) + return true; + + for (const auto & group_by_key_node : group_by_keys_nodes) + if (node->isEqual(*group_by_key_node, {.compare_aliases = false})) + return true; + + return false; + } + const QueryTreeNodes & group_by_keys_nodes; const IdentifierResolveScope & scope; }; diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp index 84fb861c9c3..789a010c737 100644 --- a/src/Planner/Planner.cpp +++ b/src/Planner/Planner.cpp @@ -1206,10 +1206,6 @@ void Planner::buildPlanForQueryNode() return; PlannerQueryProcessingInfo query_processing_info(from_stage, select_query_options.to_stage); - - if (!query_processing_info.isFirstStage() && !query_processing_info.isSecondStage() && !query_processing_info.isIntermediateStage()) - return; - QueryAnalysisResult query_analysis_result(query_tree, query_processing_info, planner_context); auto expression_analysis_result = buildExpressionAnalysisResult(query_tree, query_plan.getCurrentDataStream().header.getColumnsWithTypeAndName(), From b2cc71f413137c6575196d4c2c1c043d10316d1d Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Thu, 9 Feb 2023 18:30:57 +0100 Subject: [PATCH 312/717] Fixed tests --- src/Analyzer/ConstantNode.cpp | 62 ++++++++++++++++++- src/Analyzer/Passes/QueryAnalysisPass.cpp | 49 +++++++++++++-- src/Analyzer/QueryTreeBuilder.cpp | 5 ++ .../InterpreterSelectQueryAnalyzer.cpp | 3 - src/Planner/ActionsChain.cpp | 36 +++-------- src/Planner/ActionsChain.h | 20 ++---- src/Planner/Planner.cpp | 49 ++++++++------- src/Planner/PlannerExpressionAnalysis.cpp | 17 +++-- src/Planner/PlannerJoinTree.cpp | 7 ++- .../0_stateless/00118_storage_join.sql | 4 +- ..._duplicate_columns_in_subqueries.reference | 4 +- .../00370_duplicate_columns_in_subqueries.sql | 1 + .../0_stateless/00561_storage_join.sql | 4 +- 13 files changed, 172 insertions(+), 89 deletions(-) diff --git a/src/Analyzer/ConstantNode.cpp b/src/Analyzer/ConstantNode.cpp index 28b2f5a0ed0..32c97a1b295 100644 --- a/src/Analyzer/ConstantNode.cpp +++ b/src/Analyzer/ConstantNode.cpp @@ -10,6 +10,7 @@ #include #include +#include #include @@ -76,7 +77,66 @@ QueryTreeNodePtr ConstantNode::cloneImpl() const ASTPtr ConstantNode::toASTImpl() const { - return std::make_shared(constant_value->getValue()); + const auto & constant_value_literal = constant_value->getValue(); + auto constant_value_ast = std::make_shared(constant_value_literal); + + bool need_to_add_cast_function = false; + auto constant_value_literal_type = constant_value_literal.getType(); + WhichDataType constant_value_type(constant_value->getType()); + + switch (constant_value_literal_type) + { + case Field::Types::UInt64: + { + need_to_add_cast_function = !constant_value_type.isUInt64(); + break; + } + case Field::Types::Int64: + { + need_to_add_cast_function = !constant_value_type.isInt64(); + break; + } + case Field::Types::Float64: + { + need_to_add_cast_function = !constant_value_type.isFloat64(); + break; + } + case Field::Types::String: + { + need_to_add_cast_function = !constant_value_type.isString(); + break; + } + case Field::Types::Int128: + case Field::Types::UInt128: + case Field::Types::Int256: + case Field::Types::UInt256: + case Field::Types::Decimal32: + case Field::Types::Decimal64: + case Field::Types::Decimal128: + case Field::Types::Decimal256: + case Field::Types::AggregateFunctionState: + case Field::Types::Array: + case Field::Types::Tuple: + case Field::Types::Map: + case Field::Types::UUID: + case Field::Types::Bool: + case Field::Types::Object: + case Field::Types::IPv4: + case Field::Types::IPv6: + case Field::Types::Null: + { + need_to_add_cast_function = true; + break; + } + } + + if (need_to_add_cast_function) + { + auto constant_type_name_ast = std::make_shared(constant_value->getType()->getName()); + return makeASTFunction("_CAST", std::move(constant_value_ast), std::move(constant_type_name_ast)); + } + + return constant_value_ast; } } diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 15ef30568f0..fb50c1f9d66 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -1282,10 +1282,10 @@ private: std::unordered_map function_name_to_user_defined_lambda; /// Array join expressions counter - size_t array_join_expressions_counter = 0; + size_t array_join_expressions_counter = 1; /// Subquery counter - size_t subquery_counter = 0; + size_t subquery_counter = 1; /// Global expression node to projection name map std::unordered_map node_to_projection_name; @@ -1907,6 +1907,26 @@ void QueryAnalyzer::evaluateScalarSubqueryIfNeeded(QueryTreeNodePtr & node, Iden } else { + /** Make unique column names for tuple. + * + * Example: SELECT (SELECT 2 AS x, x) + */ + NameSet block_column_names; + size_t unique_column_name_counter = 1; + + for (auto & column_with_type : block) + { + if (!block_column_names.contains(column_with_type.name)) + { + block_column_names.insert(column_with_type.name); + continue; + } + + column_with_type.name += '_'; + column_with_type.name += std::to_string(unique_column_name_counter); + ++unique_column_name_counter; + } + scalar_block.insert({ ColumnTuple::create(block.getColumns()), std::make_shared(block.getDataTypes(), block.getNames()), @@ -1939,7 +1959,7 @@ void QueryAnalyzer::evaluateScalarSubqueryIfNeeded(QueryTreeNodePtr & node, Iden if (constant_node->getValue().isNull()) { std::string cast_type = constant_node->getResultType()->getName(); - std::string cast_function_name = "__CAST"; + std::string cast_function_name = "_CAST"; auto cast_type_constant_value = std::make_shared(std::move(cast_type), std::make_shared()); auto cast_type_constant_node = std::make_shared(std::move(cast_type_constant_value)); @@ -2131,6 +2151,11 @@ void QueryAnalyzer::validateJoinTableExpressionWithoutAlias(const QueryTreeNodeP if (table_expression_has_alias) return; + auto * query_node = table_expression_node->as(); + auto * union_node = table_expression_node->as(); + if ((query_node && !query_node->getCTEName().empty()) || (union_node && !union_node->getCTEName().empty())) + return; + auto table_expression_node_type = table_expression_node->getNodeType(); if (table_expression_node_type == QueryTreeNodeType::TABLE_FUNCTION || table_expression_node_type == QueryTreeNodeType::QUERY || @@ -5207,8 +5232,8 @@ ProjectionNames QueryAnalyzer::resolveExpressionNode(QueryTreeNodePtr & node, Id IdentifierResolveScope subquery_scope(node, &scope /*parent_scope*/); subquery_scope.subquery_depth = scope.subquery_depth + 1; - ++subquery_counter; std::string projection_name = "_subquery_" + std::to_string(subquery_counter); + ++subquery_counter; if (node_type == QueryTreeNodeType::QUERY) resolveQuery(node, subquery_scope); @@ -5547,7 +5572,7 @@ void QueryAnalyzer::initializeQueryJoinTreeNode(QueryTreeNodePtr & join_tree_nod auto resolved_identifier = table_identifier_resolve_result.resolved_identifier; if (!resolved_identifier) - throw Exception(ErrorCodes::UNKNOWN_IDENTIFIER, + throw Exception(ErrorCodes::UNKNOWN_TABLE, "Unknown table expression identifier '{}' in scope {}", from_table_identifier.getIdentifier().getFullName(), scope.scope_node->formatASTForErrorMessage()); @@ -6066,6 +6091,13 @@ void QueryAnalyzer::resolveJoin(QueryTreeNodePtr & join_node, IdentifierResolveS IdentifierLookup identifier_lookup{identifier_node->getIdentifier(), IdentifierLookupContext::EXPRESSION}; auto result_left_table_expression = tryResolveIdentifierFromJoinTreeNode(identifier_lookup, join_node_typed.getLeftTableExpression(), scope); + if (!result_left_table_expression && identifier_node->hasAlias()) + { + std::vector alias_identifier_parts = {identifier_node->getAlias()}; + IdentifierLookup alias_identifier_lookup{Identifier(std::move(alias_identifier_parts)), IdentifierLookupContext::EXPRESSION}; + result_left_table_expression = tryResolveIdentifierFromJoinTreeNode(alias_identifier_lookup, join_node_typed.getLeftTableExpression(), scope); + } + if (!result_left_table_expression) throw Exception(ErrorCodes::UNKNOWN_IDENTIFIER, "JOIN {} using identifier '{}' cannot be resolved from left table expression. In scope {}", @@ -6081,6 +6113,13 @@ void QueryAnalyzer::resolveJoin(QueryTreeNodePtr & join_node, IdentifierResolveS scope.scope_node->formatASTForErrorMessage()); auto result_right_table_expression = tryResolveIdentifierFromJoinTreeNode(identifier_lookup, join_node_typed.getRightTableExpression(), scope); + if (!result_right_table_expression && identifier_node->hasAlias()) + { + std::vector alias_identifier_parts = {identifier_node->getAlias()}; + IdentifierLookup alias_identifier_lookup{Identifier(std::move(alias_identifier_parts)), IdentifierLookupContext::EXPRESSION}; + result_right_table_expression = tryResolveIdentifierFromJoinTreeNode(alias_identifier_lookup, join_node_typed.getRightTableExpression(), scope); + } + if (!result_right_table_expression) throw Exception(ErrorCodes::UNKNOWN_IDENTIFIER, "JOIN {} using identifier '{}' cannot be resolved from right table expression. In scope {}", diff --git a/src/Analyzer/QueryTreeBuilder.cpp b/src/Analyzer/QueryTreeBuilder.cpp index 0d270c41bdf..9348332923e 100644 --- a/src/Analyzer/QueryTreeBuilder.cpp +++ b/src/Analyzer/QueryTreeBuilder.cpp @@ -438,6 +438,11 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression, co auto identifier = Identifier(ast_identifier->name_parts); result = std::make_shared(std::move(identifier)); } + else if (const auto * table_identifier = expression->as()) + { + auto identifier = Identifier(table_identifier->name_parts); + result = std::make_shared(std::move(identifier)); + } else if (const auto * asterisk = expression->as()) { auto column_transformers = buildColumnTransformers(asterisk->transformers, context); diff --git a/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp b/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp index 5f3a83b32ac..e21bf981181 100644 --- a/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp +++ b/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp @@ -18,9 +18,6 @@ #include #include -#include -#include "config_version.h" - namespace DB { diff --git a/src/Planner/ActionsChain.cpp b/src/Planner/ActionsChain.cpp index 594d26a679c..c5438b5d2d4 100644 --- a/src/Planner/ActionsChain.cpp +++ b/src/Planner/ActionsChain.cpp @@ -11,24 +11,16 @@ namespace DB { -ActionsChainStep::ActionsChainStep(ActionsDAGPtr actions_, AvailableOutputColumnsStrategy available_output_columns_stategy_) - : actions(std::move(actions_)) - , available_output_columns_strategy(available_output_columns_stategy_) -{ - initialize(); -} - ActionsChainStep::ActionsChainStep(ActionsDAGPtr actions_, - AvailableOutputColumnsStrategy available_output_columns_stategy_, + bool use_actions_nodes_as_output_columns_, ColumnsWithTypeAndName additional_output_columns_) : actions(std::move(actions_)) - , available_output_columns_strategy(available_output_columns_stategy_) + , use_actions_nodes_as_output_columns(use_actions_nodes_as_output_columns_) , additional_output_columns(std::move(additional_output_columns_)) { initialize(); } - void ActionsChainStep::finalizeInputAndOutputColumns(const NameSet & child_input_columns) { child_required_output_columns_names.clear(); @@ -68,10 +60,10 @@ void ActionsChainStep::dump(WriteBuffer & buffer) const buffer << "DAG" << '\n'; buffer << actions->dumpDAG(); - if (!additional_output_columns.empty()) + if (!available_output_columns.empty()) { - buffer << "Additional output columns " << additional_output_columns.size() << '\n'; - for (const auto & column : additional_output_columns) + buffer << "Available output columns " << available_output_columns.size() << '\n'; + for (const auto & column : available_output_columns) buffer << "Name " << column.name << " type " << column.type->getName() << '\n'; } @@ -97,11 +89,10 @@ void ActionsChainStep::initialize() available_output_columns.clear(); - /// TODO: Analyzer fix ActionsDAG input and constant nodes with same name - std::unordered_set available_output_columns_names; - - if (available_output_columns_strategy == AvailableOutputColumnsStrategy::ALL_NODES) + if (use_actions_nodes_as_output_columns) { + std::unordered_set available_output_columns_names; + for (const auto & node : actions->getNodes()) { if (available_output_columns_names.contains(node.result_name)) @@ -111,17 +102,6 @@ void ActionsChainStep::initialize() available_output_columns_names.insert(node.result_name); } } - else if (available_output_columns_strategy == AvailableOutputColumnsStrategy::OUTPUT_NODES) - { - for (const auto & node : actions->getOutputs()) - { - if (available_output_columns_names.contains(node->result_name)) - continue; - - available_output_columns.emplace_back(node->column, node->result_type, node->result_name); - available_output_columns_names.insert(node->result_name); - } - } available_output_columns.insert(available_output_columns.end(), additional_output_columns.begin(), additional_output_columns.end()); } diff --git a/src/Planner/ActionsChain.h b/src/Planner/ActionsChain.h index e2791ab7e35..7e8e93878e0 100644 --- a/src/Planner/ActionsChain.h +++ b/src/Planner/ActionsChain.h @@ -43,24 +43,12 @@ using ActionsChainSteps = std::vector; class ActionsChainStep { public: - /// Available output columns strategy for actions chain step - enum class AvailableOutputColumnsStrategy - { - ALL_NODES, - OUTPUT_NODES - }; - /** Initialize actions step with actions dag. * Input column names initialized using actions dag nodes with INPUT type. - * - * If available output columns strategy is ALL_NODES, then available output columns initialized using actions dag nodes. - * If available output columns strategy is OUTPUT_NODES, then available output columns initialized using actions dag output nodes. + * If use_actions_nodes_as_output_columns = true output columns are initialized using actions dag nodes. + * If additional output columns are specified they are added to output columns. */ - explicit ActionsChainStep(ActionsDAGPtr actions_, AvailableOutputColumnsStrategy available_output_columns_stategy_ = AvailableOutputColumnsStrategy::ALL_NODES); - - explicit ActionsChainStep(ActionsDAGPtr actions_, - AvailableOutputColumnsStrategy available_output_columns_stategy_, - ColumnsWithTypeAndName additional_output_columns_); + explicit ActionsChainStep(ActionsDAGPtr actions_, bool use_actions_nodes_as_output_columns = true, ColumnsWithTypeAndName additional_output_columns_ = {}); /// Get actions ActionsDAGPtr & getActions() @@ -110,7 +98,7 @@ private: ActionsDAGPtr actions; - AvailableOutputColumnsStrategy available_output_columns_strategy; + bool use_actions_nodes_as_output_columns = true; NameSet input_columns_names; diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp index 789a010c737..f84e4db4924 100644 --- a/src/Planner/Planner.cpp +++ b/src/Planner/Planner.cpp @@ -218,6 +218,17 @@ public: /// Constness of offset is validated during query analysis stage limit_offset = query_node.getOffset()->as().getValue().safeGet(); } + + /// Partial sort can be done if there is LIMIT, but no DISTINCT, LIMIT WITH TIES, LIMIT BY, ARRAY JOIN + if (limit_length != 0 && + !query_node.isDistinct() && + !query_node.isLimitWithTies() && + !query_node.hasLimitBy() && + !query_has_array_join_in_join_tree && + limit_length <= std::numeric_limits::max() - limit_offset) + { + partial_sorting_limit = limit_length + limit_offset; + } } bool aggregate_overflow_row = false; @@ -229,6 +240,7 @@ public: SortDescription sort_description; UInt64 limit_length = 0; UInt64 limit_offset = 0; + UInt64 partial_sorting_limit = 0; }; void addExpressionStep(QueryPlan & query_plan, @@ -513,23 +525,9 @@ void addDistinctStep(QueryPlan & query_plan, void addSortingStep(QueryPlan & query_plan, const QueryAnalysisResult & query_analysis_result, - const PlannerContextPtr & planner_context, - const QueryNode & query_node) + const PlannerContextPtr & planner_context) { const auto & sort_description = query_analysis_result.sort_description; - UInt64 limit_length = query_analysis_result.limit_length; - UInt64 limit_offset = query_analysis_result.limit_offset; - - UInt64 partial_sorting_limit = 0; - - /// Partial sort can be done if there is LIMIT, but no DISTINCT, LIMIT WITH TIES, LIMIT BY, ARRAY JOIN - if (limit_length != 0 && !query_node.isDistinct() && !query_node.hasLimitBy() && !query_node.isLimitWithTies() && - !query_analysis_result.query_has_array_join_in_join_tree && - limit_length <= std::numeric_limits::max() - limit_offset) - { - partial_sorting_limit = limit_length + limit_offset; - } - const auto & query_context = planner_context->getQueryContext(); const Settings & settings = query_context->getSettingsRef(); SortingStep::Settings sort_settings(*query_context); @@ -537,7 +535,7 @@ void addSortingStep(QueryPlan & query_plan, auto sorting_step = std::make_unique( query_plan.getCurrentDataStream(), sort_description, - partial_sorting_limit, + query_analysis_result.partial_sorting_limit, sort_settings, settings.optimize_sorting_by_input_stream_properties); sorting_step->setStepDescription("Sorting for ORDER BY"); @@ -553,10 +551,12 @@ void addMergeSortingStep(QueryPlan & query_plan, const auto & settings = query_context->getSettingsRef(); const auto & sort_description = query_analysis_result.sort_description; - UInt64 limit_length = query_analysis_result.limit_length; const auto max_block_size = settings.max_block_size; - auto merging_sorted = std::make_unique(query_plan.getCurrentDataStream(), sort_description, max_block_size, limit_length); + auto merging_sorted = std::make_unique(query_plan.getCurrentDataStream(), + sort_description, + max_block_size, + query_analysis_result.partial_sorting_limit); merging_sorted->setStepDescription("Merge sorted streams " + description); query_plan.addStep(std::move(merging_sorted)); } @@ -719,6 +719,7 @@ bool addPreliminaryLimitOptimizationStepIfNeeded(QueryPlan & query_plan, bool apply_prelimit = apply_limit && query_node.hasLimit() && !query_node.isLimitWithTies() && + !query_node.isGroupByWithTotals() && !query_analysis_result.query_has_with_totals_in_any_subquery_in_join_tree && !query_analysis_result.query_has_array_join_in_join_tree && !query_node.isDistinct() && @@ -756,7 +757,7 @@ void addPreliminarySortOrDistinctOrLimitStepsIfNeeded(QueryPlan & query_plan, return; if (expressions_analysis_result.hasSort()) - addSortingStep(query_plan, query_analysis_result, planner_context, query_node); + addSortingStep(query_plan, query_analysis_result, planner_context); /** For DISTINCT step, pre_distinct = false, because if we have limit and distinct, * we need to merge streams to one and calculate overall distinct. @@ -1381,13 +1382,13 @@ void Planner::buildPlanForQueryNode() */ if (query_processing_info.isFromAggregationState()) addMergeSortingStep(query_plan, query_analysis_result, planner_context, "after aggregation stage for ORDER BY"); - else if (!query_processing_info.isFirstStage() - && !expression_analysis_result.hasAggregation() - && !expression_analysis_result.hasWindow() - && !(query_node.isGroupByWithTotals() && !query_analysis_result.aggregate_final)) + else if (!query_processing_info.isFirstStage() && + !expression_analysis_result.hasAggregation() && + !expression_analysis_result.hasWindow() && + !(query_node.isGroupByWithTotals() && !query_analysis_result.aggregate_final)) addMergeSortingStep(query_plan, query_analysis_result, planner_context, "for ORDER BY, without aggregation"); else - addSortingStep(query_plan, query_analysis_result, planner_context, query_node); + addSortingStep(query_plan, query_analysis_result, planner_context); } /** Optimization if there are several sources and there is LIMIT, then first apply the preliminary LIMIT, diff --git a/src/Planner/PlannerExpressionAnalysis.cpp b/src/Planner/PlannerExpressionAnalysis.cpp index b83675bf86a..a813d3b753a 100644 --- a/src/Planner/PlannerExpressionAnalysis.cpp +++ b/src/Planner/PlannerExpressionAnalysis.cpp @@ -62,10 +62,10 @@ std::optional analyzeAggregation(const QueryTreeNodeP auto aggregate_function_nodes = collectAggregateFunctionNodes(query_tree); auto aggregates_descriptions = extractAggregateDescriptions(aggregate_function_nodes, *planner_context); - ColumnsWithTypeAndName aggregates_columns; - aggregates_columns.reserve(aggregates_descriptions.size()); + ColumnsWithTypeAndName available_columns_after_aggregation; + available_columns_after_aggregation.reserve(aggregates_descriptions.size()); for (auto & aggregate_description : aggregates_descriptions) - aggregates_columns.emplace_back(nullptr, aggregate_description.function->getResultType(), aggregate_description.column_name); + available_columns_after_aggregation.emplace_back(nullptr, aggregate_description.function->getResultType(), aggregate_description.column_name); Names aggregation_keys; @@ -157,6 +157,9 @@ std::optional analyzeAggregation(const QueryTreeNodeP } } + for (auto & node : before_aggregation_actions->getOutputs()) + available_columns_after_aggregation.emplace_back(nullptr, node->result_type, node->result_name); + /// Add expressions from aggregate functions arguments for (auto & aggregate_function_node : aggregate_function_nodes) @@ -183,10 +186,12 @@ std::optional analyzeAggregation(const QueryTreeNodeP * With set number, which is used as an additional key at the stage of merging aggregating data. */ if (query_node.isGroupByWithRollup() || query_node.isGroupByWithCube() || query_node.isGroupByWithGroupingSets()) - aggregates_columns.emplace_back(nullptr, std::make_shared(), "__grouping_set"); + available_columns_after_aggregation.emplace_back(nullptr, std::make_shared(), "__grouping_set"); /// Only aggregation keys and aggregates are available for next steps after GROUP BY step - auto aggregate_step = std::make_unique(before_aggregation_actions, ActionsChainStep::AvailableOutputColumnsStrategy::OUTPUT_NODES, aggregates_columns); + auto aggregate_step = std::make_unique(before_aggregation_actions, + false /*use_actions_nodes_as_output_columns*/, + available_columns_after_aggregation); actions_chain.addStep(std::move(aggregate_step)); AggregationAnalysisResult aggregation_analysis_result; @@ -277,7 +282,7 @@ std::optional analyzeWindow(const QueryTreeNodePtr & query window_functions_additional_columns.emplace_back(nullptr, window_function.aggregate_function->getResultType(), window_function.column_name); auto before_window_step = std::make_unique(before_window_actions, - ActionsChainStep::AvailableOutputColumnsStrategy::ALL_NODES, + true /*use_actions_nodes_as_output_columns*/, window_functions_additional_columns); actions_chain.addStep(std::move(before_window_step)); diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index b37fd4a6dca..a34fced26ad 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -929,6 +929,7 @@ JoinTreeQueryPlan buildJoinTreeQueryPlan(const QueryTreeNodePtr & query_node, } std::vector query_plans_stack; + bool has_remote_table = false; for (size_t i = 0; i < table_expressions_stack_size; ++i) { @@ -969,9 +970,11 @@ JoinTreeQueryPlan buildJoinTreeQueryPlan(const QueryTreeNodePtr & query_node, else { const auto & table_expression_data = planner_context->getTableExpressionDataOrThrow(table_expression); - if (table_expression_data.isRemote() && !is_single_table_expression) + if (table_expression_data.isRemote() && has_remote_table) throw Exception(ErrorCodes::UNSUPPORTED_METHOD, - "JOIN with remote storages is unsuppored"); + "JOIN with multiple remote storages is unsuppored"); + + has_remote_table = table_expression_data.isRemote(); query_plans_stack.push_back(buildQueryPlanForTableExpression(table_expression, select_query_info, diff --git a/tests/queries/0_stateless/00118_storage_join.sql b/tests/queries/0_stateless/00118_storage_join.sql index 52b0df4d796..17cfe437448 100644 --- a/tests/queries/0_stateless/00118_storage_join.sql +++ b/tests/queries/0_stateless/00118_storage_join.sql @@ -1,3 +1,5 @@ +SET allow_experimental_analyzer = 1; + DROP TABLE IF EXISTS t2; CREATE TABLE t2 (k UInt64, s String) ENGINE = Join(ANY, LEFT, k); @@ -16,6 +18,6 @@ SELECT k, js1.s, t2.s FROM (SELECT toUInt64(number / 3) AS k, sum(number) as s F SELECT k, js1.s, t2.s FROM (SELECT number AS k, number AS s FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 ON js1.k == t2.k ORDER BY k; SELECT k, t2.k, js1.s, t2.s FROM (SELECT number AS k, number AS s FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 ON js1.k == t2.k ORDER BY k; -SELECT k, js1.s, t2.s FROM (SELECT number AS k, number AS s FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 ON js1.k == t2.k OR js1.s == t2.k ORDER BY k; -- { serverError 48 } +SELECT k, js1.s, t2.s FROM (SELECT number AS k, number AS s FROM system.numbers LIMIT 10) js1 ANY LEFT JOIN t2 ON js1.k == t2.k OR js1.s == t2.k ORDER BY k; -- { serverError 264 } DROP TABLE t2; diff --git a/tests/queries/0_stateless/00370_duplicate_columns_in_subqueries.reference b/tests/queries/0_stateless/00370_duplicate_columns_in_subqueries.reference index 301032e5f43..cf25397873f 100644 --- a/tests/queries/0_stateless/00370_duplicate_columns_in_subqueries.reference +++ b/tests/queries/0_stateless/00370_duplicate_columns_in_subqueries.reference @@ -1,8 +1,8 @@ 1 2 1 1 1 -1 2 -1 1 1 +1 2 1 2 +1 1 1 1 1 1 1 1 1 1 1 diff --git a/tests/queries/0_stateless/00370_duplicate_columns_in_subqueries.sql b/tests/queries/0_stateless/00370_duplicate_columns_in_subqueries.sql index efbe20c8446..118e50c35e0 100644 --- a/tests/queries/0_stateless/00370_duplicate_columns_in_subqueries.sql +++ b/tests/queries/0_stateless/00370_duplicate_columns_in_subqueries.sql @@ -1,5 +1,6 @@ SET any_join_distinct_right_table_keys = 1; SET joined_subquery_requires_alias = 0; +SET allow_experimental_analyzer = 1; select x, y from (select 1 as x, 2 as y, x, y); select x, y from (select 1 as x, 1 as y, x, y); diff --git a/tests/queries/0_stateless/00561_storage_join.sql b/tests/queries/0_stateless/00561_storage_join.sql index 9927592465a..ebdbe4dbc0a 100644 --- a/tests/queries/0_stateless/00561_storage_join.sql +++ b/tests/queries/0_stateless/00561_storage_join.sql @@ -1,3 +1,5 @@ +SET allow_experimental_analyzer = 1; + drop table IF EXISTS joinbug; set allow_deprecated_syntax_for_merge_tree=1; @@ -36,7 +38,7 @@ SEMI LEFT JOIN joinbug_join using id2; SELECT * FROM ( SELECT toUInt32(11) AS id2 ) AS js1 SEMI LEFT JOIN joinbug_join USING (id2); -- can't convert right side in case on storage join -SELECT * FROM ( SELECT toInt64(11) AS id2 ) AS js1 SEMI LEFT JOIN joinbug_join USING (id2); -- { serverError 53 } +SELECT * FROM ( SELECT toInt64(11) AS id2 ) AS js1 SEMI LEFT JOIN joinbug_join USING (id2); -- { serverError 386 } DROP TABLE joinbug; DROP TABLE joinbug_join; From be9406ec0ea6f126241f6c33fcf7c705156a3631 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Fri, 10 Feb 2023 14:29:44 +0100 Subject: [PATCH 313/717] Fixed tests --- src/Analyzer/ConstantNode.cpp | 23 +++++-------- src/Analyzer/Passes/QueryAnalysisPass.cpp | 10 ++++-- src/Analyzer/UnionNode.cpp | 10 ++++++ src/Planner/ActionsChain.h | 4 ++- src/Planner/PlannerActionsVisitor.cpp | 32 +++++++++---------- .../02337_analyzer_columns_basic.sql | 2 +- 6 files changed, 45 insertions(+), 36 deletions(-) diff --git a/src/Analyzer/ConstantNode.cpp b/src/Analyzer/ConstantNode.cpp index 32c97a1b295..1137d40fb56 100644 --- a/src/Analyzer/ConstantNode.cpp +++ b/src/Analyzer/ConstantNode.cpp @@ -86,26 +86,19 @@ ASTPtr ConstantNode::toASTImpl() const switch (constant_value_literal_type) { - case Field::Types::UInt64: - { - need_to_add_cast_function = !constant_value_type.isUInt64(); - break; - } - case Field::Types::Int64: - { - need_to_add_cast_function = !constant_value_type.isInt64(); - break; - } - case Field::Types::Float64: - { - need_to_add_cast_function = !constant_value_type.isFloat64(); - break; - } case Field::Types::String: { need_to_add_cast_function = !constant_value_type.isString(); break; } + case Field::Types::UInt64: + case Field::Types::Int64: + case Field::Types::Float64: + { + WhichDataType constant_value_field_type(applyVisitor(FieldToDataType(), constant_value_literal)); + need_to_add_cast_function = constant_value_field_type.idx != constant_value_type.idx; + break; + } case Field::Types::Int128: case Field::Types::UInt128: case Field::Types::Int256: diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index fb50c1f9d66..488728c88f9 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -110,6 +110,7 @@ namespace ErrorCodes extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int ILLEGAL_PREWHERE; + extern const int UNKNOWN_TABLE; } /** Query analyzer implementation overview. Please check documentation in QueryAnalysisPass.h before. @@ -2063,7 +2064,12 @@ void QueryAnalyzer::replaceNodesWithPositionalArguments(QueryTreeNodePtr & node_ for (auto & node : node_list_typed.getNodes()) { - auto * constant_node = node->as(); + auto * node_to_replace = &node; + + if (auto * sort_node = node->as()) + node_to_replace = &sort_node->getExpression(); + + auto * constant_node = (*node_to_replace)->as(); if (!constant_node) continue; @@ -2088,7 +2094,7 @@ void QueryAnalyzer::replaceNodesWithPositionalArguments(QueryTreeNodePtr & node_ scope.scope_node->formatASTForErrorMessage()); --positional_argument_number; - node = projection_nodes[positional_argument_number]; + *node_to_replace = projection_nodes[positional_argument_number]; } } diff --git a/src/Analyzer/UnionNode.cpp b/src/Analyzer/UnionNode.cpp index 18733b32437..998b869cb04 100644 --- a/src/Analyzer/UnionNode.cpp +++ b/src/Analyzer/UnionNode.cpp @@ -148,6 +148,16 @@ ASTPtr UnionNode::toASTImpl() const select_with_union_query->children.push_back(getQueriesNode()->toAST()); select_with_union_query->list_of_selects = select_with_union_query->children.back(); + if (is_subquery) + { + auto subquery = std::make_shared(); + + subquery->cte_name = cte_name; + subquery->children.push_back(std::move(select_with_union_query)); + + return subquery; + } + return select_with_union_query; } diff --git a/src/Planner/ActionsChain.h b/src/Planner/ActionsChain.h index 7e8e93878e0..4907fdbad87 100644 --- a/src/Planner/ActionsChain.h +++ b/src/Planner/ActionsChain.h @@ -48,7 +48,9 @@ public: * If use_actions_nodes_as_output_columns = true output columns are initialized using actions dag nodes. * If additional output columns are specified they are added to output columns. */ - explicit ActionsChainStep(ActionsDAGPtr actions_, bool use_actions_nodes_as_output_columns = true, ColumnsWithTypeAndName additional_output_columns_ = {}); + explicit ActionsChainStep(ActionsDAGPtr actions_, + bool use_actions_nodes_as_output_columns = true, + ColumnsWithTypeAndName additional_output_columns_ = {}); /// Get actions ActionsDAGPtr & getActions() diff --git a/src/Planner/PlannerActionsVisitor.cpp b/src/Planner/PlannerActionsVisitor.cpp index a9a4d21721e..2c5c8ab5e62 100644 --- a/src/Planner/PlannerActionsVisitor.cpp +++ b/src/Planner/PlannerActionsVisitor.cpp @@ -371,8 +371,6 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::ma actions_stack_node.addInputConstantColumnIfNecessary(set_key, column); } - node_to_node_name.emplace(in_second_argument, set_key); - return {set_key, 0}; } @@ -421,6 +419,21 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi if (isNameOfInFunction(function_node.getFunctionName())) in_function_second_argument_node_name_with_level = makeSetForInFunction(node); + auto function_node_name = calculateActionNodeName(node, *planner_context, node_to_node_name); + + if (function_node.isAggregateFunction() || function_node.isWindowFunction()) + { + size_t actions_stack_size = actions_stack.size(); + + for (size_t i = 0; i < actions_stack_size; ++i) + { + auto & actions_stack_node = actions_stack[i]; + actions_stack_node.addInputColumnIfNecessary(function_node_name, function_node.getResultType()); + } + + return {function_node_name, 0}; + } + const auto & function_arguments = function_node.getArguments().getNodes(); size_t function_arguments_size = function_arguments.size(); @@ -453,21 +466,6 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi level = std::max(level, node_min_level); } - auto function_node_name = calculateActionNodeName(node, *planner_context, node_to_node_name); - - if (function_node.isAggregateFunction() || function_node.isWindowFunction()) - { - size_t actions_stack_size = actions_stack.size(); - - for (size_t i = 0; i < actions_stack_size; ++i) - { - auto & actions_stack_node = actions_stack[i]; - actions_stack_node.addInputColumnIfNecessary(function_node_name, function_node.getResultType()); - } - - return {function_node_name, 0}; - } - ActionsDAG::NodeRawConstPtrs children; children.reserve(function_arguments_size); diff --git a/tests/queries/0_stateless/02337_analyzer_columns_basic.sql b/tests/queries/0_stateless/02337_analyzer_columns_basic.sql index 368a5670d17..76f9f8b25e4 100644 --- a/tests/queries/0_stateless/02337_analyzer_columns_basic.sql +++ b/tests/queries/0_stateless/02337_analyzer_columns_basic.sql @@ -31,7 +31,7 @@ INSERT INTO test_table VALUES (0, 'Value'); SELECT 'Table access without table name qualification'; SELECT test_id FROM test_table; -- { serverError 47 } -SELECT test_id FROM test_unknown_table; -- { serverError 47 } +SELECT test_id FROM test_unknown_table; -- { serverError 60 } DESCRIBE (SELECT id FROM test_table); SELECT id FROM test_table; From 80af0666ea7ba73def55749b33256a1282cb4e78 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sat, 11 Feb 2023 12:10:53 +0100 Subject: [PATCH 314/717] Analyzer support Materialized View --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 16 +--- src/Analyzer/Utils.cpp | 23 +++--- src/Analyzer/Utils.h | 2 +- .../InterpreterSelectQueryAnalyzer.cpp | 76 +++++++++++++++++-- .../InterpreterSelectQueryAnalyzer.h | 9 +++ src/Planner/PlannerJoinTree.cpp | 9 ++- .../QueryPlan/ReadFromMergeTree.cpp | 2 +- .../Transforms/buildPushingToViewsChain.cpp | 2 +- ...02560_analyzer_materialized_view.reference | 9 +++ .../02560_analyzer_materialized_view.sql | 41 ++++++++++ 10 files changed, 153 insertions(+), 36 deletions(-) create mode 100644 tests/queries/0_stateless/02560_analyzer_materialized_view.reference create mode 100644 tests/queries/0_stateless/02560_analyzer_materialized_view.sql diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 488728c88f9..d1490d04b97 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -2070,22 +2070,10 @@ void QueryAnalyzer::replaceNodesWithPositionalArguments(QueryTreeNodePtr & node_ node_to_replace = &sort_node->getExpression(); auto * constant_node = (*node_to_replace)->as(); - if (!constant_node) + if (!constant_node || constant_node->getValue().getType() != Field::Types::UInt64) continue; - if (!isNativeNumber(removeNullable(constant_node->getResultType()))) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "Positional argument must be constant with numeric type. Actual {}. In scope {}", - constant_node->formatASTForErrorMessage(), - scope.scope_node->formatASTForErrorMessage()); - - Field converted = convertFieldToType(constant_node->getValue(), DataTypeUInt64()); - if (converted.isNull()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Positional argument numeric constant expression is not representable as UInt64. In scope {}", - scope.scope_node->formatASTForErrorMessage()); - - UInt64 positional_argument_number = converted.safeGet(); + UInt64 positional_argument_number = constant_node->getValue().get(); if (positional_argument_number == 0 || positional_argument_number > projection_nodes.size()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Positional argument number {} is out of bounds. Expected in range [1, {}]. In scope {}", diff --git a/src/Analyzer/Utils.cpp b/src/Analyzer/Utils.cpp index 8433bc5a90c..8bba79a0400 100644 --- a/src/Analyzer/Utils.cpp +++ b/src/Analyzer/Utils.cpp @@ -246,22 +246,25 @@ QueryTreeNodes extractTableExpressions(const QueryTreeNodePtr & join_tree_node) return result; } -QueryTreeNodePtr extractLeftTableExpression(const QueryTreeNodePtr & join_tree_node) +QueryTreeNodePtr & extractLeftTableExpression(QueryTreeNodePtr & join_tree_node) { - QueryTreeNodePtr result; + QueryTreeNodePtr * result = nullptr; - std::deque nodes_to_process; - nodes_to_process.push_back(join_tree_node); + std::deque nodes_to_process; + nodes_to_process.push_back(&join_tree_node); while (!nodes_to_process.empty()) { - auto node_to_process = std::move(nodes_to_process.front()); + auto * node_to_process_ptr = nodes_to_process.front(); nodes_to_process.pop_front(); + const auto & node_to_process = *node_to_process_ptr; auto node_type = node_to_process->getNodeType(); switch (node_type) { + case QueryTreeNodeType::IDENTIFIER: + [[fallthrough]]; case QueryTreeNodeType::TABLE: [[fallthrough]]; case QueryTreeNodeType::QUERY: @@ -270,32 +273,32 @@ QueryTreeNodePtr extractLeftTableExpression(const QueryTreeNodePtr & join_tree_n [[fallthrough]]; case QueryTreeNodeType::TABLE_FUNCTION: { - result = std::move(node_to_process); + result = node_to_process_ptr; break; } case QueryTreeNodeType::ARRAY_JOIN: { auto & array_join_node = node_to_process->as(); - nodes_to_process.push_front(array_join_node.getTableExpression()); + nodes_to_process.push_front(&array_join_node.getTableExpression()); break; } case QueryTreeNodeType::JOIN: { auto & join_node = node_to_process->as(); - nodes_to_process.push_front(join_node.getLeftTableExpression()); + nodes_to_process.push_front(&join_node.getLeftTableExpression()); break; } default: { throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected node type for table expression. " - "Expected table, table function, query, union, join or array join. Actual {}", + "Expected identifier, table, table function, query, union, join or array join. Actual {}", node_to_process->getNodeTypeName()); } } } - return result; + return *result; } namespace diff --git a/src/Analyzer/Utils.h b/src/Analyzer/Utils.h index d8e3a35c3f5..3a99d005556 100644 --- a/src/Analyzer/Utils.h +++ b/src/Analyzer/Utils.h @@ -20,7 +20,7 @@ void addTableExpressionOrJoinIntoTablesInSelectQuery(ASTPtr & tables_in_select_q QueryTreeNodes extractTableExpressions(const QueryTreeNodePtr & join_tree_node); /// Extract left table expression from join tree -QueryTreeNodePtr extractLeftTableExpression(const QueryTreeNodePtr & join_tree_node); +QueryTreeNodePtr & extractLeftTableExpression(QueryTreeNodePtr & join_tree_node); /** Build table expressions stack that consists from table, table function, query, union, join, array join from join tree. * diff --git a/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp b/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp index e21bf981181..288cec006ed 100644 --- a/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp +++ b/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp @@ -7,14 +7,20 @@ #include -#include -#include - #include #include #include #include +#include + +#include +#include +#include +#include +#include +#include + #include #include @@ -63,10 +69,57 @@ ContextMutablePtr buildContext(const ContextPtr & context, const SelectQueryOpti return result_context; } -QueryTreeNodePtr buildQueryTreeAndRunPasses(const ASTPtr & query, const SelectQueryOptions & select_query_options, const ContextPtr & context) +void replaceStorageInQueryTree(const QueryTreeNodePtr & query_tree, const ContextPtr & context, const StoragePtr & storage) +{ + auto query_to_replace_table_expression = query_tree; + + while (true) + { + if (auto * union_node = query_to_replace_table_expression->as()) + query_to_replace_table_expression = union_node->getQueries().getNodes().at(0); + + auto & query_to_replace_table_expression_typed = query_to_replace_table_expression->as(); + auto & left_table_expression = extractLeftTableExpression(query_to_replace_table_expression_typed.getJoinTree()); + auto left_table_expression_node_type = left_table_expression->getNodeType(); + + switch (left_table_expression_node_type) + { + case QueryTreeNodeType::QUERY: + case QueryTreeNodeType::UNION: + { + query_to_replace_table_expression = left_table_expression; + continue; + } + case QueryTreeNodeType::TABLE: + case QueryTreeNodeType::IDENTIFIER: + { + auto storage_lock = storage->lockForShare(context->getInitialQueryId(), context->getSettingsRef().lock_acquire_timeout); + auto storage_snapshot = storage->getStorageSnapshot(storage->getInMemoryMetadataPtr(), context); + left_table_expression = std::make_shared(storage, + std::move(storage_lock), + std::move(storage_snapshot)); + return; + } + default: + { + throw Exception(ErrorCodes::UNSUPPORTED_METHOD, + "Expected table or identifier node to replace with storage. Actual {}", + left_table_expression->formatASTForErrorMessage()); + } + } + } +} + +QueryTreeNodePtr buildQueryTreeAndRunPasses(const ASTPtr & query, + const SelectQueryOptions & select_query_options, + const ContextPtr & context, + const StoragePtr & storage) { auto query_tree = buildQueryTree(query, context); + if (storage) + replaceStorageInQueryTree(query_tree, context, storage); + QueryTreePassManager query_tree_pass_manager(context); addQueryTreePasses(query_tree_pass_manager); @@ -92,7 +145,20 @@ InterpreterSelectQueryAnalyzer::InterpreterSelectQueryAnalyzer( : query(normalizeAndValidateQuery(query_)) , context(buildContext(context_, select_query_options_)) , select_query_options(select_query_options_) - , query_tree(buildQueryTreeAndRunPasses(query, select_query_options, context)) + , query_tree(buildQueryTreeAndRunPasses(query, select_query_options, context, nullptr /*storage*/)) + , planner(query_tree, select_query_options, buildPlannerConfiguration(select_query_options)) +{ +} + +InterpreterSelectQueryAnalyzer::InterpreterSelectQueryAnalyzer( + const ASTPtr & query_, + const ContextPtr & context_, + const SelectQueryOptions & select_query_options_, + const StoragePtr & storage) + : query(normalizeAndValidateQuery(query_)) + , context(buildContext(context_, select_query_options_)) + , select_query_options(select_query_options_) + , query_tree(buildQueryTreeAndRunPasses(query, select_query_options, context, storage)) , planner(query_tree, select_query_options, buildPlannerConfiguration(select_query_options)) { } diff --git a/src/Interpreters/InterpreterSelectQueryAnalyzer.h b/src/Interpreters/InterpreterSelectQueryAnalyzer.h index bd6a96f597b..c6d84dbc6e1 100644 --- a/src/Interpreters/InterpreterSelectQueryAnalyzer.h +++ b/src/Interpreters/InterpreterSelectQueryAnalyzer.h @@ -20,6 +20,15 @@ public: const ContextPtr & context_, const SelectQueryOptions & select_query_options_); + /** Initialize interpreter with query AST and storage. + * After query tree is built left most table expression is replaced with table node that + * is initialized with provided storage. + */ + InterpreterSelectQueryAnalyzer(const ASTPtr & query_, + const ContextPtr & context_, + const SelectQueryOptions & select_query_options_, + const StoragePtr & storage); + /// Initialize interpreter with query tree InterpreterSelectQueryAnalyzer(const QueryTreeNodePtr & query_tree_, const ContextPtr & context_, diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index a34fced26ad..0a6d38a15be 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -934,8 +934,9 @@ JoinTreeQueryPlan buildJoinTreeQueryPlan(const QueryTreeNodePtr & query_node, for (size_t i = 0; i < table_expressions_stack_size; ++i) { const auto & table_expression = table_expressions_stack[i]; + auto table_expression_node_type = table_expression->getNodeType(); - if (auto * array_join_node = table_expression->as()) + if (table_expression_node_type == QueryTreeNodeType::ARRAY_JOIN) { if (query_plans_stack.empty()) throw Exception(ErrorCodes::LOGICAL_ERROR, @@ -948,7 +949,7 @@ JoinTreeQueryPlan buildJoinTreeQueryPlan(const QueryTreeNodePtr & query_node, table_expressions_outer_scope_columns[i], planner_context); } - else if (auto * join_node = table_expression->as()) + else if (table_expression_node_type == QueryTreeNodeType::JOIN) { if (query_plans_stack.size() < 2) throw Exception(ErrorCodes::LOGICAL_ERROR, @@ -970,9 +971,9 @@ JoinTreeQueryPlan buildJoinTreeQueryPlan(const QueryTreeNodePtr & query_node, else { const auto & table_expression_data = planner_context->getTableExpressionDataOrThrow(table_expression); - if (table_expression_data.isRemote() && has_remote_table) + if (table_expression_data.isRemote() && (has_remote_table || i != 0)) throw Exception(ErrorCodes::UNSUPPORTED_METHOD, - "JOIN with multiple remote storages is unsuppored"); + "JOIN with multiple remote storages is unsupported"); has_remote_table = table_expression_data.isRemote(); diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 9205efd3525..46422bd32cc 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -1101,7 +1101,7 @@ MergeTreeDataSelectAnalysisResultPtr ReadFromMergeTree::selectRangesToRead( std::unordered_map node_name_to_input_node_column; - if (settings.allow_experimental_analyzer) + if (settings.allow_experimental_analyzer && query_info.planner_context) { const auto & table_expression_data = query_info.planner_context->getTableExpressionDataOrThrow(query_info.table_expression); for (const auto & [column_identifier, column_name] : table_expression_data.getColumnIdentifierToColumnName()) diff --git a/src/Processors/Transforms/buildPushingToViewsChain.cpp b/src/Processors/Transforms/buildPushingToViewsChain.cpp index 0dcafd275eb..c57eb90c46b 100644 --- a/src/Processors/Transforms/buildPushingToViewsChain.cpp +++ b/src/Processors/Transforms/buildPushingToViewsChain.cpp @@ -486,7 +486,7 @@ static QueryPipeline process(Block block, ViewRuntimeData & view, const ViewsDat if (local_context->getSettingsRef().allow_experimental_analyzer) { - InterpreterSelectQueryAnalyzer interpreter(view.query, local_context, SelectQueryOptions()); + InterpreterSelectQueryAnalyzer interpreter(view.query, local_context, SelectQueryOptions(), local_context->getViewSource()); pipeline = interpreter.buildQueryPipeline(); } else diff --git a/tests/queries/0_stateless/02560_analyzer_materialized_view.reference b/tests/queries/0_stateless/02560_analyzer_materialized_view.reference new file mode 100644 index 00000000000..52a3e82d671 --- /dev/null +++ b/tests/queries/0_stateless/02560_analyzer_materialized_view.reference @@ -0,0 +1,9 @@ +0 Value_0 +-- +0 Value_0 +1 Value_1 +-- +0 Value_0 +1 Value_1 +2 Value_2 +3 Value_3 diff --git a/tests/queries/0_stateless/02560_analyzer_materialized_view.sql b/tests/queries/0_stateless/02560_analyzer_materialized_view.sql new file mode 100644 index 00000000000..1f268fe1e16 --- /dev/null +++ b/tests/queries/0_stateless/02560_analyzer_materialized_view.sql @@ -0,0 +1,41 @@ +SET allow_experimental_analyzer = 1; + +DROP TABLE IF EXISTS test_table; +CREATE TABLE test_table +( + id UInt64, + value String +) ENGINE=MergeTree ORDER BY id; + +DROP VIEW IF EXISTS test_materialized_view; +CREATE MATERIALIZED VIEW test_materialized_view +( + id UInt64, + value String +) ENGINE=MergeTree ORDER BY id AS SELECT id, value FROM test_table; + +INSERT INTO test_table VALUES (0, 'Value_0'); +SELECT id, value FROM test_materialized_view ORDER BY id; + +SELECT '--'; + +INSERT INTO test_table VALUES (1, 'Value_1'); +SELECT id, value FROM test_materialized_view ORDER BY id; + +DROP TABLE IF EXISTS test_table_data; +CREATE TABLE test_table_data +( + id UInt64, + value String +) ENGINE=MergeTree ORDER BY id; + +INSERT INTO test_table_data VALUES (2, 'Value_2'), (3, 'Value_3'); + +SELECT '--'; + +INSERT INTO test_table SELECT id, value FROM test_table_data; +SELECT id, value FROM test_materialized_view ORDER BY id; + +DROP TABLE test_table_data; +DROP VIEW test_materialized_view; +DROP TABLE test_table; From 469ad2c51bf4900969e5c420e266f45d6ed2da42 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sat, 11 Feb 2023 19:51:17 +0100 Subject: [PATCH 315/717] Fixed build --- src/Analyzer/ConstantNode.cpp | 1 + src/Analyzer/Passes/QueryAnalysisPass.cpp | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Analyzer/ConstantNode.cpp b/src/Analyzer/ConstantNode.cpp index 1137d40fb56..1c55039abbc 100644 --- a/src/Analyzer/ConstantNode.cpp +++ b/src/Analyzer/ConstantNode.cpp @@ -117,6 +117,7 @@ ASTPtr ConstantNode::toASTImpl() const case Field::Types::IPv4: case Field::Types::IPv6: case Field::Types::Null: + case Field::Types::CustomType: { need_to_add_cast_function = true; break; diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index d1490d04b97..f9119c2c472 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -107,7 +107,6 @@ namespace ErrorCodes extern const int NO_COMMON_TYPE; extern const int NOT_IMPLEMENTED; extern const int ALIAS_REQUIRED; - extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int ILLEGAL_PREWHERE; extern const int UNKNOWN_TABLE; From b1ab2af7adb7b117071d52c271a7559c6f366f94 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 14 Feb 2023 12:20:01 +0100 Subject: [PATCH 316/717] Analyzer support storage Merge --- src/Analyzer/IQueryTreeNode.cpp | 8 + src/Analyzer/IQueryTreeNode.h | 5 + src/Analyzer/Utils.cpp | 76 +++- src/Analyzer/Utils.h | 11 +- .../InterpreterSelectQueryAnalyzer.cpp | 37 +- .../InterpreterSelectQueryAnalyzer.h | 2 +- .../getHeaderForProcessingStage.cpp | 84 ++++- .../getHeaderForProcessingStage.h | 10 +- src/Storages/LiveView/StorageLiveView.cpp | 17 +- src/Storages/StorageDistributed.cpp | 5 +- src/Storages/StorageMerge.cpp | 328 ++++++++++-------- src/Storages/StorageMerge.h | 5 + .../02563_analyzer_merge.reference | 2 + .../0_stateless/02563_analyzer_merge.sql | 38 ++ 14 files changed, 419 insertions(+), 209 deletions(-) create mode 100644 tests/queries/0_stateless/02563_analyzer_merge.reference create mode 100644 tests/queries/0_stateless/02563_analyzer_merge.sql diff --git a/src/Analyzer/IQueryTreeNode.cpp b/src/Analyzer/IQueryTreeNode.cpp index 640c93be415..a8759318fda 100644 --- a/src/Analyzer/IQueryTreeNode.cpp +++ b/src/Analyzer/IQueryTreeNode.cpp @@ -306,6 +306,14 @@ QueryTreeNodePtr IQueryTreeNode::cloneAndReplace(const ReplacementMap & replacem return result_cloned_node_place; } +QueryTreeNodePtr IQueryTreeNode::cloneAndReplace(const QueryTreeNodePtr & node_to_replace, QueryTreeNodePtr replacement_node) const +{ + ReplacementMap replacement_map; + replacement_map.emplace(node_to_replace.get(), std::move(replacement_node)); + + return cloneAndReplace(replacement_map); +} + ASTPtr IQueryTreeNode::toAST() const { auto converted_node = toASTImpl(); diff --git a/src/Analyzer/IQueryTreeNode.h b/src/Analyzer/IQueryTreeNode.h index 0f0614aa39a..748a947274e 100644 --- a/src/Analyzer/IQueryTreeNode.h +++ b/src/Analyzer/IQueryTreeNode.h @@ -122,6 +122,11 @@ public: using ReplacementMap = std::unordered_map; QueryTreeNodePtr cloneAndReplace(const ReplacementMap & replacement_map) const; + /** Get a deep copy of the query tree. + * If node to clone is node to replace, then instead of clone it use replacement node. + */ + QueryTreeNodePtr cloneAndReplace(const QueryTreeNodePtr & node_to_replace, QueryTreeNodePtr replacement_node) const; + /// Returns true if node has alias, false otherwise bool hasAlias() const { diff --git a/src/Analyzer/Utils.cpp b/src/Analyzer/Utils.cpp index 8bba79a0400..5356ba308e4 100644 --- a/src/Analyzer/Utils.cpp +++ b/src/Analyzer/Utils.cpp @@ -246,25 +246,22 @@ QueryTreeNodes extractTableExpressions(const QueryTreeNodePtr & join_tree_node) return result; } -QueryTreeNodePtr & extractLeftTableExpression(QueryTreeNodePtr & join_tree_node) +QueryTreeNodePtr extractLeftTableExpression(const QueryTreeNodePtr & join_tree_node) { - QueryTreeNodePtr * result = nullptr; + QueryTreeNodePtr result; - std::deque nodes_to_process; - nodes_to_process.push_back(&join_tree_node); + std::deque nodes_to_process; + nodes_to_process.push_back(join_tree_node); while (!nodes_to_process.empty()) { - auto * node_to_process_ptr = nodes_to_process.front(); + auto node_to_process = std::move(nodes_to_process.front()); nodes_to_process.pop_front(); - const auto & node_to_process = *node_to_process_ptr; auto node_type = node_to_process->getNodeType(); switch (node_type) { - case QueryTreeNodeType::IDENTIFIER: - [[fallthrough]]; case QueryTreeNodeType::TABLE: [[fallthrough]]; case QueryTreeNodeType::QUERY: @@ -273,32 +270,32 @@ QueryTreeNodePtr & extractLeftTableExpression(QueryTreeNodePtr & join_tree_node) [[fallthrough]]; case QueryTreeNodeType::TABLE_FUNCTION: { - result = node_to_process_ptr; + result = std::move(node_to_process); break; } case QueryTreeNodeType::ARRAY_JOIN: { auto & array_join_node = node_to_process->as(); - nodes_to_process.push_front(&array_join_node.getTableExpression()); + nodes_to_process.push_front(array_join_node.getTableExpression()); break; } case QueryTreeNodeType::JOIN: { auto & join_node = node_to_process->as(); - nodes_to_process.push_front(&join_node.getLeftTableExpression()); + nodes_to_process.push_front(join_node.getLeftTableExpression()); break; } default: { throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected node type for table expression. " - "Expected identifier, table, table function, query, union, join or array join. Actual {}", + "Expected table, table function, query, union, join or array join. Actual {}", node_to_process->getNodeTypeName()); } } } - return *result; + return result; } namespace @@ -485,4 +482,57 @@ bool hasFunctionNode(const QueryTreeNodePtr & node, std::string_view function_na return visitor.hasFunction(); } +namespace +{ + +class ReplaceColumnsVisitor : public InDepthQueryTreeVisitor +{ +public: + explicit ReplaceColumnsVisitor(const QueryTreeNodePtr & table_expression_node_, + const std::unordered_map & column_name_to_node_) + : table_expression_node(table_expression_node_) + , column_name_to_node(column_name_to_node_) + {} + + void visitImpl(QueryTreeNodePtr & node) + { + auto * column_node = node->as(); + if (!column_node) + return; + + auto column_source = column_node->getColumnSourceOrNull(); + if (column_source != table_expression_node) + return; + + auto node_it = column_name_to_node.find(column_node->getColumnName()); + if (node_it == column_name_to_node.end()) + return; + + node = node_it->second; + } + + bool needChildVisit(const QueryTreeNodePtr & parent_node, const QueryTreeNodePtr & child_node) + { + if (parent_node->getNodeType() == QueryTreeNodeType::CONSTANT) + return false; + + auto child_node_type = child_node->getNodeType(); + return !(child_node_type == QueryTreeNodeType::QUERY || child_node_type == QueryTreeNodeType::UNION); + } + +private: + QueryTreeNodePtr table_expression_node; + const std::unordered_map & column_name_to_node; +}; + +} + +void replaceColumns(QueryTreeNodePtr & node, + const QueryTreeNodePtr & table_expression_node, + const std::unordered_map & column_name_to_node) +{ + ReplaceColumnsVisitor visitor(table_expression_node, column_name_to_node); + visitor.visit(node); +} + } diff --git a/src/Analyzer/Utils.h b/src/Analyzer/Utils.h index 3a99d005556..0f54b5cadd5 100644 --- a/src/Analyzer/Utils.h +++ b/src/Analyzer/Utils.h @@ -20,7 +20,7 @@ void addTableExpressionOrJoinIntoTablesInSelectQuery(ASTPtr & tables_in_select_q QueryTreeNodes extractTableExpressions(const QueryTreeNodePtr & join_tree_node); /// Extract left table expression from join tree -QueryTreeNodePtr & extractLeftTableExpression(QueryTreeNodePtr & join_tree_node); +QueryTreeNodePtr extractLeftTableExpression(const QueryTreeNodePtr & join_tree_node); /** Build table expressions stack that consists from table, table function, query, union, join, array join from join tree. * @@ -56,4 +56,13 @@ void assertNoFunctionNodes(const QueryTreeNodePtr & node, */ bool hasFunctionNode(const QueryTreeNodePtr & node, std::string_view function_name); +/** Replace columns in node children. + * If there is column node and its source is specified table expression node and there is + * node for column name in map replace column node with node from map. + * Do not visit subqueries. + */ +void replaceColumns(QueryTreeNodePtr & node, + const QueryTreeNodePtr & table_expression_node, + const std::unordered_map & column_name_to_node); + } diff --git a/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp b/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp index 288cec006ed..0f5ccb3ae06 100644 --- a/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp +++ b/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp @@ -69,17 +69,18 @@ ContextMutablePtr buildContext(const ContextPtr & context, const SelectQueryOpti return result_context; } -void replaceStorageInQueryTree(const QueryTreeNodePtr & query_tree, const ContextPtr & context, const StoragePtr & storage) +void replaceStorageInQueryTree(QueryTreeNodePtr & query_tree, const ContextPtr & context, const StoragePtr & storage) { auto query_to_replace_table_expression = query_tree; + QueryTreeNodePtr table_expression_to_replace; - while (true) + while (!table_expression_to_replace) { if (auto * union_node = query_to_replace_table_expression->as()) query_to_replace_table_expression = union_node->getQueries().getNodes().at(0); auto & query_to_replace_table_expression_typed = query_to_replace_table_expression->as(); - auto & left_table_expression = extractLeftTableExpression(query_to_replace_table_expression_typed.getJoinTree()); + auto left_table_expression = extractLeftTableExpression(query_to_replace_table_expression_typed.getJoinTree()); auto left_table_expression_node_type = left_table_expression->getNodeType(); switch (left_table_expression_node_type) @@ -87,27 +88,27 @@ void replaceStorageInQueryTree(const QueryTreeNodePtr & query_tree, const Contex case QueryTreeNodeType::QUERY: case QueryTreeNodeType::UNION: { - query_to_replace_table_expression = left_table_expression; - continue; + query_to_replace_table_expression = std::move(left_table_expression); + break; } case QueryTreeNodeType::TABLE: + case QueryTreeNodeType::TABLE_FUNCTION: case QueryTreeNodeType::IDENTIFIER: { - auto storage_lock = storage->lockForShare(context->getInitialQueryId(), context->getSettingsRef().lock_acquire_timeout); - auto storage_snapshot = storage->getStorageSnapshot(storage->getInMemoryMetadataPtr(), context); - left_table_expression = std::make_shared(storage, - std::move(storage_lock), - std::move(storage_snapshot)); - return; + table_expression_to_replace = std::move(left_table_expression); + break; } default: { throw Exception(ErrorCodes::UNSUPPORTED_METHOD, - "Expected table or identifier node to replace with storage. Actual {}", + "Expected table, table function or identifier node to replace with storage. Actual {}", left_table_expression->formatASTForErrorMessage()); } } } + + auto replacement_table_expression = std::make_shared(storage, context); + query_tree = query_tree->cloneAndReplace(table_expression_to_replace, std::move(replacement_table_expression)); } QueryTreeNodePtr buildQueryTreeAndRunPasses(const ASTPtr & query, @@ -117,9 +118,6 @@ QueryTreeNodePtr buildQueryTreeAndRunPasses(const ASTPtr & query, { auto query_tree = buildQueryTree(query, context); - if (storage) - replaceStorageInQueryTree(query_tree, context, storage); - QueryTreePassManager query_tree_pass_manager(context); addQueryTreePasses(query_tree_pass_manager); @@ -128,6 +126,9 @@ QueryTreeNodePtr buildQueryTreeAndRunPasses(const ASTPtr & query, else query_tree_pass_manager.run(query_tree); + if (storage) + replaceStorageInQueryTree(query_tree, context, storage); + return query_tree; } @@ -154,11 +155,11 @@ InterpreterSelectQueryAnalyzer::InterpreterSelectQueryAnalyzer( const ASTPtr & query_, const ContextPtr & context_, const SelectQueryOptions & select_query_options_, - const StoragePtr & storage) + const StoragePtr & storage_) : query(normalizeAndValidateQuery(query_)) , context(buildContext(context_, select_query_options_)) , select_query_options(select_query_options_) - , query_tree(buildQueryTreeAndRunPasses(query, select_query_options, context, storage)) + , query_tree(buildQueryTreeAndRunPasses(query, select_query_options, context, storage_)) , planner(query_tree, select_query_options, buildPlannerConfiguration(select_query_options)) { } @@ -210,7 +211,7 @@ BlockIO InterpreterSelectQueryAnalyzer::execute() BlockIO result; result.pipeline = QueryPipelineBuilder::getPipeline(std::move(pipeline_builder)); - if (!select_query_options.ignore_quota && (select_query_options.to_stage == QueryProcessingStage::Complete)) + if (!select_query_options.ignore_quota && select_query_options.to_stage == QueryProcessingStage::Complete) result.pipeline.setQuota(context->getQuota()); return result; diff --git a/src/Interpreters/InterpreterSelectQueryAnalyzer.h b/src/Interpreters/InterpreterSelectQueryAnalyzer.h index c6d84dbc6e1..f5151505d68 100644 --- a/src/Interpreters/InterpreterSelectQueryAnalyzer.h +++ b/src/Interpreters/InterpreterSelectQueryAnalyzer.h @@ -27,7 +27,7 @@ public: InterpreterSelectQueryAnalyzer(const ASTPtr & query_, const ContextPtr & context_, const SelectQueryOptions & select_query_options_, - const StoragePtr & storage); + const StoragePtr & storage_); /// Initialize interpreter with query tree InterpreterSelectQueryAnalyzer(const QueryTreeNodePtr & query_tree_, diff --git a/src/Interpreters/getHeaderForProcessingStage.cpp b/src/Interpreters/getHeaderForProcessingStage.cpp index d40cbd9366e..d725a06ca16 100644 --- a/src/Interpreters/getHeaderForProcessingStage.cpp +++ b/src/Interpreters/getHeaderForProcessingStage.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -82,12 +83,50 @@ bool removeJoin(ASTSelectQuery & select, TreeRewriterResult & rewriter_result, C return true; } +class StorageDummy : public IStorage +{ +public: + StorageDummy(const StorageID & table_id_, const ColumnsDescription & columns_) + : IStorage(table_id_) + { + StorageInMemoryMetadata storage_metadata; + storage_metadata.setColumns(columns_); + setInMemoryMetadata(storage_metadata); + } + + std::string getName() const override { return "StorageDummy"; } + + bool supportsSampling() const override { return true; } + bool supportsFinal() const override { return true; } + bool supportsPrewhere() const override { return true; } + bool supportsSubcolumns() const override { return true; } + bool supportsDynamicSubcolumns() const override { return true; } + bool canMoveConditionsToPrewhere() const override { return false; } + + QueryProcessingStage::Enum + getQueryProcessingStage(ContextPtr, QueryProcessingStage::Enum, const StorageSnapshotPtr &, SelectQueryInfo &) const override + { + throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "StorageDummy does not support getQueryProcessingStage method"); + } + + Pipe read(const Names & /*column_names*/, + const StorageSnapshotPtr & /*storage_snapshot*/, + SelectQueryInfo & /*query_info*/, + ContextPtr /*context*/, + QueryProcessingStage::Enum /*processed_stage*/, + size_t /*max_block_size*/, + size_t /*num_streams*/) override + { + throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "StorageDummy does not support read method"); + } +}; + Block getHeaderForProcessingStage( - const Names & column_names, - const StorageSnapshotPtr & storage_snapshot, - const SelectQueryInfo & query_info, - ContextPtr context, - QueryProcessingStage::Enum processed_stage) + const Names & column_names, + const StorageSnapshotPtr & storage_snapshot, + const SelectQueryInfo & query_info, + ContextPtr context, + QueryProcessingStage::Enum processed_stage) { switch (processed_stage) { @@ -119,17 +158,34 @@ Block getHeaderForProcessingStage( case QueryProcessingStage::WithMergeableStateAfterAggregationAndLimit: case QueryProcessingStage::MAX: { - /// TODO: Analyzer syntax analyzer result - if (!query_info.syntax_analyzer_result) - throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "getHeaderForProcessingStage is unsupported"); + ASTPtr query = query_info.query; + if (const auto * select = query_info.query->as(); select && hasJoin(*select)) + { + /// TODO: Analyzer syntax analyzer result + if (!query_info.syntax_analyzer_result) + throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "getHeaderForProcessingStage is unsupported"); - auto query = query_info.query->clone(); - TreeRewriterResult new_rewriter_result = *query_info.syntax_analyzer_result; - removeJoin(*query->as(), new_rewriter_result, context); + query = query_info.query->clone(); + TreeRewriterResult new_rewriter_result = *query_info.syntax_analyzer_result; + removeJoin(*query->as(), new_rewriter_result, context); + } - auto pipe = Pipe(std::make_shared( - storage_snapshot->getSampleBlockForColumns(column_names))); - return InterpreterSelectQuery(query, context, std::move(pipe), SelectQueryOptions(processed_stage).analyze()).getSampleBlock(); + Block result; + + if (context->getSettingsRef().allow_experimental_analyzer) + { + auto storage = std::make_shared(storage_snapshot->storage.getStorageID(), storage_snapshot->metadata->getColumns()); + InterpreterSelectQueryAnalyzer interpreter(query, context, SelectQueryOptions(processed_stage).analyze(), storage); + result = interpreter.getSampleBlock(); + } + else + { + auto pipe = Pipe(std::make_shared( + storage_snapshot->getSampleBlockForColumns(column_names))); + result = InterpreterSelectQuery(query, context, std::move(pipe), SelectQueryOptions(processed_stage).analyze()).getSampleBlock(); + } + + return result; } } throw Exception(ErrorCodes::LOGICAL_ERROR, "Logical Error: unknown processed stage."); diff --git a/src/Interpreters/getHeaderForProcessingStage.h b/src/Interpreters/getHeaderForProcessingStage.h index a578c414e3b..51966d291ce 100644 --- a/src/Interpreters/getHeaderForProcessingStage.h +++ b/src/Interpreters/getHeaderForProcessingStage.h @@ -20,10 +20,10 @@ bool hasJoin(const ASTSelectQuery & select); bool removeJoin(ASTSelectQuery & select, TreeRewriterResult & rewriter_result, ContextPtr context); Block getHeaderForProcessingStage( - const Names & column_names, - const StorageSnapshotPtr & storage_snapshot, - const SelectQueryInfo & query_info, - ContextPtr context, - QueryProcessingStage::Enum processed_stage); + const Names & column_names, + const StorageSnapshotPtr & storage_snapshot, + const SelectQueryInfo & query_info, + ContextPtr context, + QueryProcessingStage::Enum processed_stage); } diff --git a/src/Storages/LiveView/StorageLiveView.cpp b/src/Storages/LiveView/StorageLiveView.cpp index 83aa927d584..0e9b35fe720 100644 --- a/src/Storages/LiveView/StorageLiveView.cpp +++ b/src/Storages/LiveView/StorageLiveView.cpp @@ -199,15 +199,6 @@ SelectQueryTreeDescription buildSelectQueryTreeDescription(const ASTPtr & select return {std::move(select_query_node), std::move(inner_query_node), std::move(dependent_table_node)}; } -QueryTreeNodePtr cloneQueryTreeAndReplaceTableExpression(const QueryTreeNodePtr & query_tree, - const QueryTreeNodePtr table_expression_to_replace, - const QueryTreeNodePtr & replacement_table_expression) -{ - std::unordered_map replacement_map; - replacement_map.emplace(table_expression_to_replace.get(), replacement_table_expression); - return query_tree->cloneAndReplace(replacement_map); -} - } StorageLiveView::StorageLiveView( @@ -430,9 +421,9 @@ void StorageLiveView::writeBlock(const Block & block, ContextPtr local_context) TableLockHolder{}, std::move(storage_snapshot)); - select_description.inner_query_node = cloneQueryTreeAndReplaceTableExpression(select_description.inner_query_node, + select_description.inner_query_node = select_description.inner_query_node->cloneAndReplace( select_description.dependent_table_node, - replacement_table_expression); + std::move(replacement_table_expression)); } InterpreterSelectQueryAnalyzer interpreter(select_description.inner_query_node, @@ -637,9 +628,9 @@ QueryPipelineBuilder StorageLiveView::completeQuery(Pipes pipes) TableLockHolder{}, std::move(storage_snapshot)); - select_description.select_query_node = cloneQueryTreeAndReplaceTableExpression(select_description.select_query_node, + select_description.select_query_node = select_description.select_query_node->cloneAndReplace( select_description.dependent_table_node, - replacement_table_expression); + std::move(replacement_table_expression)); } InterpreterSelectQueryAnalyzer interpreter(select_description.select_query_node, diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 37aeaa650b0..cafb9c1abe8 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -701,10 +701,7 @@ QueryTreeNodePtr buildQueryTreeDistributedTableReplacedWithLocalTable(const Sele replacement_table_expression->setAlias(query_info.table_expression->getAlias()); - std::unordered_map replacement_map; - replacement_map.emplace(query_info.table_expression.get(), std::move(replacement_table_expression)); - - return query_info.query_tree->cloneAndReplace(replacement_map); + return query_info.query_tree->cloneAndReplace(query_info.table_expression, std::move(replacement_table_expression)); } } diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index a30841d2975..3f31dbac311 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -11,10 +11,15 @@ #include #include #include +#include #include #include #include #include +#include +#include +#include +#include #include #include #include @@ -257,33 +262,6 @@ QueryProcessingStage::Enum StorageMerge::getQueryProcessingStage( return selected_table_size == 1 ? stage_in_source_tables : std::min(stage_in_source_tables, QueryProcessingStage::WithMergeableState); } - -SelectQueryInfo getModifiedQueryInfo( - const SelectQueryInfo & query_info, ContextPtr modified_context, const StorageID & current_storage_id, bool is_merge_engine) -{ - SelectQueryInfo modified_query_info = query_info; - modified_query_info.query = query_info.query->clone(); - - /// TODO: Analyzer syntax analyzer result - if (modified_query_info.syntax_analyzer_result) - { - /// Original query could contain JOIN but we need only the first joined table and its columns. - auto & modified_select = modified_query_info.query->as(); - TreeRewriterResult new_analyzer_res = *modified_query_info.syntax_analyzer_result; - removeJoin(modified_select, new_analyzer_res, modified_context); - modified_query_info.syntax_analyzer_result = std::make_shared(std::move(new_analyzer_res)); - } - - if (!is_merge_engine) - { - VirtualColumnUtils::rewriteEntityInAst(modified_query_info.query, "_table", current_storage_id.table_name); - VirtualColumnUtils::rewriteEntityInAst(modified_query_info.query, "_database", current_storage_id.database_name); - } - - return modified_query_info; -} - - void StorageMerge::read( QueryPlan & query_plan, const Names & column_names, @@ -447,59 +425,67 @@ void ReadFromMerge::initializePipeline(QueryPipelineBuilder & pipeline, const Bu const auto & storage = std::get<1>(table); + bool sampling_requested = query_info.query->as()->sampleSize() != nullptr; + if (query_info.table_expression_modifiers) + sampling_requested = query_info.table_expression_modifiers->hasSampleSizeRatio(); + /// If sampling requested, then check that table supports it. - if (query_info.query->as()->sampleSize() && !storage->supportsSampling()) + if (sampling_requested && !storage->supportsSampling()) throw Exception(ErrorCodes::SAMPLING_NOT_SUPPORTED, "Illegal SAMPLE: table doesn't support sampling"); Aliases aliases; auto storage_metadata_snapshot = storage->getInMemoryMetadataPtr(); - auto storage_columns = storage_metadata_snapshot->getColumns(); auto nested_storage_snaphsot = storage->getStorageSnapshot(storage_metadata_snapshot, context); - auto modified_query_info = getModifiedQueryInfo(query_info, context, storage->getStorageID(), storage->as()); - auto syntax_result = TreeRewriter(context).analyzeSelect( - modified_query_info.query, TreeRewriterResult({}, storage, nested_storage_snaphsot)); - + auto modified_query_info = getModifiedQueryInfo(query_info, context, table, nested_storage_snaphsot); Names column_names_as_aliases; - bool with_aliases = common_processed_stage == QueryProcessingStage::FetchColumns && !storage_columns.getAliases().empty(); - if (with_aliases) + + if (!context->getSettingsRef().allow_experimental_analyzer) { - ASTPtr required_columns_expr_list = std::make_shared(); - ASTPtr column_expr; + auto storage_columns = storage_metadata_snapshot->getColumns(); + auto syntax_result = TreeRewriter(context).analyzeSelect( + modified_query_info.query, TreeRewriterResult({}, storage, nested_storage_snaphsot)); - for (const auto & column : column_names) + bool with_aliases = common_processed_stage == QueryProcessingStage::FetchColumns && !storage_columns.getAliases().empty(); + if (with_aliases) { - const auto column_default = storage_columns.getDefault(column); - bool is_alias = column_default && column_default->kind == ColumnDefaultKind::Alias; + ASTPtr required_columns_expr_list = std::make_shared(); + ASTPtr column_expr; - if (is_alias) + for (const auto & column : column_names) { - column_expr = column_default->expression->clone(); - replaceAliasColumnsInQuery(column_expr, storage_metadata_snapshot->getColumns(), - syntax_result->array_join_result_to_source, context); + const auto column_default = storage_columns.getDefault(column); + bool is_alias = column_default && column_default->kind == ColumnDefaultKind::Alias; - auto column_description = storage_columns.get(column); - column_expr = addTypeConversionToAST(std::move(column_expr), column_description.type->getName(), - storage_metadata_snapshot->getColumns().getAll(), context); - column_expr = setAlias(column_expr, column); + if (is_alias) + { + column_expr = column_default->expression->clone(); + replaceAliasColumnsInQuery(column_expr, storage_metadata_snapshot->getColumns(), + syntax_result->array_join_result_to_source, context); - auto type = sample_block.getByName(column).type; - aliases.push_back({ .name = column, .type = type, .expression = column_expr->clone() }); + auto column_description = storage_columns.get(column); + column_expr = addTypeConversionToAST(std::move(column_expr), column_description.type->getName(), + storage_metadata_snapshot->getColumns().getAll(), context); + column_expr = setAlias(column_expr, column); + + auto type = sample_block.getByName(column).type; + aliases.push_back({ .name = column, .type = type, .expression = column_expr->clone() }); + } + else + column_expr = std::make_shared(column); + + required_columns_expr_list->children.emplace_back(std::move(column_expr)); } - else - column_expr = std::make_shared(column); - required_columns_expr_list->children.emplace_back(std::move(column_expr)); + syntax_result = TreeRewriter(context).analyze( + required_columns_expr_list, storage_columns.getAllPhysical(), storage, storage->getStorageSnapshot(storage_metadata_snapshot, context)); + + auto alias_actions = ExpressionAnalyzer(required_columns_expr_list, syntax_result, context).getActionsDAG(true); + + column_names_as_aliases = alias_actions->getRequiredColumns().getNames(); + if (column_names_as_aliases.empty()) + column_names_as_aliases.push_back(ExpressionActions::getSmallestColumn(storage_metadata_snapshot->getColumns().getAllPhysical()).name); } - - syntax_result = TreeRewriter(context).analyze( - required_columns_expr_list, storage_columns.getAllPhysical(), storage, storage->getStorageSnapshot(storage_metadata_snapshot, context)); - - auto alias_actions = ExpressionAnalyzer(required_columns_expr_list, syntax_result, context).getActionsDAG(true); - - column_names_as_aliases = alias_actions->getRequiredColumns().getNames(); - if (column_names_as_aliases.empty()) - column_names_as_aliases.push_back(ExpressionActions::getSmallestColumn(storage_metadata_snapshot->getColumns().getAllPhysical()).name); } auto source_pipeline = createSources( @@ -540,6 +526,57 @@ void ReadFromMerge::initializePipeline(QueryPipelineBuilder & pipeline, const Bu pipeline.addResources(std::move(resources)); } +SelectQueryInfo ReadFromMerge::getModifiedQueryInfo(const SelectQueryInfo & query_info, + const ContextPtr & modified_context, + const StorageWithLockAndName & storage_with_lock_and_name, + const StorageSnapshotPtr & storage_snapshot) +{ + const auto & [database_name, storage, storage_lock, table_name] = storage_with_lock_and_name; + const StorageID current_storage_id = storage->getStorageID(); + bool is_storage_merge_engine = storage->as(); + + SelectQueryInfo modified_query_info = query_info; + + if (modified_query_info.table_expression) + { + auto replacement_table_expression = std::make_shared(storage, storage_lock, storage_snapshot); + modified_query_info.query_tree = modified_query_info.query_tree->cloneAndReplace(modified_query_info.table_expression, + replacement_table_expression); + modified_query_info.table_expression = replacement_table_expression; + + if (!is_storage_merge_engine) + { + std::unordered_map column_name_to_node; + column_name_to_node.emplace("_table", std::make_shared(current_storage_id.table_name)); + column_name_to_node.emplace("_database", std::make_shared(current_storage_id.database_name)); + + replaceColumns(modified_query_info.query_tree, + replacement_table_expression, + column_name_to_node); + } + + modified_query_info.query = queryNodeToSelectQuery(modified_query_info.query_tree); + } + else + { + modified_query_info.query = query_info.query->clone(); + + /// Original query could contain JOIN but we need only the first joined table and its columns. + auto & modified_select = modified_query_info.query->as(); + TreeRewriterResult new_analyzer_res = *modified_query_info.syntax_analyzer_result; + removeJoin(modified_select, new_analyzer_res, modified_context); + modified_query_info.syntax_analyzer_result = std::make_shared(std::move(new_analyzer_res)); + + if (!is_storage_merge_engine) + { + VirtualColumnUtils::rewriteEntityInAst(modified_query_info.query, "_table", current_storage_id.table_name); + VirtualColumnUtils::rewriteEntityInAst(modified_query_info.query, "_database", current_storage_id.database_name); + } + } + + return modified_query_info; +} + QueryPipelineBuilderPtr ReadFromMerge::createSources( const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & modified_query_info, @@ -558,14 +595,6 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( QueryPipelineBuilderPtr builder; - if (!storage) - { - return std::make_unique(InterpreterSelectQuery( - modified_query_info.query, modified_context, - Pipe(std::make_shared(header)), - SelectQueryOptions(processed_stage).analyze()).buildQueryPipeline()); - } - bool final = isFinal(modified_query_info); if (!final && storage->needRewriteQueryWithFinal(real_column_names)) @@ -575,65 +604,12 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( modified_select.setFinal(); } - auto storage_stage - = storage->getQueryProcessingStage(modified_context, QueryProcessingStage::Complete, storage_snapshot, modified_query_info); - if (processed_stage <= storage_stage) + if (modified_context->getSettingsRef().allow_experimental_analyzer) { - /// If there are only virtual columns in query, you must request at least one other column. - if (real_column_names.empty()) - real_column_names.push_back(ExpressionActions::getSmallestColumn(storage_snapshot->metadata->getColumns().getAllPhysical()).name); - - QueryPlan plan; - if (StorageView * view = dynamic_cast(storage.get())) - { - /// For view storage, we need to rewrite the `modified_query_info.view_query` to optimize read. - /// The most intuitive way is to use InterpreterSelectQuery. - - /// Intercept the settings - modified_context->setSetting("max_threads", streams_num); - modified_context->setSetting("max_streams_to_max_threads_ratio", 1); - modified_context->setSetting("max_block_size", max_block_size); - - InterpreterSelectQuery( - modified_query_info.query, modified_context, storage, view->getInMemoryMetadataPtr(), SelectQueryOptions(processed_stage)) - .buildQueryPlan(plan); - } - else - { - storage->read( - plan, - real_column_names, - storage_snapshot, - modified_query_info, - modified_context, - processed_stage, - max_block_size, - UInt32(streams_num)); - - } - - if (!plan.isInitialized()) - return {}; - - if (auto * read_from_merge_tree = typeid_cast(plan.getRootNode()->step.get())) - read_from_merge_tree->addFilterNodes(added_filter_nodes); - - builder = plan.buildQueryPipeline( - QueryPlanOptimizationSettings::fromContext(modified_context), - BuildQueryPipelineSettings::fromContext(modified_context)); - } - else if (processed_stage > storage_stage) - { - modified_select.replaceDatabaseAndTable(database_name, table_name); - - /// Maximum permissible parallelism is streams_num - modified_context->setSetting("max_threads", streams_num); - modified_context->setSetting("max_streams_to_max_threads_ratio", 1); - - /// TODO: Find a way to support projections for StorageMerge - InterpreterSelectQuery interpreter{ - modified_query_info.query, modified_context, SelectQueryOptions(processed_stage).ignoreProjections()}; - + InterpreterSelectQueryAnalyzer interpreter(modified_query_info.query, + modified_context, + SelectQueryOptions(processed_stage).ignoreProjections(), + storage); builder = std::make_unique(interpreter.buildQueryPipeline()); /** Materialization is needed, since from distributed storage the constants come materialized. @@ -642,6 +618,75 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( */ builder->addSimpleTransform([](const Block & stream_header) { return std::make_shared(stream_header); }); } + else + { + auto storage_stage + = storage->getQueryProcessingStage(modified_context, QueryProcessingStage::Complete, storage_snapshot, modified_query_info); + if (processed_stage <= storage_stage) + { + /// If there are only virtual columns in query, you must request at least one other column. + if (real_column_names.empty()) + real_column_names.push_back(ExpressionActions::getSmallestColumn(storage_snapshot->metadata->getColumns().getAllPhysical()).name); + + QueryPlan plan; + if (StorageView * view = dynamic_cast(storage.get())) + { + /// For view storage, we need to rewrite the `modified_query_info.view_query` to optimize read. + /// The most intuitive way is to use InterpreterSelectQuery. + + /// Intercept the settings + modified_context->setSetting("max_threads", streams_num); + modified_context->setSetting("max_streams_to_max_threads_ratio", 1); + modified_context->setSetting("max_block_size", max_block_size); + + InterpreterSelectQuery( + modified_query_info.query, modified_context, storage, view->getInMemoryMetadataPtr(), SelectQueryOptions(processed_stage)) + .buildQueryPlan(plan); + } + else + { + storage->read( + plan, + real_column_names, + storage_snapshot, + modified_query_info, + modified_context, + processed_stage, + max_block_size, + UInt32(streams_num)); + } + + if (!plan.isInitialized()) + return {}; + + if (auto * read_from_merge_tree = typeid_cast(plan.getRootNode()->step.get())) + read_from_merge_tree->addFilterNodes(added_filter_nodes); + + builder = plan.buildQueryPipeline( + QueryPlanOptimizationSettings::fromContext(modified_context), + BuildQueryPipelineSettings::fromContext(modified_context)); + } + else if (processed_stage > storage_stage) + { + modified_select.replaceDatabaseAndTable(database_name, table_name); + + /// Maximum permissible parallelism is streams_num + modified_context->setSetting("max_threads", streams_num); + modified_context->setSetting("max_streams_to_max_threads_ratio", 1); + + /// TODO: Find a way to support projections for StorageMerge + InterpreterSelectQuery interpreter{ + modified_query_info.query, modified_context, SelectQueryOptions(processed_stage).ignoreProjections()}; + + builder = std::make_unique(interpreter.buildQueryPipeline()); + + /** Materialization is needed, since from distributed storage the constants come materialized. + * If you do not do this, different types (Const and non-Const) columns will be produced in different threads, + * And this is not allowed, since all code is based on the assumption that in the block stream all types are the same. + */ + builder->addSimpleTransform([](const Block & stream_header) { return std::make_shared(stream_header); }); + } + } if (builder->initialized()) { @@ -893,19 +938,22 @@ void ReadFromMerge::convertingSourceStream( }); } - { - auto convert_actions_dag = ActionsDAG::makeConvertingActions(builder.getHeader().getColumnsWithTypeAndName(), - header.getColumnsWithTypeAndName(), - ActionsDAG::MatchColumnsMode::Name); - auto actions = std::make_shared( - convert_actions_dag, - ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes)); + ActionsDAG::MatchColumnsMode convert_actions_match_columns_mode = ActionsDAG::MatchColumnsMode::Name; - builder.addSimpleTransform([&](const Block & stream_header) - { - return std::make_shared(stream_header, actions); - }); - } + if (local_context->getSettingsRef().allow_experimental_analyzer) + convert_actions_match_columns_mode = ActionsDAG::MatchColumnsMode::Position; + + auto convert_actions_dag = ActionsDAG::makeConvertingActions(builder.getHeader().getColumnsWithTypeAndName(), + header.getColumnsWithTypeAndName(), + convert_actions_match_columns_mode); + auto actions = std::make_shared( + std::move(convert_actions_dag), + ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes)); + + builder.addSimpleTransform([&](const Block & stream_header) + { + return std::make_shared(stream_header, actions); + }); } bool ReadFromMerge::requestReadingInOrder(InputOrderInfoPtr order_info_) diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index 3e0a5d1456f..0d34052c7cd 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -184,6 +184,11 @@ private: using Aliases = std::vector; + static SelectQueryInfo getModifiedQueryInfo(const SelectQueryInfo & query_info, + const ContextPtr & modified_context, + const StorageWithLockAndName & storage_with_lock_and_name, + const StorageSnapshotPtr & storage_snapshot); + QueryPipelineBuilderPtr createSources( const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, diff --git a/tests/queries/0_stateless/02563_analyzer_merge.reference b/tests/queries/0_stateless/02563_analyzer_merge.reference new file mode 100644 index 00000000000..8be01c88d6f --- /dev/null +++ b/tests/queries/0_stateless/02563_analyzer_merge.reference @@ -0,0 +1,2 @@ +0 Value_0 02563_db test_merge_table_1 +1 Value_1 02563_db test_merge_table_2 diff --git a/tests/queries/0_stateless/02563_analyzer_merge.sql b/tests/queries/0_stateless/02563_analyzer_merge.sql new file mode 100644 index 00000000000..c90f7dcb2a5 --- /dev/null +++ b/tests/queries/0_stateless/02563_analyzer_merge.sql @@ -0,0 +1,38 @@ +-- Tags: no-parallel + +SET allow_experimental_analyzer = 1; + +DROP DATABASE IF EXISTS 02563_db; +CREATE DATABASE 02563_db; + +DROP TABLE IF EXISTS 02563_db.test_merge_table_1; +CREATE TABLE 02563_db.test_merge_table_1 +( + id UInt64, + value String +) ENGINE=MergeTree ORDER BY id; + +INSERT INTO 02563_db.test_merge_table_1 VALUES (0, 'Value_0'); + +DROP TABLE IF EXISTS 02563_db.test_merge_table_2; +CREATE TABLE 02563_db.test_merge_table_2 +( + id UInt64, + value String +) ENGINE=MergeTree ORDER BY id; + +INSERT INTO 02563_db.test_merge_table_2 VALUES (1, 'Value_1'); + +DROP TABLE IF EXISTS 02563_db.test_merge_table; +CREATE TABLE 02563_db.test_merge_table +( + id UInt64, + value String +) ENGINE=Merge(02563_db, '^test_merge_table'); + +SELECT id, value, _database, _table FROM 02563_db.test_merge_table ORDER BY id; + +DROP TABLE 02563_db.test_merge_table; +DROP TABLE 02563_db.test_merge_table_1; +DROP TABLE 02563_db.test_merge_table_2; +DROP DATABASE 02563_db; From 6b2adc1ec264786afbf65a97fd4015424ed649b0 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 15 Feb 2023 15:03:52 +0100 Subject: [PATCH 317/717] Analyzer storage Merge fixes --- src/Analyzer/FunctionNode.h | 3 +- .../InterpreterSelectQueryAnalyzer.cpp | 25 ++- src/Planner/Planner.cpp | 16 +- src/Planner/Planner.h | 15 +- src/Planner/PlannerJoinTree.cpp | 5 +- src/Storages/StorageDistributed.cpp | 4 +- src/Storages/StorageMerge.cpp | 179 ++++++++++-------- src/Storages/StorageMerge.h | 9 +- 8 files changed, 132 insertions(+), 124 deletions(-) diff --git a/src/Analyzer/FunctionNode.h b/src/Analyzer/FunctionNode.h index a3212b3fb07..6819232b4be 100644 --- a/src/Analyzer/FunctionNode.h +++ b/src/Analyzer/FunctionNode.h @@ -125,7 +125,8 @@ public: { if (kind != FunctionKind::ORDINARY) throw Exception(ErrorCodes::LOGICAL_ERROR, - "Function node with name '{}' is not resolved as ordinary function"); + "Function node with name '{}' is not resolved as ordinary function", + function_name); return std::static_pointer_cast(function); } diff --git a/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp b/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp index 0f5ccb3ae06..1a83d93c87d 100644 --- a/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp +++ b/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp @@ -16,9 +16,11 @@ #include #include +#include #include #include #include +#include #include #include @@ -108,6 +110,18 @@ void replaceStorageInQueryTree(QueryTreeNodePtr & query_tree, const ContextPtr & } auto replacement_table_expression = std::make_shared(storage, context); + std::optional table_expression_modifiers; + + if (auto * table_node = table_expression_to_replace->as()) + table_expression_modifiers = table_node->getTableExpressionModifiers(); + else if (auto * table_function_node = table_expression_to_replace->as()) + table_expression_modifiers = table_function_node->getTableExpressionModifiers(); + else if (auto * identifier_node = table_expression_to_replace->as()) + table_expression_modifiers = identifier_node->getTableExpressionModifiers(); + + if (table_expression_modifiers) + replacement_table_expression->setTableExpressionModifiers(*table_expression_modifiers); + query_tree = query_tree->cloneAndReplace(table_expression_to_replace, std::move(replacement_table_expression)); } @@ -132,11 +146,6 @@ QueryTreeNodePtr buildQueryTreeAndRunPasses(const ASTPtr & query, return query_tree; } -PlannerConfiguration buildPlannerConfiguration(const SelectQueryOptions & select_query_options) -{ - return {.only_analyze = select_query_options.only_analyze}; -} - } InterpreterSelectQueryAnalyzer::InterpreterSelectQueryAnalyzer( @@ -147,7 +156,7 @@ InterpreterSelectQueryAnalyzer::InterpreterSelectQueryAnalyzer( , context(buildContext(context_, select_query_options_)) , select_query_options(select_query_options_) , query_tree(buildQueryTreeAndRunPasses(query, select_query_options, context, nullptr /*storage*/)) - , planner(query_tree, select_query_options, buildPlannerConfiguration(select_query_options)) + , planner(query_tree, select_query_options) { } @@ -160,7 +169,7 @@ InterpreterSelectQueryAnalyzer::InterpreterSelectQueryAnalyzer( , context(buildContext(context_, select_query_options_)) , select_query_options(select_query_options_) , query_tree(buildQueryTreeAndRunPasses(query, select_query_options, context, storage_)) - , planner(query_tree, select_query_options, buildPlannerConfiguration(select_query_options)) + , planner(query_tree, select_query_options) { } @@ -172,7 +181,7 @@ InterpreterSelectQueryAnalyzer::InterpreterSelectQueryAnalyzer( , context(buildContext(context_, select_query_options_)) , select_query_options(select_query_options_) , query_tree(query_tree_) - , planner(query_tree, select_query_options, buildPlannerConfiguration(select_query_options)) + , planner(query_tree, select_query_options) { } diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp index f84e4db4924..a4714c05a23 100644 --- a/src/Planner/Planner.cpp +++ b/src/Planner/Planner.cpp @@ -979,34 +979,28 @@ PlannerContextPtr buildPlannerContext(const QueryTreeNodePtr & query_tree_node, } Planner::Planner(const QueryTreeNodePtr & query_tree_, - const SelectQueryOptions & select_query_options_, - PlannerConfiguration planner_configuration_) + const SelectQueryOptions & select_query_options_) : query_tree(query_tree_) , select_query_options(select_query_options_) , planner_context(buildPlannerContext(query_tree, select_query_options, std::make_shared())) - , planner_configuration(std::move(planner_configuration_)) { } Planner::Planner(const QueryTreeNodePtr & query_tree_, const SelectQueryOptions & select_query_options_, - GlobalPlannerContextPtr global_planner_context_, - PlannerConfiguration planner_configuration_) + GlobalPlannerContextPtr global_planner_context_) : query_tree(query_tree_) , select_query_options(select_query_options_) , planner_context(buildPlannerContext(query_tree_, select_query_options, std::move(global_planner_context_))) - , planner_configuration(std::move(planner_configuration_)) { } Planner::Planner(const QueryTreeNodePtr & query_tree_, const SelectQueryOptions & select_query_options_, - PlannerContextPtr planner_context_, - PlannerConfiguration planner_configuration_) + PlannerContextPtr planner_context_) : query_tree(query_tree_) , select_query_options(select_query_options_) , planner_context(std::move(planner_context_)) - , planner_configuration(std::move(planner_configuration_)) { } @@ -1015,7 +1009,7 @@ void Planner::buildQueryPlanIfNeeded() if (query_plan.isInitialized()) return; - if (query_tree->as()) + if (query_tree->getNodeType() == QueryTreeNodeType::UNION) buildPlanForUnionNode(); else buildPlanForQueryNode(); @@ -1174,7 +1168,7 @@ void Planner::buildPlanForQueryNode() QueryProcessingStage::Enum from_stage = QueryProcessingStage::FetchColumns; - if (planner_configuration.only_analyze) + if (select_query_options.only_analyze) { Block join_tree_block; diff --git a/src/Planner/Planner.h b/src/Planner/Planner.h index a87880eab6b..4427e8cc504 100644 --- a/src/Planner/Planner.h +++ b/src/Planner/Planner.h @@ -16,30 +16,22 @@ using GlobalPlannerContextPtr = std::shared_ptr; class PlannerContext; using PlannerContextPtr = std::shared_ptr; -struct PlannerConfiguration -{ - bool only_analyze = false; -}; - class Planner { public: /// Initialize planner with query tree after analysis phase Planner(const QueryTreeNodePtr & query_tree_, - const SelectQueryOptions & select_query_options_, - PlannerConfiguration planner_configuration_ = {}); + const SelectQueryOptions & select_query_options_); /// Initialize planner with query tree after query analysis phase and global planner context Planner(const QueryTreeNodePtr & query_tree_, const SelectQueryOptions & select_query_options_, - GlobalPlannerContextPtr global_planner_context_, - PlannerConfiguration planner_configuration_ = {}); + GlobalPlannerContextPtr global_planner_context_); /// Initialize planner with query tree after query analysis phase and planner context Planner(const QueryTreeNodePtr & query_tree_, const SelectQueryOptions & select_query_options_, - PlannerContextPtr planner_context_, - PlannerConfiguration planner_configuration_ = {}); + PlannerContextPtr planner_context_); const QueryPlan & getQueryPlan() const { @@ -69,7 +61,6 @@ private: QueryPlan query_plan; SelectQueryOptions select_query_options; PlannerContextPtr planner_context; - PlannerConfiguration planner_configuration; StorageLimitsList storage_limits; }; diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index 0a6d38a15be..0c74479615a 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -370,9 +370,8 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(const QueryTreeNodePtr & tabl else { Planner planner(select_query_info.query_tree, - SelectQueryOptions(from_stage), - select_query_info.planner_context, - PlannerConfiguration{.only_analyze = true}); + SelectQueryOptions(from_stage).analyze(), + select_query_info.planner_context); planner.buildQueryPlanIfNeeded(); auto expected_header = planner.getQueryPlan().getCurrentDataStream().header; diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index cafb9c1abe8..22ebd401377 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -736,9 +736,7 @@ void StorageDistributed::read( query_ast = queryNodeToSelectQuery(query_tree_with_replaced_distributed_table); - Planner planner(query_tree_with_replaced_distributed_table, - SelectQueryOptions(processed_stage), - PlannerConfiguration{.only_analyze = true}); + Planner planner(query_tree_with_replaced_distributed_table, SelectQueryOptions(processed_stage).analyze()); planner.buildQueryPlanIfNeeded(); header = planner.getQueryPlan().getCurrentDataStream().header; diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 3f31dbac311..af7bd96cd0b 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -533,23 +533,34 @@ SelectQueryInfo ReadFromMerge::getModifiedQueryInfo(const SelectQueryInfo & quer { const auto & [database_name, storage, storage_lock, table_name] = storage_with_lock_and_name; const StorageID current_storage_id = storage->getStorageID(); - bool is_storage_merge_engine = storage->as(); SelectQueryInfo modified_query_info = query_info; if (modified_query_info.table_expression) { auto replacement_table_expression = std::make_shared(storage, storage_lock, storage_snapshot); + if (query_info.table_expression_modifiers) + replacement_table_expression->setTableExpressionModifiers(*query_info.table_expression_modifiers); + modified_query_info.query_tree = modified_query_info.query_tree->cloneAndReplace(modified_query_info.table_expression, replacement_table_expression); modified_query_info.table_expression = replacement_table_expression; + modified_query_info.planner_context->getOrCreateTableExpressionData(replacement_table_expression); - if (!is_storage_merge_engine) - { - std::unordered_map column_name_to_node; + auto get_column_options = GetColumnsOptions(GetColumnsOptions::All).withExtendedObjects().withVirtuals(); + if (storage_snapshot->storage.supportsSubcolumns()) + get_column_options.withSubcolumns(); + + std::unordered_map column_name_to_node; + + if (!storage_snapshot->tryGetColumn(get_column_options, "_table")) column_name_to_node.emplace("_table", std::make_shared(current_storage_id.table_name)); + + if (!storage_snapshot->tryGetColumn(get_column_options, "_database")) column_name_to_node.emplace("_database", std::make_shared(current_storage_id.database_name)); + if (!column_name_to_node.empty()) + { replaceColumns(modified_query_info.query_tree, replacement_table_expression, column_name_to_node); @@ -559,6 +570,7 @@ SelectQueryInfo ReadFromMerge::getModifiedQueryInfo(const SelectQueryInfo & quer } else { + bool is_storage_merge_engine = storage->as(); modified_query_info.query = query_info.query->clone(); /// Original query could contain JOIN but we need only the first joined table and its columns. @@ -585,7 +597,7 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( const Block & header, const Aliases & aliases, const StorageWithLockAndName & storage_with_lock, - Names & real_column_names, + Names real_column_names, ContextMutablePtr modified_context, size_t streams_num, bool concat_streams) @@ -604,13 +616,82 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( modified_select.setFinal(); } - if (modified_context->getSettingsRef().allow_experimental_analyzer) + bool allow_experimental_analyzer = modified_context->getSettingsRef().allow_experimental_analyzer; + + auto storage_stage = storage->getQueryProcessingStage(modified_context, + QueryProcessingStage::Complete, + storage_snapshot, + modified_query_info); + if (processed_stage <= storage_stage || (allow_experimental_analyzer && processed_stage == QueryProcessingStage::FetchColumns)) { - InterpreterSelectQueryAnalyzer interpreter(modified_query_info.query, - modified_context, - SelectQueryOptions(processed_stage).ignoreProjections(), - storage); - builder = std::make_unique(interpreter.buildQueryPipeline()); + /// If there are only virtual columns in query, you must request at least one other column. + if (real_column_names.empty()) + real_column_names.push_back(ExpressionActions::getSmallestColumn(storage_snapshot->metadata->getColumns().getAllPhysical()).name); + + QueryPlan plan; + + StorageView * view = dynamic_cast(storage.get()); + if (!view || allow_experimental_analyzer) + { + storage->read(plan, + real_column_names, + storage_snapshot, + modified_query_info, + modified_context, + processed_stage, + max_block_size, + UInt32(streams_num)); + } + else + { + /// For view storage, we need to rewrite the `modified_query_info.view_query` to optimize read. + /// The most intuitive way is to use InterpreterSelectQuery. + + /// Intercept the settings + modified_context->setSetting("max_threads", streams_num); + modified_context->setSetting("max_streams_to_max_threads_ratio", 1); + modified_context->setSetting("max_block_size", max_block_size); + + InterpreterSelectQuery interpreter(modified_query_info.query, + modified_context, + storage, + view->getInMemoryMetadataPtr(), + SelectQueryOptions(processed_stage)); + interpreter.buildQueryPlan(plan); + } + + if (!plan.isInitialized()) + return {}; + + if (auto * read_from_merge_tree = typeid_cast(plan.getRootNode()->step.get())) + read_from_merge_tree->addFilterNodes(added_filter_nodes); + + builder = plan.buildQueryPipeline( + QueryPlanOptimizationSettings::fromContext(modified_context), + BuildQueryPipelineSettings::fromContext(modified_context)); + } + else if (processed_stage > storage_stage || (allow_experimental_analyzer && processed_stage != QueryProcessingStage::FetchColumns)) + { + /// Maximum permissible parallelism is streams_num + modified_context->setSetting("max_threads", streams_num); + modified_context->setSetting("max_streams_to_max_threads_ratio", 1); + + if (allow_experimental_analyzer) + { + InterpreterSelectQueryAnalyzer interpreter(modified_query_info.query_tree, + modified_context, + SelectQueryOptions(processed_stage).ignoreProjections()); + builder = std::make_unique(interpreter.buildQueryPipeline()); + } + else + { + modified_select.replaceDatabaseAndTable(database_name, table_name); + /// TODO: Find a way to support projections for StorageMerge + InterpreterSelectQuery interpreter{modified_query_info.query, + modified_context, + SelectQueryOptions(processed_stage).ignoreProjections()}; + builder = std::make_unique(interpreter.buildQueryPipeline()); + } /** Materialization is needed, since from distributed storage the constants come materialized. * If you do not do this, different types (Const and non-Const) columns will be produced in different threads, @@ -618,75 +699,6 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( */ builder->addSimpleTransform([](const Block & stream_header) { return std::make_shared(stream_header); }); } - else - { - auto storage_stage - = storage->getQueryProcessingStage(modified_context, QueryProcessingStage::Complete, storage_snapshot, modified_query_info); - if (processed_stage <= storage_stage) - { - /// If there are only virtual columns in query, you must request at least one other column. - if (real_column_names.empty()) - real_column_names.push_back(ExpressionActions::getSmallestColumn(storage_snapshot->metadata->getColumns().getAllPhysical()).name); - - QueryPlan plan; - if (StorageView * view = dynamic_cast(storage.get())) - { - /// For view storage, we need to rewrite the `modified_query_info.view_query` to optimize read. - /// The most intuitive way is to use InterpreterSelectQuery. - - /// Intercept the settings - modified_context->setSetting("max_threads", streams_num); - modified_context->setSetting("max_streams_to_max_threads_ratio", 1); - modified_context->setSetting("max_block_size", max_block_size); - - InterpreterSelectQuery( - modified_query_info.query, modified_context, storage, view->getInMemoryMetadataPtr(), SelectQueryOptions(processed_stage)) - .buildQueryPlan(plan); - } - else - { - storage->read( - plan, - real_column_names, - storage_snapshot, - modified_query_info, - modified_context, - processed_stage, - max_block_size, - UInt32(streams_num)); - } - - if (!plan.isInitialized()) - return {}; - - if (auto * read_from_merge_tree = typeid_cast(plan.getRootNode()->step.get())) - read_from_merge_tree->addFilterNodes(added_filter_nodes); - - builder = plan.buildQueryPipeline( - QueryPlanOptimizationSettings::fromContext(modified_context), - BuildQueryPipelineSettings::fromContext(modified_context)); - } - else if (processed_stage > storage_stage) - { - modified_select.replaceDatabaseAndTable(database_name, table_name); - - /// Maximum permissible parallelism is streams_num - modified_context->setSetting("max_threads", streams_num); - modified_context->setSetting("max_streams_to_max_threads_ratio", 1); - - /// TODO: Find a way to support projections for StorageMerge - InterpreterSelectQuery interpreter{ - modified_query_info.query, modified_context, SelectQueryOptions(processed_stage).ignoreProjections()}; - - builder = std::make_unique(interpreter.buildQueryPipeline()); - - /** Materialization is needed, since from distributed storage the constants come materialized. - * If you do not do this, different types (Const and non-Const) columns will be produced in different threads, - * And this is not allowed, since all code is based on the assumption that in the block stream all types are the same. - */ - builder->addSimpleTransform([](const Block & stream_header) { return std::make_shared(stream_header); }); - } - } if (builder->initialized()) { @@ -739,7 +751,7 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( /// Subordinary tables could have different but convertible types, like numeric types of different width. /// We must return streams with structure equals to structure of Merge table. - convertingSourceStream(header, storage_snapshot->metadata, aliases, modified_context, *builder); + convertingSourceStream(header, storage_snapshot->metadata, aliases, modified_context, *builder, processed_stage); } return builder; @@ -914,7 +926,8 @@ void ReadFromMerge::convertingSourceStream( const StorageMetadataPtr & metadata_snapshot, const Aliases & aliases, ContextPtr local_context, - QueryPipelineBuilder & builder) + QueryPipelineBuilder & builder, + const QueryProcessingStage::Enum & processed_stage) { Block before_block_header = builder.getHeader(); @@ -940,7 +953,7 @@ void ReadFromMerge::convertingSourceStream( ActionsDAG::MatchColumnsMode convert_actions_match_columns_mode = ActionsDAG::MatchColumnsMode::Name; - if (local_context->getSettingsRef().allow_experimental_analyzer) + if (local_context->getSettingsRef().allow_experimental_analyzer && processed_stage != QueryProcessingStage::FetchColumns) convert_actions_match_columns_mode = ActionsDAG::MatchColumnsMode::Position; auto convert_actions_dag = ActionsDAG::makeConvertingActions(builder.getHeader().getColumnsWithTypeAndName(), diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index 0d34052c7cd..05098b6df51 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -197,15 +197,18 @@ private: const Block & header, const Aliases & aliases, const StorageWithLockAndName & storage_with_lock, - Names & real_column_names, + Names real_column_names, ContextMutablePtr modified_context, size_t streams_num, bool concat_streams = false); static void convertingSourceStream( - const Block & header, const StorageMetadataPtr & metadata_snapshot, const Aliases & aliases, + const Block & header, + const StorageMetadataPtr & metadata_snapshot, + const Aliases & aliases, ContextPtr context, - QueryPipelineBuilder & builder); + QueryPipelineBuilder & builder, + const QueryProcessingStage::Enum & processed_stage); }; } From 4ddf1b0f488d8d7956624c9113d3875e26632887 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 15 Feb 2023 15:11:16 +0100 Subject: [PATCH 318/717] Planner filter push down optimization fix --- .../QueryPlan/Optimizations/filterPushDown.cpp | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp index dbf389163be..46fe3055e32 100644 --- a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp +++ b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp @@ -333,8 +333,19 @@ size_t tryPushDownFilter(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes // { // } - if (auto updated_steps = simplePushDownOverStep(parent_node, nodes, child)) - return updated_steps; + if (auto * sorting = typeid_cast(child.get())) + { + const auto & sort_description = sorting->getSortDescription(); + auto sort_description_it = std::find_if(sort_description.begin(), sort_description.end(), [&](auto & sort_column_description) + { + return sort_column_description.column_name == filter->getFilterColumnName(); + }); + bool can_remove_filter = sort_description_it == sort_description.end(); + + Names allowed_inputs = child->getOutputStream().header.getNames(); + if (auto updated_steps = tryAddNewFilterStep(parent_node, nodes, allowed_inputs, can_remove_filter)) + return updated_steps; + } if (auto updated_steps = simplePushDownOverStep(parent_node, nodes, child)) return updated_steps; From b72ea982b007a40d4cb186763857c189599761be Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 15 Feb 2023 15:14:04 +0100 Subject: [PATCH 319/717] Remove unnecessary includes of InterpreterSelectQuery header --- src/Interpreters/ClusterProxy/executeQuery.cpp | 1 - src/Processors/QueryPlan/ReadFromRemote.cpp | 3 ++- src/Storages/MergeTree/MergeTreeDataWriter.cpp | 1 - src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp | 2 -- src/Storages/WindowView/StorageWindowView.h | 2 +- 5 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index 2e035ef883f..85a012d126f 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index 6d36146c7e8..6a3670f964e 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -1,4 +1,6 @@ #include + +#include #include #include #include @@ -11,7 +13,6 @@ #include #include #include -#include #include "Common/logger_useful.h" #include #include diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index 93b0abeca35..846685c5093 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include diff --git a/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp b/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp index a30dd6deb77..d194c8b8201 100644 --- a/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp +++ b/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp @@ -27,8 +27,6 @@ #include #include -#include -#include #include #include diff --git a/src/Storages/WindowView/StorageWindowView.h b/src/Storages/WindowView/StorageWindowView.h index b313e466211..af933b0c35c 100644 --- a/src/Storages/WindowView/StorageWindowView.h +++ b/src/Storages/WindowView/StorageWindowView.h @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include From 57b5d9852f0b6f34681c579940377cfa1cc6fbf8 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 15 Feb 2023 18:26:02 +0100 Subject: [PATCH 320/717] Fixed build --- src/Storages/WindowView/StorageWindowView.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Storages/WindowView/StorageWindowView.cpp b/src/Storages/WindowView/StorageWindowView.cpp index 0d24bf7899f..72df3bc9464 100644 --- a/src/Storages/WindowView/StorageWindowView.cpp +++ b/src/Storages/WindowView/StorageWindowView.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include From 05baf271f028013bee33fb3904b1ae109d32d284 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Thu, 16 Feb 2023 11:31:24 +0100 Subject: [PATCH 321/717] Analyzer fix table functions with invalid arguments analysis --- .../Passes/ComparisonTupleEliminationPass.cpp | 5 ++++ .../OptimizeGroupByFunctionKeysPass.cpp | 5 +++- src/Analyzer/Passes/QueryAnalysisPass.cpp | 28 ++++++++++++++++++- src/Analyzer/QueryNode.cpp | 1 + src/Analyzer/QueryTreeBuilder.cpp | 13 +++++++++ src/Storages/StorageDistributed.cpp | 7 +---- 6 files changed, 51 insertions(+), 8 deletions(-) diff --git a/src/Analyzer/Passes/ComparisonTupleEliminationPass.cpp b/src/Analyzer/Passes/ComparisonTupleEliminationPass.cpp index 29dec874e2c..4e0562a2fe8 100644 --- a/src/Analyzer/Passes/ComparisonTupleEliminationPass.cpp +++ b/src/Analyzer/Passes/ComparisonTupleEliminationPass.cpp @@ -23,6 +23,11 @@ public: : context(std::move(context_)) {} + static bool needChildVisit(QueryTreeNodePtr &, QueryTreeNodePtr & child) + { + return child->getNodeType() != QueryTreeNodeType::TABLE_FUNCTION; + } + void visitImpl(QueryTreeNodePtr & node) const { auto * function_node = node->as(); diff --git a/src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.cpp b/src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.cpp index b313765ee31..f6c4d2bc15d 100644 --- a/src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.cpp +++ b/src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.cpp @@ -18,8 +18,11 @@ public: using Base = InDepthQueryTreeVisitorWithContext; using Base::Base; - static bool needChildVisit(QueryTreeNodePtr & /*parent*/, QueryTreeNodePtr & child) + static bool needChildVisit(QueryTreeNodePtr & parent, QueryTreeNodePtr & child) { + if (parent->getNodeType() == QueryTreeNodeType::TABLE_FUNCTION) + return false; + return !child->as(); } diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index f9119c2c472..f58e1c619db 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -1072,6 +1072,12 @@ public: break; } + case QueryTreeNodeType::TABLE_FUNCTION: + { + QueryExpressionsAliasVisitor expressions_alias_visitor(scope); + resolveTableFunction(node, scope, expressions_alias_visitor, false /*nested_table_function*/); + break; + } default: { throw Exception(ErrorCodes::BAD_ARGUMENTS, @@ -5902,7 +5908,27 @@ void QueryAnalyzer::resolveTableFunction(QueryTreeNodePtr & table_function_node, } } - resolveExpressionNode(table_function_argument, scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); + /** Table functions arguments can contain expressions with invalid identifiers. + * We cannot skip analysis for such arguments, because some table functions cannot provide + * information if analysis for argument should be skipped until other arguments will be resolved. + * + * Example: SELECT key from remote('127.0.0.{1,2}', view(select number AS key from numbers(2)), key); + * Example: SELECT id from remote('127.0.0.{1,2}', 'default', 'test_table', id); + */ + try + { + resolveExpressionNode(table_function_argument, scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); + } + catch (const Exception & exception) + { + if (exception.code() == ErrorCodes::UNKNOWN_IDENTIFIER) + { + result_table_function_arguments.push_back(table_function_argument); + continue; + } + + throw; + } if (auto * expression_list = table_function_argument->as()) { diff --git a/src/Analyzer/QueryNode.cpp b/src/Analyzer/QueryNode.cpp index 06f72d32253..774f3376f48 100644 --- a/src/Analyzer/QueryNode.cpp +++ b/src/Analyzer/QueryNode.cpp @@ -336,6 +336,7 @@ ASTPtr QueryNode::toASTImpl() const { auto settings_query = std::make_shared(); settings_query->changes = settings_changes; + settings_query->is_standalone = false; select_query->setExpression(ASTSelectQuery::Expression::SETTINGS, std::move(settings_query)); } diff --git a/src/Analyzer/QueryTreeBuilder.cpp b/src/Analyzer/QueryTreeBuilder.cpp index 9348332923e..98bc1d0eeb4 100644 --- a/src/Analyzer/QueryTreeBuilder.cpp +++ b/src/Analyzer/QueryTreeBuilder.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -61,6 +62,7 @@ namespace ErrorCodes extern const int EXPECTED_ALL_OR_ANY; extern const int NOT_IMPLEMENTED; extern const int BAD_ARGUMENTS; + extern const int UNKNOWN_QUERY_PARAMETER; } namespace @@ -540,6 +542,11 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression, co result = std::move(query_node); } + else if (const auto * select_with_union_query = expression->as()) + { + auto query_node = buildSelectWithUnionExpression(expression, false /*is_subquery*/, {} /*cte_name*/, context); + result = std::move(query_node); + } else if (const auto * with_element = expression->as()) { auto with_element_subquery = with_element->subquery->as().children.at(0); @@ -588,6 +595,12 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression, co auto column_transformers = buildColumnTransformers(qualified_columns_list_matcher->transformers, context); result = std::make_shared(Identifier(qualified_identifier.name_parts), std::move(column_list_identifiers), std::move(column_transformers)); } + else if (const auto * query_parameter = expression->as()) + { + throw Exception(ErrorCodes::UNKNOWN_QUERY_PARAMETER, + "Query parameter {} was not set", + backQuote(query_parameter->name)); + } else { throw Exception(ErrorCodes::UNSUPPORTED_METHOD, diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 22ebd401377..2719b95ba5d 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -678,13 +678,8 @@ QueryTreeNodePtr buildQueryTreeDistributedTableReplacedWithLocalTable(const Sele table_function_node->getArgumentsNode() = remote_table_function_node.getArgumentsNode(); QueryAnalysisPass query_analysis_pass; - query_analysis_pass.run(table_function_node->getArgumentsNode(), query_context); + query_analysis_pass.run(table_function_node, query_context); - auto remote_table_function_to_resolve = table_function_node->toAST(); - TableFunctionPtr table_function_ptr = TableFunctionFactory::instance().get(remote_table_function_to_resolve, query_context); - auto table_function_storage = table_function_ptr->execute(remote_table_function_to_resolve, query_context, table_function_ptr->getName()); - - table_function_node->resolve(std::move(table_function_ptr), std::move(table_function_storage), query_context); replacement_table_expression = std::move(table_function_node); } else From ac62356477c87a84142609afa2e315a1ce81f4cd Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 16 Feb 2023 13:05:13 +0100 Subject: [PATCH 322/717] fix --- .../configs/config.d/storage_conf.xml | 7 ------- .../test.py | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/tests/integration/test_replicated_merge_tree_s3_zero_copy/configs/config.d/storage_conf.xml b/tests/integration/test_replicated_merge_tree_s3_zero_copy/configs/config.d/storage_conf.xml index 801e2072f17..15239041478 100644 --- a/tests/integration/test_replicated_merge_tree_s3_zero_copy/configs/config.d/storage_conf.xml +++ b/tests/integration/test_replicated_merge_tree_s3_zero_copy/configs/config.d/storage_conf.xml @@ -55,11 +55,4 @@ 3 0 - - - system - zookeeper_log
- 5000 - 5000 - diff --git a/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py b/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py index fb93dc8aa7c..f0bc12e3125 100644 --- a/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py +++ b/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py @@ -160,7 +160,11 @@ def test_drop_table(cluster): "action": "REJECT --reject-with tcp-reset", } ) + + # Will drop in background with retries node.query("drop table test_drop_table") + + # It should not be possible to create a replica with the same path until the previous one is completely dropped for i in range(0, 100): node.query_and_get_answer_with_error( "create table if not exists test_drop_table (n int) " @@ -169,11 +173,21 @@ def test_drop_table(cluster): ) time.sleep(0.2) + # Wait for drop to actually finish + node.wait_for_log_line( + "Removing metadata /var/lib/clickhouse/metadata_dropped/default.test_drop_table", + timeout=60, + look_behind_lines=1000000, + ) + + # It could leave some leftovers, remove them replicas = node.query_with_retry( "select name from system.zookeeper where path='/test/drop_table/replicas'" ) if "1" in replicas and "test_drop_table" not in node.query("show tables"): node2.query("system drop replica '1' from table test_drop_table") + + # Just in case table was not created due to connection errors node.query( "create table if not exists test_drop_table (n int) engine=ReplicatedMergeTree('/test/drop_table', '1') " "order by n partition by n % 99 settings storage_policy='s3'" From 0954a57a6a029bb2f3b9b80a0eea00119682aa0a Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 7 Feb 2023 12:25:38 +0100 Subject: [PATCH 323/717] Enable back --- .../test.py | 11 --------- .../test.py | 23 ------------------- 2 files changed, 34 deletions(-) diff --git a/tests/integration/test_postgresql_replica_database_engine_1/test.py b/tests/integration/test_postgresql_replica_database_engine_1/test.py index 5072c261cf7..0b3fb68708a 100644 --- a/tests/integration/test_postgresql_replica_database_engine_1/test.py +++ b/tests/integration/test_postgresql_replica_database_engine_1/test.py @@ -1,12 +1,5 @@ import pytest -# FIXME Tests with MaterializedPostgresSQL are temporarily disabled -# https://github.com/ClickHouse/ClickHouse/issues/36898 -# https://github.com/ClickHouse/ClickHouse/issues/38677 -# https://github.com/ClickHouse/ClickHouse/pull/39272#issuecomment-1190087190 - -pytestmark = pytest.mark.skip - import time import os.path as p import random @@ -774,8 +767,6 @@ def test_concurrent_transactions(started_cluster): def test_abrupt_connection_loss_while_heavy_replication(started_cluster): - pytest.skip("Temporary disabled (FIXME)") - def transaction(thread_id): if thread_id % 2: conn = get_postgres_conn( @@ -851,8 +842,6 @@ def test_restart_server_while_replication_startup_not_finished(started_cluster): def test_abrupt_server_restart_while_heavy_replication(started_cluster): - pytest.skip("Temporary disabled (FIXME)") - def transaction(thread_id): if thread_id % 2: conn = get_postgres_conn( diff --git a/tests/integration/test_postgresql_replica_database_engine_2/test.py b/tests/integration/test_postgresql_replica_database_engine_2/test.py index 9b4de5356bf..666bab29bec 100644 --- a/tests/integration/test_postgresql_replica_database_engine_2/test.py +++ b/tests/integration/test_postgresql_replica_database_engine_2/test.py @@ -1,12 +1,5 @@ import pytest -# FIXME Tests with MaterializedPostgresSQL are temporarily disabled -# https://github.com/ClickHouse/ClickHouse/issues/36898 -# https://github.com/ClickHouse/ClickHouse/issues/38677 -# https://github.com/ClickHouse/ClickHouse/pull/39272#issuecomment-1190087190 - -pytestmark = pytest.mark.skip - import time import psycopg2 import os.path as p @@ -75,8 +68,6 @@ def setup_teardown(): def test_add_new_table_to_replication(started_cluster): - if instance.is_built_with_sanitizer() or instance.is_debug_build(): - pytest.skip("Temporary disabled (FIXME)") cursor = pg_manager.get_db_cursor() cursor.execute("DROP TABLE IF EXISTS test_table") NUM_TABLES = 5 @@ -192,8 +183,6 @@ def test_add_new_table_to_replication(started_cluster): def test_remove_table_from_replication(started_cluster): - if instance.is_built_with_sanitizer() or instance.is_debug_build(): - pytest.skip("Temporary disabled (FIXME)") NUM_TABLES = 5 pg_manager.create_and_fill_postgres_tables(NUM_TABLES, 10000) pg_manager.create_materialized_db( @@ -279,8 +268,6 @@ def test_remove_table_from_replication(started_cluster): def test_predefined_connection_configuration(started_cluster): - if instance.is_built_with_sanitizer() or instance.is_debug_build(): - pytest.skip("Temporary disabled (FIXME)") cursor = pg_manager.get_db_cursor() cursor.execute(f"DROP TABLE IF EXISTS test_table") cursor.execute(f"CREATE TABLE test_table (key integer PRIMARY KEY, value integer)") @@ -296,8 +283,6 @@ insert_counter = 0 def test_database_with_single_non_default_schema(started_cluster): - if instance.is_built_with_sanitizer() or instance.is_debug_build(): - pytest.skip("Temporary disabled (FIXME)") cursor = pg_manager.get_db_cursor() NUM_TABLES = 5 schema_name = "test_schema" @@ -399,8 +384,6 @@ def test_database_with_single_non_default_schema(started_cluster): def test_database_with_multiple_non_default_schemas_1(started_cluster): - if instance.is_built_with_sanitizer() or instance.is_debug_build(): - pytest.skip("Temporary disabled (FIXME)") cursor = pg_manager.get_db_cursor() NUM_TABLES = 5 @@ -521,8 +504,6 @@ def test_database_with_multiple_non_default_schemas_1(started_cluster): def test_database_with_multiple_non_default_schemas_2(started_cluster): - if instance.is_built_with_sanitizer() or instance.is_debug_build(): - pytest.skip("Temporary disabled (FIXME)") cursor = pg_manager.get_db_cursor() NUM_TABLES = 2 schemas_num = 2 @@ -646,8 +627,6 @@ def test_database_with_multiple_non_default_schemas_2(started_cluster): def test_table_override(started_cluster): - if instance.is_built_with_sanitizer() or instance.is_debug_build(): - pytest.skip("Temporary disabled (FIXME)") cursor = pg_manager.get_db_cursor() table_name = "table_override" materialized_database = "test_database" @@ -678,8 +657,6 @@ def test_table_override(started_cluster): def test_table_schema_changes_2(started_cluster): - if instance.is_built_with_sanitizer() or instance.is_debug_build(): - pytest.skip("Temporary disabled (FIXME)") cursor = pg_manager.get_db_cursor() table_name = "test_table" From 7f4c23ec8a22e231332090070b0bb82fb8318dfd Mon Sep 17 00:00:00 2001 From: flynn Date: Thu, 16 Feb 2023 12:48:22 +0000 Subject: [PATCH 324/717] fix --- src/Storages/IStorageDataLake.h | 4 ++-- src/Storages/StorageDeltaLake.cpp | 2 +- src/Storages/StorageHudi.cpp | 2 +- src/Storages/StorageIceberg.cpp | 17 ++++++------- src/Storages/StorageIceberg.h | 6 ++--- src/Storages/StorageS3.h | 24 ++----------------- src/TableFunctions/ITableFunctionDataLake.h | 18 ++++---------- src/TableFunctions/TableFunctionDeltaLake.cpp | 4 +++- src/TableFunctions/TableFunctionHudi.cpp | 3 ++- src/TableFunctions/TableFunctionIceberg.cpp | 3 ++- src/TableFunctions/TableFunctionS3.cpp | 12 +--------- src/TableFunctions/TableFunctionS3.h | 8 +++++++ 12 files changed, 37 insertions(+), 66 deletions(-) diff --git a/src/Storages/IStorageDataLake.h b/src/Storages/IStorageDataLake.h index 19ac2d65c2a..4efa63e7be2 100644 --- a/src/Storages/IStorageDataLake.h +++ b/src/Storages/IStorageDataLake.h @@ -11,7 +11,7 @@ # include # include -# include +# include namespace DB { @@ -33,7 +33,7 @@ public: std::optional format_settings_) : IStorage(table_id_) , base_configuration{configuration_} - , log(&Poco::Logger::get("Storage" + String(name) + "(" + table_id_.table_name + ")")) + , log(&Poco::Logger::get(fmt::format("Storage{}({})", name, table_id_.table_name))) { StorageInMemoryMetadata storage_metadata; StorageS3::updateS3Configuration(context_, base_configuration); diff --git a/src/Storages/StorageDeltaLake.cpp b/src/Storages/StorageDeltaLake.cpp index 6b09a29f628..ac3730d5c45 100644 --- a/src/Storages/StorageDeltaLake.cpp +++ b/src/Storages/StorageDeltaLake.cpp @@ -153,7 +153,7 @@ void registerStorageDeltaLake(StorageFactory & factory) configuration, args.table_id, args.columns, args.constraints, args.comment, args.getContext(), format_settings); }, { - .supports_settings = true, + .supports_settings = false, .supports_schema_inference = true, .source_access_type = AccessType::S3, }); diff --git a/src/Storages/StorageHudi.cpp b/src/Storages/StorageHudi.cpp index 48a92f5acd6..1dd2684d59b 100644 --- a/src/Storages/StorageHudi.cpp +++ b/src/Storages/StorageHudi.cpp @@ -112,7 +112,7 @@ void registerStorageHudi(StorageFactory & factory) configuration, args.table_id, args.columns, args.constraints, args.comment, args.getContext(), format_settings); }, { - .supports_settings = true, + .supports_settings = false, .supports_schema_inference = true, .source_access_type = AccessType::S3, }); diff --git a/src/Storages/StorageIceberg.cpp b/src/Storages/StorageIceberg.cpp index b25d2448a43..7626f4dbe82 100644 --- a/src/Storages/StorageIceberg.cpp +++ b/src/Storages/StorageIceberg.cpp @@ -39,8 +39,8 @@ IcebergMetadataParser::IcebergMetadataParser( template std::vector IcebergMetadataParser::getFiles() const { - auto metadata = getNewestMetaFile(); - auto manifest_list = getManiFestList(metadata); + auto metadata = fetchMetadataFile(); + auto manifest_list = getManifestList(metadata); /// When table first created and does not have any data if (manifest_list.empty()) @@ -53,10 +53,10 @@ std::vector IcebergMetadataParser::ge } template -String IcebergMetadataParser::getNewestMetaFile() const +String IcebergMetadataParser::fetchMetadataFile() const { /// Iceberg stores all the metadata.json in metadata directory, and the - /// newest version has the max version name, so we should list all of them + /// newest version has the max version name, so we should list all of them, /// then find the newest metadata. static constexpr auto meta_file_suffix = ".json"; auto metadata_files = MetadataReadHelper::listFilesMatchSuffix(base_configuration, metadata_directory, meta_file_suffix); @@ -65,12 +65,13 @@ String IcebergMetadataParser::getNewestMetaFi throw Exception( ErrorCodes::FILE_DOESNT_EXIST, "The metadata file for Iceberg table with path {} doesn't exist", base_configuration.url.key); + /// See comment above auto it = std::max_element(metadata_files.begin(), metadata_files.end()); return *it; } template -String IcebergMetadataParser::getManiFestList(const String & metadata_name) const +String IcebergMetadataParser::getManifestList(const String & metadata_name) const { auto buffer = MetadataReadHelper::createReadBuffer(metadata_name, context, base_configuration); String json_str; @@ -225,9 +226,9 @@ template std::vector IcebergMetadataParser::generateQueryFromKeys( const std::vector & keys, const String & format); -template String IcebergMetadataParser::getNewestMetaFile() const; +template String IcebergMetadataParser::fetchMetadataFile() const; -template String IcebergMetadataParser::getManiFestList(const String & metadata_name) const; +template String IcebergMetadataParser::getManifestList(const String & metadata_name) const; template std::vector IcebergMetadataParser::getManifestFiles(const String & manifest_list) const; @@ -250,7 +251,7 @@ void registerStorageIceberg(StorageFactory & factory) configuration, args.table_id, args.columns, args.constraints, args.comment, args.getContext(), format_settings); }, { - .supports_settings = true, + .supports_settings = false, .supports_schema_inference = true, .source_access_type = AccessType::S3, }); diff --git a/src/Storages/StorageIceberg.h b/src/Storages/StorageIceberg.h index 84670511625..50bd40aff1b 100644 --- a/src/Storages/StorageIceberg.h +++ b/src/Storages/StorageIceberg.h @@ -31,12 +31,10 @@ private: Configuration base_configuration; ContextPtr context; - String getNewestMetaFile() const; - String getManiFestList(const String & metadata_name) const; + String fetchMetadataFile() const; + String getManifestList(const String & metadata_name) const; std::vector getManifestFiles(const String & manifest_list) const; std::vector getFilesForRead(const std::vector & manifest_files) const; - - std::shared_ptr createS3ReadBuffer(const String & key) const; }; struct StorageIcebergName diff --git a/src/Storages/StorageS3.h b/src/Storages/StorageS3.h index dabbb25d828..9199d0066b5 100644 --- a/src/Storages/StorageS3.h +++ b/src/Storages/StorageS3.h @@ -34,23 +34,6 @@ class PullingPipelineExecutor; class StorageS3SequentialSource; class NamedCollection; -template -class IStorageDataLake; - -struct S3DataLakeMetadataReadHelper; - -struct StorageIcebergName; -template -class IcebergMetadataParser; - -struct StorageDeltaLakeName; -template -class DeltaLakeMetadataParser; - -struct StorageHudiName; -template -class HudiMetadataParser; - class StorageS3Source : public ISource, WithContext { public: @@ -315,12 +298,11 @@ public: static SchemaCache & getSchemaCache(const ContextPtr & ctx); + static void updateS3Configuration(ContextPtr, Configuration &); + private: friend class StorageS3Cluster; friend class TableFunctionS3Cluster; - friend class IStorageDataLake>; - friend class IStorageDataLake>; - friend class IStorageDataLake>; Configuration s3_configuration; std::vector keys; @@ -337,8 +319,6 @@ private: ObjectInfos object_infos; - static void updateS3Configuration(ContextPtr, Configuration &); - static std::shared_ptr createFileIterator( const Configuration & s3_configuration, const std::vector & keys, diff --git a/src/TableFunctions/ITableFunctionDataLake.h b/src/TableFunctions/ITableFunctionDataLake.h index 7948d72e0c5..f758c99d583 100644 --- a/src/TableFunctions/ITableFunctionDataLake.h +++ b/src/TableFunctions/ITableFunctionDataLake.h @@ -9,7 +9,6 @@ # include # include # include -# include namespace DB { @@ -18,7 +17,7 @@ namespace ErrorCodes extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } -template +template class ITableFunctionDataLake : public ITableFunction { public: @@ -63,24 +62,15 @@ protected: /// Parse args ASTs & args_func = ast_function->children; - const auto message = fmt::format( - "The signature of table function {} could be the following:\n" - " - url\n" - " - url, format\n" - " - url, format, structure\n" - " - url, access_key_id, secret_access_key\n" - " - url, format, structure, compression_method\n" - " - url, access_key_id, secret_access_key, format\n" - " - url, access_key_id, secret_access_key, format, structure\n" - " - url, access_key_id, secret_access_key, format, structure, compression_method", - getName()); + const auto message + = fmt::format("The signature of table function '{}' could be the following:\n{}", getName(), TableFunction::signature); if (args_func.size() != 1) throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Table function '{}' must have arguments", getName()); auto & args = args_func.at(0)->children; - TableFunctionS3::parseArgumentsImpl(message, args, context, configuration, false); + TableFunction::parseArgumentsImpl(message, args, context, configuration, false); if (configuration.format == "auto") configuration.format = "Parquet"; diff --git a/src/TableFunctions/TableFunctionDeltaLake.cpp b/src/TableFunctions/TableFunctionDeltaLake.cpp index a5602814abc..4b1f8ff62d5 100644 --- a/src/TableFunctions/TableFunctionDeltaLake.cpp +++ b/src/TableFunctions/TableFunctionDeltaLake.cpp @@ -5,6 +5,7 @@ # include # include # include +# include # include "registerTableFunctions.h" namespace DB @@ -15,7 +16,8 @@ struct TableFunctionDeltaLakeName static constexpr auto name = "deltaLake"; }; -using TableFunctionDeltaLake = ITableFunctionDataLake; +using TableFunctionDeltaLake + = ITableFunctionDataLake; void registerTableFunctionDeltaLake(TableFunctionFactory & factory) { diff --git a/src/TableFunctions/TableFunctionHudi.cpp b/src/TableFunctions/TableFunctionHudi.cpp index d7ddc49c1de..633132d2f9b 100644 --- a/src/TableFunctions/TableFunctionHudi.cpp +++ b/src/TableFunctions/TableFunctionHudi.cpp @@ -5,6 +5,7 @@ # include # include # include +# include # include "registerTableFunctions.h" namespace DB @@ -14,7 +15,7 @@ struct TableFunctionHudiName { static constexpr auto name = "hudi"; }; -using TableFunctionHudi = ITableFunctionDataLake; +using TableFunctionHudi = ITableFunctionDataLake; void registerTableFunctionHudi(TableFunctionFactory & factory) { diff --git a/src/TableFunctions/TableFunctionIceberg.cpp b/src/TableFunctions/TableFunctionIceberg.cpp index eb25a115055..e16a2ab04fb 100644 --- a/src/TableFunctions/TableFunctionIceberg.cpp +++ b/src/TableFunctions/TableFunctionIceberg.cpp @@ -5,6 +5,7 @@ # include # include # include +# include # include "registerTableFunctions.h" @@ -16,7 +17,7 @@ struct TableFunctionIcebergName static constexpr auto name = "iceberg"; }; -using TableFunctionIceberg = ITableFunctionDataLake; +using TableFunctionIceberg = ITableFunctionDataLake; void registerTableFunctionIceberg(TableFunctionFactory & factory) { diff --git a/src/TableFunctions/TableFunctionS3.cpp b/src/TableFunctions/TableFunctionS3.cpp index dfe20807421..1994787f831 100644 --- a/src/TableFunctions/TableFunctionS3.cpp +++ b/src/TableFunctions/TableFunctionS3.cpp @@ -116,17 +116,7 @@ void TableFunctionS3::parseArguments(const ASTPtr & ast_function, ContextPtr con /// Parse args ASTs & args_func = ast_function->children; - const auto message = fmt::format( - "The signature of table function {} could be the following:\n" \ - " - url\n" \ - " - url, format\n" \ - " - url, format, structure\n" \ - " - url, access_key_id, secret_access_key\n" \ - " - url, format, structure, compression_method\n" \ - " - url, access_key_id, secret_access_key, format\n" \ - " - url, access_key_id, secret_access_key, format, structure\n" \ - " - url, access_key_id, secret_access_key, format, structure, compression_method", - getName()); + const auto message = fmt::format("The signature of table function '{}' could be the following:\n{}", getName(), signature); if (args_func.size() != 1) throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Table function '{}' must have arguments.", getName()); diff --git a/src/TableFunctions/TableFunctionS3.h b/src/TableFunctions/TableFunctionS3.h index 29dea5d3631..859da9e9201 100644 --- a/src/TableFunctions/TableFunctionS3.h +++ b/src/TableFunctions/TableFunctionS3.h @@ -19,6 +19,14 @@ class TableFunctionS3 : public ITableFunction { public: static constexpr auto name = "s3"; + static constexpr auto signature = " - url\n" + " - url, format\n" + " - url, format, structure\n" + " - url, access_key_id, secret_access_key\n" + " - url, format, structure, compression_method\n" + " - url, access_key_id, secret_access_key, format\n" + " - url, access_key_id, secret_access_key, format, structure\n" + " - url, access_key_id, secret_access_key, format, structure, compression_method"; std::string getName() const override { return name; From 86d177e0ee708499898f8db4fce969d8561f0df1 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 16 Feb 2023 14:19:55 +0100 Subject: [PATCH 325/717] fix flaky test --- .../test_broken_detached_part_clean_up/test.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integration/test_broken_detached_part_clean_up/test.py b/tests/integration/test_broken_detached_part_clean_up/test.py index d39946102ef..1b53845517f 100644 --- a/tests/integration/test_broken_detached_part_clean_up/test.py +++ b/tests/integration/test_broken_detached_part_clean_up/test.py @@ -222,9 +222,9 @@ def test_store_cleanup(started_cluster): node1.wait_for_log_line( "Removing access rights for unused directory", timeout=60, - look_behind_lines=1000, + look_behind_lines=1000000, ) - node1.wait_for_log_line("directories from store", look_behind_lines=1000) + node1.wait_for_log_line("directories from store", timeout=60, look_behind_lines=1000000) store = node1.exec_in_container(["ls", f"{path_to_data}/store"]) assert "100" in store @@ -277,10 +277,10 @@ def test_store_cleanup(started_cluster): ) node1.wait_for_log_line( - "Removing unused directory", timeout=90, look_behind_lines=1000 + "Removing unused directory", timeout=90, look_behind_lines=1000000 ) - node1.wait_for_log_line("directories from store") - node1.wait_for_log_line("Nothing to clean up from store/") + node1.wait_for_log_line("directories from store", timeout=90, look_behind_lines=1000000) + node1.wait_for_log_line("Nothing to clean up from store/", timeout=90, look_behind_lines=1000000) store = node1.exec_in_container(["ls", f"{path_to_data}/store"]) assert "100" in store From a96d6af17ef362784b3b0ad359d7302840661df2 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 16 Feb 2023 13:37:40 +0000 Subject: [PATCH 326/717] Automatic style fix --- .../test_broken_detached_part_clean_up/test.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_broken_detached_part_clean_up/test.py b/tests/integration/test_broken_detached_part_clean_up/test.py index 1b53845517f..5b18fa34494 100644 --- a/tests/integration/test_broken_detached_part_clean_up/test.py +++ b/tests/integration/test_broken_detached_part_clean_up/test.py @@ -224,7 +224,9 @@ def test_store_cleanup(started_cluster): timeout=60, look_behind_lines=1000000, ) - node1.wait_for_log_line("directories from store", timeout=60, look_behind_lines=1000000) + node1.wait_for_log_line( + "directories from store", timeout=60, look_behind_lines=1000000 + ) store = node1.exec_in_container(["ls", f"{path_to_data}/store"]) assert "100" in store @@ -279,8 +281,12 @@ def test_store_cleanup(started_cluster): node1.wait_for_log_line( "Removing unused directory", timeout=90, look_behind_lines=1000000 ) - node1.wait_for_log_line("directories from store", timeout=90, look_behind_lines=1000000) - node1.wait_for_log_line("Nothing to clean up from store/", timeout=90, look_behind_lines=1000000) + node1.wait_for_log_line( + "directories from store", timeout=90, look_behind_lines=1000000 + ) + node1.wait_for_log_line( + "Nothing to clean up from store/", timeout=90, look_behind_lines=1000000 + ) store = node1.exec_in_container(["ls", f"{path_to_data}/store"]) assert "100" in store From 9feb448a5e25d2269329d16354d7daab9d562115 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 16 Feb 2023 14:41:18 +0100 Subject: [PATCH 327/717] fix a race condition --- programs/local/LocalServer.cpp | 2 +- programs/server/Server.cpp | 2 +- src/Interpreters/DatabaseCatalog.cpp | 2 +- src/Interpreters/DatabaseCatalog.h | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 133d629bbb1..571e3a3a4df 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -654,8 +654,8 @@ void LocalServer::processConfig() if (!config().has("only-system-tables")) { + DatabaseCatalog::instance().startupBackgroundCleanup(); loadMetadata(global_context); - DatabaseCatalog::instance().loadDatabases(); } /// For ClickHouse local if path is not set the loader will be disabled. diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index b97b48d9c68..df223317fcc 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -1656,11 +1656,11 @@ try /// that may execute DROP before loadMarkedAsDroppedTables() in background, /// and so loadMarkedAsDroppedTables() will find it and try to add, and UUID will overlap. database_catalog.loadMarkedAsDroppedTables(); + database_catalog.startupBackgroundCleanup(); /// Then, load remaining databases loadMetadata(global_context, default_database); convertDatabasesEnginesIfNeed(global_context); startupSystemTables(); - database_catalog.loadDatabases(); /// After loading validate that default database exists database_catalog.assertDatabaseExists(default_database); /// Load user-defined SQL functions. diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index ad5d9d4d325..24d5f3491fd 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -152,7 +152,7 @@ void DatabaseCatalog::initializeAndLoadTemporaryDatabase() attachDatabase(TEMPORARY_DATABASE, db_for_temporary_and_external_tables); } -void DatabaseCatalog::loadDatabases() +void DatabaseCatalog::startupBackgroundCleanup() { if (Context::getGlobalContextInstance()->getApplicationType() == Context::ApplicationType::SERVER && unused_dir_cleanup_period_sec) { diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h index ba3625626da..5526e1496a3 100644 --- a/src/Interpreters/DatabaseCatalog.h +++ b/src/Interpreters/DatabaseCatalog.h @@ -136,7 +136,7 @@ public: static void shutdown(); void initializeAndLoadTemporaryDatabase(); - void loadDatabases(); + void startupBackgroundCleanup(); void loadMarkedAsDroppedTables(); /// Get an object that protects the table from concurrently executing multiple DDL operations. From eae73a1511f47ef5284be44d076cb8a5867cde64 Mon Sep 17 00:00:00 2001 From: flynn Date: Thu, 16 Feb 2023 13:42:19 +0000 Subject: [PATCH 328/717] fix --- src/Storages/IStorageDataLake.h | 1 + src/TableFunctions/ITableFunctionDataLake.h | 7 ++++--- src/TableFunctions/TableFunctionDeltaLake.cpp | 3 +-- src/TableFunctions/TableFunctionHudi.cpp | 2 +- src/TableFunctions/TableFunctionIceberg.cpp | 2 +- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/Storages/IStorageDataLake.h b/src/Storages/IStorageDataLake.h index 4efa63e7be2..3f17596f9b4 100644 --- a/src/Storages/IStorageDataLake.h +++ b/src/Storages/IStorageDataLake.h @@ -20,6 +20,7 @@ template class IStorageDataLake : public IStorage { public: + using Configuration = StorageS3::Configuration; // 1. Parses internal file structure of table // 2. Finds out parts with latest version // 3. Creates url for underlying StorageS3 enigne to handle reads diff --git a/src/TableFunctions/ITableFunctionDataLake.h b/src/TableFunctions/ITableFunctionDataLake.h index f758c99d583..6f8e8dce121 100644 --- a/src/TableFunctions/ITableFunctionDataLake.h +++ b/src/TableFunctions/ITableFunctionDataLake.h @@ -4,11 +4,12 @@ #if USE_AWS_S3 -# include # include # include # include # include +# include +# include namespace DB { @@ -17,7 +18,7 @@ namespace ErrorCodes extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } -template +template class ITableFunctionDataLake : public ITableFunction { public: @@ -76,7 +77,7 @@ protected: configuration.format = "Parquet"; } - mutable Configuration configuration; + mutable typename Storage::Configuration configuration; }; } diff --git a/src/TableFunctions/TableFunctionDeltaLake.cpp b/src/TableFunctions/TableFunctionDeltaLake.cpp index 4b1f8ff62d5..97726025365 100644 --- a/src/TableFunctions/TableFunctionDeltaLake.cpp +++ b/src/TableFunctions/TableFunctionDeltaLake.cpp @@ -16,8 +16,7 @@ struct TableFunctionDeltaLakeName static constexpr auto name = "deltaLake"; }; -using TableFunctionDeltaLake - = ITableFunctionDataLake; +using TableFunctionDeltaLake = ITableFunctionDataLake; void registerTableFunctionDeltaLake(TableFunctionFactory & factory) { diff --git a/src/TableFunctions/TableFunctionHudi.cpp b/src/TableFunctions/TableFunctionHudi.cpp index 633132d2f9b..c8f18df16f7 100644 --- a/src/TableFunctions/TableFunctionHudi.cpp +++ b/src/TableFunctions/TableFunctionHudi.cpp @@ -15,7 +15,7 @@ struct TableFunctionHudiName { static constexpr auto name = "hudi"; }; -using TableFunctionHudi = ITableFunctionDataLake; +using TableFunctionHudi = ITableFunctionDataLake; void registerTableFunctionHudi(TableFunctionFactory & factory) { diff --git a/src/TableFunctions/TableFunctionIceberg.cpp b/src/TableFunctions/TableFunctionIceberg.cpp index e16a2ab04fb..16119b8ea78 100644 --- a/src/TableFunctions/TableFunctionIceberg.cpp +++ b/src/TableFunctions/TableFunctionIceberg.cpp @@ -17,7 +17,7 @@ struct TableFunctionIcebergName static constexpr auto name = "iceberg"; }; -using TableFunctionIceberg = ITableFunctionDataLake; +using TableFunctionIceberg = ITableFunctionDataLake; void registerTableFunctionIceberg(TableFunctionFactory & factory) { From cc4932aa35af9f828d697e8604cea84a194a8d28 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 16 Feb 2023 15:11:34 +0100 Subject: [PATCH 329/717] disable flaky test with cassandra dictionaries --- .../test_cassandra.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_cassandra.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_cassandra.py index 2213623379a..90287e19bd0 100644 --- a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_cassandra.py +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_cassandra.py @@ -2,6 +2,11 @@ import os import math import pytest +# FIXME This test is too flaky +# https://github.com/ClickHouse/ClickHouse/issues/33006 + +pytestmark = pytest.mark.skip + from .common import * from helpers.cluster import ClickHouseCluster From 6763a9d1ff3acee68a7fb2d82dbc6479c0d4fe69 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Thu, 16 Feb 2023 16:15:18 +0100 Subject: [PATCH 330/717] Updated to not clear on_expression from table_join as its used by future analyze runs --- src/Interpreters/TreeRewriter.cpp | 3 --- .../02552_inner_join_with_where_true.reference | 3 +++ .../0_stateless/02552_inner_join_with_where_true.sql | 9 +++++++++ 3 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/02552_inner_join_with_where_true.reference create mode 100644 tests/queries/0_stateless/02552_inner_join_with_where_true.sql diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index f88094ffbc6..025572fd5db 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -655,10 +655,7 @@ void collectJoinedColumns(TableJoin & analyzed_join, ASTTableJoin & table_join, { bool join_on_const_ok = tryJoinOnConst(analyzed_join, table_join.on_expression, context); if (join_on_const_ok) - { - table_join.on_expression = nullptr; return; - } bool is_asof = (table_join.strictness == JoinStrictness::Asof); diff --git a/tests/queries/0_stateless/02552_inner_join_with_where_true.reference b/tests/queries/0_stateless/02552_inner_join_with_where_true.reference new file mode 100644 index 00000000000..55e1eabd3b5 --- /dev/null +++ b/tests/queries/0_stateless/02552_inner_join_with_where_true.reference @@ -0,0 +1,3 @@ +1 2 +1 1 +0 0 diff --git a/tests/queries/0_stateless/02552_inner_join_with_where_true.sql b/tests/queries/0_stateless/02552_inner_join_with_where_true.sql new file mode 100644 index 00000000000..223fafb5319 --- /dev/null +++ b/tests/queries/0_stateless/02552_inner_join_with_where_true.sql @@ -0,0 +1,9 @@ +CREATE TABLE t0 (c0 Int32) ENGINE = Memory; +CREATE TABLE t1 (c1 Int32) ENGINE = Memory; + +INSERT INTO t0(c0) VALUES (1), (2); +INSERT INTO t1(c1) VALUES (1); + +SELECT max(1), count() FROM t0 AS t0 LEFT JOIN t1 ON true WHERE 1; +SELECT max(1), count() FROM t0 AS t0 INNER JOIN t1 ON t0.c0 = t1.c1 WHERE 1; +SELECT max(1), count() FROM t0 AS t0 INNER JOIN t1 ON true WHERE 0; From 75d62ee24af196886b0a025273b4d1f57b673776 Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 16 Feb 2023 16:41:31 +0100 Subject: [PATCH 331/717] Remove support of feature which does not work reliably --- .../materialized-postgresql.md | 15 +- .../MaterializedPostgreSQLConsumer.cpp | 48 ++----- .../MaterializedPostgreSQLConsumer.h | 8 +- .../MaterializedPostgreSQLSettings.h | 1 - .../PostgreSQLReplicationHandler.cpp | 110 +------------- .../PostgreSQL/PostgreSQLReplicationHandler.h | 6 - .../test.py | 134 +++--------------- 7 files changed, 34 insertions(+), 288 deletions(-) diff --git a/docs/en/engines/database-engines/materialized-postgresql.md b/docs/en/engines/database-engines/materialized-postgresql.md index 180e7578441..b43f71a7576 100644 --- a/docs/en/engines/database-engines/materialized-postgresql.md +++ b/docs/en/engines/database-engines/materialized-postgresql.md @@ -8,7 +8,7 @@ sidebar_position: 60 Creates a ClickHouse database with tables from PostgreSQL database. Firstly, database with engine `MaterializedPostgreSQL` creates a snapshot of PostgreSQL database and loads required tables. Required tables can include any subset of tables from any subset of schemas from specified database. Along with the snapshot database engine acquires LSN and once initial dump of tables is performed - it starts pulling updates from WAL. After database is created, newly added tables to PostgreSQL database are not automatically added to replication. They have to be added manually with `ATTACH TABLE db.table` query. -Replication is implemented with PostgreSQL Logical Replication Protocol, which does not allow to replicate DDL, but allows to know whether replication breaking changes happened (column type changes, adding/removing columns). Such changes are detected and according tables stop receiving updates. Such tables can be automatically reloaded in the background in case required setting is turned on (can be used starting from 22.1). Safest way for now is to use `ATTACH`/ `DETACH` queries to reload table completely. If DDL does not break replication (for example, renaming a column) table will still receive updates (insertion is done by position). +Replication is implemented with PostgreSQL Logical Replication Protocol, which does not allow to replicate DDL, but allows to know whether replication breaking changes happened (column type changes, adding/removing columns). Such changes are detected and according tables stop receiving updates. In this case you should use `ATTACH`/ `DETACH` queries to reload table completely. If DDL does not break replication (for example, renaming a column) table will still receive updates (insertion is done by position). :::note This database engine is experimental. To use it, set `allow_experimental_database_materialized_postgresql` to 1 in your configuration files or by using the `SET` command: @@ -165,19 +165,6 @@ Replication of [**TOAST**](https://www.postgresql.org/docs/9.5/storage-toast.htm Default value: empty list. (Default schema is used) -### `materialized_postgresql_allow_automatic_update` {#materialized-postgresql-allow-automatic-update} - - Do not use this setting before 22.1 version. - - Allows reloading table in the background, when schema changes are detected. DDL queries on the PostgreSQL side are not replicated via ClickHouse [MaterializedPostgreSQL](../../engines/database-engines/materialized-postgresql.md) engine, because it is not allowed with PostgreSQL logical replication protocol, but the fact of DDL changes is detected transactionally. In this case, the default behaviour is to stop replicating those tables once DDL is detected. However, if this setting is enabled, then, instead of stopping the replication of those tables, they will be reloaded in the background via database snapshot without data losses and replication will continue for them. - - Possible values: - - - 0 — The table is not automatically updated in the background, when schema changes are detected. - - 1 — The table is automatically updated in the background, when schema changes are detected. - - Default value: `0`. - ### `materialized_postgresql_max_block_size` {#materialized-postgresql-max-block-size} Sets the number of rows collected in memory before flushing data into PostgreSQL database table. diff --git a/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.cpp b/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.cpp index 574b5d76bbe..35222080776 100644 --- a/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.cpp +++ b/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.cpp @@ -31,7 +31,6 @@ MaterializedPostgreSQLConsumer::MaterializedPostgreSQLConsumer( const std::string & start_lsn, const size_t max_block_size_, bool schema_as_a_part_of_table_name_, - bool allow_automatic_update_, StorageInfos storages_info_, const String & name_for_logger) : log(&Poco::Logger::get("PostgreSQLReplicaConsumer(" + name_for_logger + ")")) @@ -43,7 +42,6 @@ MaterializedPostgreSQLConsumer::MaterializedPostgreSQLConsumer( , lsn_value(getLSNValue(start_lsn)) , max_block_size(max_block_size_) , schema_as_a_part_of_table_name(schema_as_a_part_of_table_name_) - , allow_automatic_update(allow_automatic_update_) { final_lsn = start_lsn; auto tx = std::make_shared(connection->getRef()); @@ -657,7 +655,9 @@ bool MaterializedPostgreSQLConsumer::isSyncAllowed(Int32 relation_id, const Stri /// Table is not present in a skip list - allow synchronization. if (skipped_table_with_lsn == skip_list.end()) + { return true; + } const auto & table_start_lsn = skipped_table_with_lsn->second; @@ -686,11 +686,11 @@ void MaterializedPostgreSQLConsumer::markTableAsSkipped(Int32 relation_id, const { skip_list.insert({relation_id, ""}); /// Empty lsn string means - continue waiting for valid lsn. storages.erase(relation_name); - - if (allow_automatic_update) - LOG_TRACE(log, "Table {} (relation_id: {}) is skipped temporarily. It will be reloaded in the background", relation_name, relation_id); - else - LOG_WARNING(log, "Table {} (relation_id: {}) is skipped, because table schema has changed", relation_name, relation_id); + LOG_WARNING( + log, + "Table {} is skipped from replication stream because its structure has changes. " + "Please detach this table and reattach to resume the replication (relation id: {})", + relation_name, relation_id); } @@ -733,13 +733,13 @@ void MaterializedPostgreSQLConsumer::setSetting(const SettingChange & setting) { if (setting.name == "materialized_postgresql_max_block_size") max_block_size = setting.value.safeGet(); - else if (setting.name == "materialized_postgresql_allow_automatic_update") - allow_automatic_update = setting.value.safeGet(); + else + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unsupported setting: {}", setting.name); } /// Read binary changes from replication slot via COPY command (starting from current lsn in a slot). -bool MaterializedPostgreSQLConsumer::readFromReplicationSlot() +bool MaterializedPostgreSQLConsumer::consume() { bool slot_empty = true; @@ -845,32 +845,4 @@ bool MaterializedPostgreSQLConsumer::readFromReplicationSlot() return true; } - -bool MaterializedPostgreSQLConsumer::consume(std::vector> & skipped_tables) -{ - /// Read up to max_block_size changed (approximately - in same cases might be more). - /// false: no data was read, reschedule. - /// true: some data was read, schedule as soon as possible. - auto read_next = readFromReplicationSlot(); - - /// Check if there are tables, which are skipped from being updated by changes from replication stream, - /// because schema changes were detected. Update them, if it is allowed. - if (allow_automatic_update && !skip_list.empty()) - { - for (const auto & [relation_id, lsn] : skip_list) - { - /// Non-empty lsn in this place means that table was already updated, but no changes for that table were - /// received in a previous stream. A table is removed from skip list only when there came - /// changes for table with lsn higher than lsn of snapshot, from which table was reloaded. Since table - /// reaload and reading from replication stream are done in the same thread, no lsn will be skipped - /// between these two events. - if (lsn.empty()) - skipped_tables.emplace_back(std::make_pair(relation_id, relation_id_to_name[relation_id])); - } - } - - return read_next; -} - - } diff --git a/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.h b/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.h index 37caa66aae5..3412e6e422f 100644 --- a/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.h +++ b/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.h @@ -72,11 +72,10 @@ public: const String & start_lsn, size_t max_block_size_, bool schema_as_a_part_of_table_name_, - bool allow_automatic_update_, StorageInfos storages_, const String & name_for_logger); - bool consume(std::vector> & skipped_tables); + bool consume(); /// Called from reloadFromSnapshot by replication handler. This method is needed to move a table back into synchronization /// process if it was skipped due to schema changes. @@ -89,9 +88,6 @@ public: void setSetting(const SettingChange & setting); private: - /// Read approximarely up to max_block_size changes from WAL. - bool readFromReplicationSlot(); - void syncTables(); void updateLsn(); @@ -151,8 +147,6 @@ private: bool schema_as_a_part_of_table_name; - bool allow_automatic_update; - String table_to_insert; /// List of tables which need to be synced after last replication stream. diff --git a/src/Storages/PostgreSQL/MaterializedPostgreSQLSettings.h b/src/Storages/PostgreSQL/MaterializedPostgreSQLSettings.h index 72e9c88c8eb..e8d42ef3668 100644 --- a/src/Storages/PostgreSQL/MaterializedPostgreSQLSettings.h +++ b/src/Storages/PostgreSQL/MaterializedPostgreSQLSettings.h @@ -15,7 +15,6 @@ namespace DB M(UInt64, materialized_postgresql_max_block_size, 65536, "Number of row collected before flushing data into table.", 0) \ M(String, materialized_postgresql_tables_list, "", "List of tables for MaterializedPostgreSQL database engine", 0) \ M(String, materialized_postgresql_schema_list, "", "List of schemas for MaterializedPostgreSQL database engine", 0) \ - M(Bool, materialized_postgresql_allow_automatic_update, false, "Allow to reload table in the background, when schema changes are detected", 0) \ M(String, materialized_postgresql_replication_slot, "", "A user-created replication slot", 0) \ M(String, materialized_postgresql_snapshot, "", "User provided snapshot in case he manages replication slots himself", 0) \ M(String, materialized_postgresql_schema, "", "PostgreSQL schema", 0) \ diff --git a/src/Storages/PostgreSQL/PostgreSQLReplicationHandler.cpp b/src/Storages/PostgreSQL/PostgreSQLReplicationHandler.cpp index 99f6fded669..f9bfe1d174a 100644 --- a/src/Storages/PostgreSQL/PostgreSQLReplicationHandler.cpp +++ b/src/Storages/PostgreSQL/PostgreSQLReplicationHandler.cpp @@ -75,7 +75,6 @@ PostgreSQLReplicationHandler::PostgreSQLReplicationHandler( , current_database_name(current_database_name_) , connection_info(connection_info_) , max_block_size(replication_settings.materialized_postgresql_max_block_size) - , allow_automatic_update(replication_settings.materialized_postgresql_allow_automatic_update) , is_materialized_postgresql_database(is_materialized_postgresql_database_) , tables_list(replication_settings.materialized_postgresql_tables_list) , schema_list(replication_settings.materialized_postgresql_schema_list) @@ -318,7 +317,6 @@ void PostgreSQLReplicationHandler::startSynchronization(bool throw_on_error) start_lsn, max_block_size, schema_as_a_part_of_table_name, - allow_automatic_update, nested_storages, (is_materialized_postgresql_database ? postgres_database : postgres_database + '.' + tables_list)); @@ -419,22 +417,7 @@ void PostgreSQLReplicationHandler::consumerFunc() { assertInitialized(); - std::vector> skipped_tables; - - bool schedule_now = getConsumer()->consume(skipped_tables); - - LOG_DEBUG(log, "checking for skipped tables: {}", skipped_tables.size()); - if (!skipped_tables.empty()) - { - try - { - reloadFromSnapshot(skipped_tables); - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__); - } - } + bool schedule_now = getConsumer()->consume(); if (stop_synchronization) { @@ -941,95 +924,4 @@ void PostgreSQLReplicationHandler::removeTableFromReplication(const String & pos } -void PostgreSQLReplicationHandler::reloadFromSnapshot(const std::vector> & relation_data) -{ - /// If table schema has changed, the table stops consuming changes from replication stream. - /// If `allow_automatic_update` is true, create a new table in the background, load new table schema - /// and all data from scratch. Then execute REPLACE query. - /// This is only allowed for MaterializedPostgreSQL database engine. - try - { - postgres::Connection replication_connection(connection_info, /* replication */true); - auto tx = std::make_shared(replication_connection.getRef()); - - { - String snapshot_name, start_lsn; - if (isReplicationSlotExist(*tx, start_lsn, /* temporary */true)) - dropReplicationSlot(*tx, /* temporary */true); - - TemporaryReplicationSlot temporary_slot(this, tx, start_lsn, snapshot_name); - postgres::Connection tmp_connection(connection_info); - - for (const auto & [relation_id, table_name] : relation_data) - { - auto storage = DatabaseCatalog::instance().getTable(StorageID(current_database_name, table_name), getContext()); - auto * materialized_storage = storage->as (); - auto materialized_table_lock = materialized_storage->lockForShare(String(), getContext()->getSettingsRef().lock_acquire_timeout); - - /// If for some reason this temporary table already exists - also drop it. - auto temp_materialized_storage = materialized_storage->createTemporary(); - - /// This snapshot is valid up to the end of the transaction, which exported it. - auto [temp_nested_storage, table_attributes] = loadFromSnapshot( - tmp_connection, snapshot_name, table_name, temp_materialized_storage->as ()); - - auto table_id = materialized_storage->getNestedStorageID(); - auto temp_table_id = temp_nested_storage->getStorageID(); - - LOG_DEBUG(log, "Starting background update of table {} ({} with {})", - table_name, table_id.getNameForLogs(), temp_table_id.getNameForLogs()); - - auto ast_rename = std::make_shared(); - ASTRenameQuery::Element elem - { - ASTRenameQuery::Table{table_id.database_name, table_id.table_name}, - ASTRenameQuery::Table{temp_table_id.database_name, temp_table_id.table_name} - }; - ast_rename->elements.push_back(std::move(elem)); - ast_rename->exchange = true; - - auto nested_context = materialized_storage->getNestedTableContext(); - - try - { - InterpreterRenameQuery(ast_rename, nested_context).execute(); - - auto nested_storage = DatabaseCatalog::instance().getTable(StorageID(table_id.database_name, table_id.table_name, temp_table_id.uuid), nested_context); - materialized_storage->set(nested_storage); - - auto nested_sample_block = nested_storage->getInMemoryMetadataPtr()->getSampleBlock(); - auto materialized_sample_block = materialized_storage->getInMemoryMetadataPtr()->getSampleBlock(); - assertBlocksHaveEqualStructure(nested_sample_block, materialized_sample_block, "while reloading table in the background"); - - LOG_INFO(log, "Updated table {}. New structure: {}", - nested_storage->getStorageID().getNameForLogs(), nested_sample_block.dumpStructure()); - - /// Pass pointer to new nested table into replication consumer, remove current table from skip list and set start lsn position. - getConsumer()->updateNested(table_name, StorageInfo(nested_storage, std::move(table_attributes)), relation_id, start_lsn); - - auto table_to_drop = DatabaseCatalog::instance().getTable(StorageID(temp_table_id.database_name, temp_table_id.table_name, table_id.uuid), nested_context); - auto drop_table_id = table_to_drop->getStorageID(); - - if (drop_table_id == nested_storage->getStorageID()) - throw Exception(ErrorCodes::LOGICAL_ERROR, - "Cannot drop table because is has the same uuid as new table: {}", drop_table_id.getNameForLogs()); - - LOG_DEBUG(log, "Dropping table {}", drop_table_id.getNameForLogs()); - InterpreterDropQuery::executeDropQuery(ASTDropQuery::Kind::Drop, nested_context, nested_context, drop_table_id, true); - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__); - } - } - } - - tx->commit(); - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__); - } -} - } diff --git a/src/Storages/PostgreSQL/PostgreSQLReplicationHandler.h b/src/Storages/PostgreSQL/PostgreSQLReplicationHandler.h index 89f16457bfe..10a196cf31b 100644 --- a/src/Storages/PostgreSQL/PostgreSQLReplicationHandler.h +++ b/src/Storages/PostgreSQL/PostgreSQLReplicationHandler.h @@ -93,8 +93,6 @@ private: StorageInfo loadFromSnapshot(postgres::Connection & connection, std::string & snapshot_name, const String & table_name, StorageMaterializedPostgreSQL * materialized_storage); - void reloadFromSnapshot(const std::vector> & relation_data); - PostgreSQLTableStructurePtr fetchTableStructure(pqxx::ReplicationTransaction & tx, const String & table_name) const; String doubleQuoteWithSchema(const String & table_name) const; @@ -118,10 +116,6 @@ private: /// max_block_size for replication stream. const size_t max_block_size; - /// Table structure changes are always tracked. By default, table with changed schema will get into a skip list. - /// This setting allows to reloas table in the background. - bool allow_automatic_update = false; - /// To distinguish whether current replication handler belongs to a MaterializedPostgreSQL database engine or single storage. bool is_materialized_postgresql_database; diff --git a/tests/integration/test_postgresql_replica_database_engine_2/test.py b/tests/integration/test_postgresql_replica_database_engine_2/test.py index 666bab29bec..6452803c36e 100644 --- a/tests/integration/test_postgresql_replica_database_engine_2/test.py +++ b/tests/integration/test_postgresql_replica_database_engine_2/test.py @@ -335,7 +335,6 @@ def test_database_with_single_non_default_schema(started_cluster): port=started_cluster.postgres_port, settings=[ f"materialized_postgresql_schema = '{schema_name}'", - "materialized_postgresql_allow_automatic_update = 1", ], ) @@ -353,7 +352,6 @@ def test_database_with_single_non_default_schema(started_cluster): insert_into_tables() check_all_tables_are_synchronized() - print("ALTER") altered_table = random.randint(0, NUM_TABLES - 1) cursor.execute( "ALTER TABLE test_schema.postgresql_replica_{} ADD COLUMN value2 integer".format( @@ -364,24 +362,20 @@ def test_database_with_single_non_default_schema(started_cluster): instance.query( f"INSERT INTO {clickhouse_postgres_db}.postgresql_replica_{altered_table} SELECT number, number, number from numbers(5000, 1000)" ) - assert_number_of_columns(instance, 3, f"postgresql_replica_{altered_table}") + + assert instance.wait_for_log_line(f"Table postgresql_replica_{altered_table} is skipped from replication stream") + instance.query(f"DETACH TABLE test_database.postgresql_replica_{altered_table} PERMANENTLY") + assert not instance.contains_in_log( + "from publication, because table does not exist in PostgreSQL" + ) + instance.query(f"ATTACH TABLE test_database.postgresql_replica_{altered_table}") + check_tables_are_synchronized( instance, f"postgresql_replica_{altered_table}", postgres_database=clickhouse_postgres_db, ) - print("DETACH-ATTACH") - detached_table_name = "postgresql_replica_1" - instance.query(f"DETACH TABLE {materialized_db}.{detached_table_name} PERMANENTLY") - assert not instance.contains_in_log( - "from publication, because table does not exist in PostgreSQL" - ) - instance.query(f"ATTACH TABLE {materialized_db}.{detached_table_name}") - check_tables_are_synchronized( - instance, detached_table_name, postgres_database=clickhouse_postgres_db - ) - def test_database_with_multiple_non_default_schemas_1(started_cluster): cursor = pg_manager.get_db_cursor() @@ -442,7 +436,6 @@ def test_database_with_multiple_non_default_schemas_1(started_cluster): settings=[ f"materialized_postgresql_tables_list = '{publication_tables}'", "materialized_postgresql_tables_list_with_schema=1", - "materialized_postgresql_allow_automatic_update = 1", ], ) @@ -460,7 +453,6 @@ def test_database_with_multiple_non_default_schemas_1(started_cluster): insert_into_tables() check_all_tables_are_synchronized() - print("ALTER") altered_table = random.randint(0, NUM_TABLES - 1) cursor.execute( "ALTER TABLE test_schema.postgresql_replica_{} ADD COLUMN value2 integer".format( @@ -471,33 +463,21 @@ def test_database_with_multiple_non_default_schemas_1(started_cluster): instance.query( f"INSERT INTO {clickhouse_postgres_db}.postgresql_replica_{altered_table} SELECT number, number, number from numbers(5000, 1000)" ) - assert_number_of_columns( - instance, 3, f"{schema_name}.postgresql_replica_{altered_table}" - ) - check_tables_are_synchronized( - instance, - f"postgresql_replica_{altered_table}", - schema_name=schema_name, - postgres_database=clickhouse_postgres_db, - ) - print("DETACH-ATTACH") - detached_table_name = "postgresql_replica_1" - instance.query( - f"DETACH TABLE {materialized_db}.`{schema_name}.{detached_table_name}` PERMANENTLY" - ) + assert instance.wait_for_log_line(f"Table test_schema.postgresql_replica_{altered_table} is skipped from replication stream") + altered_materialized_table = f"{materialized_db}.`test_schema.postgresql_replica_{altered_table}`" + instance.query(f"DETACH TABLE {altered_materialized_table} PERMANENTLY") assert not instance.contains_in_log( "from publication, because table does not exist in PostgreSQL" ) - instance.query( - f"ATTACH TABLE {materialized_db}.`{schema_name}.{detached_table_name}`" - ) + instance.query(f"ATTACH TABLE {altered_materialized_table}") + assert_show_tables( "test_schema.postgresql_replica_0\ntest_schema.postgresql_replica_1\ntest_schema.postgresql_replica_2\ntest_schema.postgresql_replica_3\ntest_schema.postgresql_replica_4\n" ) check_tables_are_synchronized( instance, - detached_table_name, + f"postgresql_replica_{altered_table}", schema_name=schema_name, postgres_database=clickhouse_postgres_db, ) @@ -563,7 +543,6 @@ def test_database_with_multiple_non_default_schemas_2(started_cluster): port=started_cluster.postgres_port, settings=[ f"materialized_postgresql_schema_list = '{schema_list}'", - "materialized_postgresql_allow_automatic_update = 1", ], ) @@ -592,36 +571,23 @@ def test_database_with_multiple_non_default_schemas_2(started_cluster): instance.query( f"INSERT INTO clickhouse_postgres_db{altered_schema}.postgresql_replica_{altered_table} SELECT number, number, number from numbers(1000 * {insert_counter}, 1000)" ) - assert_number_of_columns( - instance, 3, f"schema{altered_schema}.postgresql_replica_{altered_table}" - ) - check_tables_are_synchronized( - instance, - f"postgresql_replica_{altered_table}", - schema_name=f"schema{altered_schema}", - postgres_database=clickhouse_postgres_db, - ) - print("DETACH-ATTACH") - detached_table_name = "postgresql_replica_1" - detached_table_schema = "schema0" - clickhouse_postgres_db = f"clickhouse_postgres_db0" - instance.query( - f"DETACH TABLE {materialized_db}.`{detached_table_schema}.{detached_table_name}` PERMANENTLY" - ) + assert instance.wait_for_log_line(f"Table schema{altered_schema}.postgresql_replica_{altered_table} is skipped from replication stream") + + altered_materialized_table = f"{materialized_db}.`schema{altered_schema}.postgresql_replica_{altered_table}`" + instance.query(f"DETACH TABLE {altered_materialized_table} PERMANENTLY") assert not instance.contains_in_log( "from publication, because table does not exist in PostgreSQL" ) - instance.query( - f"ATTACH TABLE {materialized_db}.`{detached_table_schema}.{detached_table_name}`" - ) + instance.query(f"ATTACH TABLE {altered_materialized_table}") + assert_show_tables( "schema0.postgresql_replica_0\nschema0.postgresql_replica_1\nschema1.postgresql_replica_0\nschema1.postgresql_replica_1\n" ) check_tables_are_synchronized( instance, f"postgresql_replica_{altered_table}", - schema_name=detached_table_schema, + schema_name=f"schema{altered_schema}", postgres_database=clickhouse_postgres_db, ) @@ -656,64 +622,6 @@ def test_table_override(started_cluster): assert_eq_with_retry(instance, query, expected) -def test_table_schema_changes_2(started_cluster): - cursor = pg_manager.get_db_cursor() - table_name = "test_table" - - create_postgres_table(cursor, table_name, template=postgres_table_template_2) - instance.query( - f"INSERT INTO postgres_database.{table_name} SELECT number, number, number, number from numbers(25)" - ) - - pg_manager.create_materialized_db( - ip=started_cluster.postgres_ip, - port=started_cluster.postgres_port, - settings=[ - "materialized_postgresql_allow_automatic_update = 1, materialized_postgresql_tables_list='test_table'" - ], - ) - - instance.query( - f"INSERT INTO postgres_database.{table_name} SELECT number, number, number, number from numbers(25, 25)" - ) - check_tables_are_synchronized(instance, table_name) - - cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN value1") - cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN value2") - cursor.execute(f"ALTER TABLE {table_name} ADD COLUMN value1 Text") - cursor.execute(f"ALTER TABLE {table_name} ADD COLUMN value2 Text") - cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN value3") - cursor.execute(f"ALTER TABLE {table_name} ADD COLUMN value3 Text") - cursor.execute(f"ALTER TABLE {table_name} ADD COLUMN value4 Text") - cursor.execute(f"UPDATE {table_name} SET value3 = 'kek' WHERE key%2=0") - check_tables_are_synchronized(instance, table_name) - instance.query( - f"INSERT INTO postgres_database.{table_name} SELECT number, toString(number), toString(number), toString(number), toString(number) from numbers(50, 25)" - ) - cursor.execute(f"ALTER TABLE {table_name} ADD COLUMN value5 Integer") - cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN value2") - instance.query( - f"INSERT INTO postgres_database.{table_name} SELECT number, toString(number), toString(number), toString(number), number from numbers(75, 25)" - ) - check_tables_are_synchronized(instance, table_name) - instance.restart_clickhouse() - check_tables_are_synchronized(instance, table_name) - cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN value5") - cursor.execute(f"ALTER TABLE {table_name} ADD COLUMN value5 Text") - instance.query( - f"INSERT INTO postgres_database.{table_name} SELECT number, toString(number), toString(number), toString(number), toString(number) from numbers(100, 25)" - ) - check_tables_are_synchronized(instance, table_name) - cursor.execute(f"ALTER TABLE {table_name} ADD COLUMN value6 Text") - cursor.execute(f"ALTER TABLE {table_name} ADD COLUMN value7 Integer") - cursor.execute(f"ALTER TABLE {table_name} ADD COLUMN value8 Integer") - cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN value5") - instance.query( - f"INSERT INTO postgres_database.{table_name} SELECT number, toString(number), toString(number), toString(number), toString(number), number, number from numbers(125, 25)" - ) - check_tables_are_synchronized(instance, table_name) - - if __name__ == "__main__": cluster.start() input("Cluster created, press any key to destroy...") From 7bedfee7ea80e2799319ef063ef315e209bae303 Mon Sep 17 00:00:00 2001 From: Jus <40656180+jus1096@users.noreply.github.com> Date: Thu, 16 Feb 2023 20:53:40 +0400 Subject: [PATCH 332/717] add info for Kafka engine add info for Kafka engine --- .../table-engines/integrations/kafka.md | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/ru/engines/table-engines/integrations/kafka.md b/docs/ru/engines/table-engines/integrations/kafka.md index 07601d047e8..58e03ba30cc 100644 --- a/docs/ru/engines/table-engines/integrations/kafka.md +++ b/docs/ru/engines/table-engines/integrations/kafka.md @@ -31,9 +31,17 @@ SETTINGS [kafka_row_delimiter = 'delimiter_symbol',] [kafka_schema = '',] [kafka_num_consumers = N,] + [kafka_max_block_size = 0,] [kafka_skip_broken_messages = N] [kafka_commit_every_batch = 0,] - [kafka_thread_per_consumer = 0] + [kafka_client_id = '',] + [kafka_poll_timeout_ms = 0,] + [kafka_poll_max_batch_size = 0,] + [kafka_flush_interval_ms = 0,] + [kafka_thread_per_consumer = 0,] + [kafka_handle_error_mode = 'default',] + [kafka_commit_on_select = false,] + [kafka_max_rows_per_message = 1]; ``` Обязательные параметры: @@ -51,7 +59,14 @@ SETTINGS - `kafka_max_block_size` — максимальный размер пачек (в сообщениях) для poll (по умолчанию `max_block_size`). - `kafka_skip_broken_messages` — максимальное количество некорректных сообщений в блоке. Если `kafka_skip_broken_messages = N`, то движок отбрасывает `N` сообщений Кафки, которые не получилось обработать. Одно сообщение в точности соответствует одной записи (строке). Значение по умолчанию – 0. - `kafka_commit_every_batch` — включает или отключает режим записи каждой принятой и обработанной пачки по отдельности вместо единой записи целого блока (по умолчанию `0`). +- `kafka_client_id` — идентификатор клиента. Значение по умолчанию пусто – ''. +- `kafka_poll_timeout_ms` - Таймаут для poll. По умолчанию: (../../../operations/settings/settings.md#stream_poll_timeout_ms) +- `kafka_poll_max_batch_size` - Максимальное количество сообщений в одном poll Kafka. По умолчанию: (../../../operations/settings/settings.md#setting-max_block_size) +- `kafka_flush_interval_ms` - Таймаут для сброса данных из Kafka. По умолчанию: (../../../operations/settings/settings.md#stream-flush-interval-ms) - `kafka_thread_per_consumer` — включает или отключает предоставление отдельного потока каждому потребителю (по умолчанию `0`). При включенном режиме каждый потребитель сбрасывает данные независимо и параллельно, при отключённом — строки с данными от нескольких потребителей собираются в один блок. +- `kafka_handle_error_mode` - Способ обработки ошибок для Kafka. Возможные значения: default, stream. +- `kafka_commit_on_select` - Сообщение о commit при запросе select. По умолчанию: `false`. +- `kafka_max_rows_per_message` - Максимальное количество строк записанных в одно сообщение Kafka для формата row-based. По умолчанию: `1`. Примеры @@ -188,7 +203,10 @@ ClickHouse может поддерживать учетные данные Kerbe - `_key` — ключ сообщения. - `_offset` — оффсет сообщения. - `_timestamp` — временная метка сообщения. +- `_timestamp_ms` — временная метка сообщения в миллисекундах. - `_partition` — секция топика Kafka. +- `_headers.name` - Массив ключей заголовков сообщений. +- `_headers.value` - Массив значений заголовков сообщений. **Смотрите также** From df4abf4d879b3279a1e702601269d1d7ef1cb885 Mon Sep 17 00:00:00 2001 From: Jus <40656180+jus1096@users.noreply.github.com> Date: Thu, 16 Feb 2023 20:53:51 +0400 Subject: [PATCH 333/717] add settings add settings --- docs/ru/operations/settings/settings.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index b44d0d276a7..4025966ac21 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -1315,6 +1315,12 @@ SELECT type, query FROM system.query_log WHERE log_comment = 'log_comment test' Чем меньше значение, тем чаще данные сбрасываются в таблицу. Установка слишком низкого значения приводит к снижению производительности. +## stream_poll_timeout_ms {#stream_poll_timeout_ms} + +Таймаут для poll стримнга данных. + +Значение по умолчанию: 500. + ## load_balancing {#settings-load_balancing} Задает алгоритм выбора реплик, используемый при обработке распределенных запросов. From 417052e4b2bac02e9808078de5eb158174552838 Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 16 Feb 2023 17:22:29 +0100 Subject: [PATCH 334/717] Fix remaining tests --- .../MaterializedPostgreSQLConsumer.cpp | 1 + .../test.py | 35 ++++--------------- .../configs/log_conf.xml | 2 -- .../test.py | 34 ++++++++++-------- 4 files changed, 27 insertions(+), 45 deletions(-) diff --git a/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.cpp b/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.cpp index 35222080776..9c6eeceb605 100644 --- a/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.cpp +++ b/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.cpp @@ -21,6 +21,7 @@ namespace ErrorCodes { extern const int LOGICAL_ERROR; extern const int POSTGRESQL_REPLICATION_INTERNAL_ERROR; + extern const int BAD_ARGUMENTS; } MaterializedPostgreSQLConsumer::MaterializedPostgreSQLConsumer( diff --git a/tests/integration/test_postgresql_replica_database_engine_1/test.py b/tests/integration/test_postgresql_replica_database_engine_1/test.py index 0b3fb68708a..f7d33143a82 100644 --- a/tests/integration/test_postgresql_replica_database_engine_1/test.py +++ b/tests/integration/test_postgresql_replica_database_engine_1/test.py @@ -389,7 +389,6 @@ def test_table_schema_changes(started_cluster): pg_manager.create_materialized_db( ip=started_cluster.postgres_ip, port=started_cluster.postgres_port, - settings=["materialized_postgresql_allow_automatic_update = 1"], ) for i in range(NUM_TABLES): @@ -407,37 +406,16 @@ def test_table_schema_changes(started_cluster): altered_idx = random.randint(0, 4) altered_table = f"postgresql_replica_{altered_idx}" - cursor.execute(f"ALTER TABLE {altered_table} DROP COLUMN value2") + prev_count = int(instance.query(f"SELECT count() FROM test_database.{altered_table}")) + cursor.execute(f"ALTER TABLE {altered_table} DROP COLUMN value2") for i in range(NUM_TABLES): cursor.execute(f"INSERT INTO postgresql_replica_{i} VALUES (50, {i}, {i})") - cursor.execute(f"UPDATE {altered_table} SET value3 = 12 WHERE key%2=0") - time.sleep(2) - assert_nested_table_is_created(instance, altered_table) - assert_number_of_columns(instance, 3, altered_table) - check_tables_are_synchronized(instance, altered_table) - print("check1 OK") - - check_several_tables_are_synchronized(instance, NUM_TABLES) - - for i in range(NUM_TABLES): - if i != altered_idx: - instance.query( - "INSERT INTO postgres_database.postgresql_replica_{} SELECT 51 + number, {}, {}, {} from numbers(49)".format( - i, i, i, i - ) - ) - else: - instance.query( - "INSERT INTO postgres_database.postgresql_replica_{} SELECT 51 + number, {}, {} from numbers(49)".format( - i, i, i - ) - ) - - check_tables_are_synchronized(instance, altered_table) - print("check2 OK") - check_several_tables_are_synchronized(instance, NUM_TABLES) + assert instance.wait_for_log_line( + f"Table postgresql_replica_{altered_idx} is skipped from replication stream" + ) + assert prev_count == int(instance.query(f"SELECT count() FROM test_database.{altered_table}")) def test_many_concurrent_queries(started_cluster): @@ -585,7 +563,6 @@ def test_virtual_columns(started_cluster): pg_manager.create_materialized_db( ip=started_cluster.postgres_ip, port=started_cluster.postgres_port, - settings=["materialized_postgresql_allow_automatic_update = 1"], ) assert_nested_table_is_created(instance, table_name) diff --git a/tests/integration/test_postgresql_replica_database_engine_2/configs/log_conf.xml b/tests/integration/test_postgresql_replica_database_engine_2/configs/log_conf.xml index c42a3aba833..6cc1128e130 100644 --- a/tests/integration/test_postgresql_replica_database_engine_2/configs/log_conf.xml +++ b/tests/integration/test_postgresql_replica_database_engine_2/configs/log_conf.xml @@ -15,7 +15,6 @@ postgres1 5432 postgres_database - test_table
postgres @@ -23,7 +22,6 @@ postgres1 1111 postgres_database - test_table
 diff --git a/tests/integration/test_postgresql_replica_database_engine_2/test.py b/tests/integration/test_postgresql_replica_database_engine_2/test.py index 6452803c36e..68c7cb96b71 100644 --- a/tests/integration/test_postgresql_replica_database_engine_2/test.py +++ b/tests/integration/test_postgresql_replica_database_engine_2/test.py @@ -92,10 +92,7 @@ def test_add_new_table_to_replication(started_cluster): result[:63] == "CREATE DATABASE test_database\\nENGINE = MaterializedPostgreSQL(" ) # Check without ip - assert ( - result[-59:] - == "\\'postgres_database\\', \\'postgres\\', \\'mysecretpassword\\')\n" - ) + assert result[-51:] == "\\'postgres_database\\', \\'postgres\\', \\'[HIDDEN]\\')\n" result = instance.query_and_get_error( "ALTER DATABASE test_database MODIFY SETTING materialized_postgresql_tables_list='tabl1'" @@ -201,10 +198,7 @@ def test_remove_table_from_replication(started_cluster): result[:63] == "CREATE DATABASE test_database\\nENGINE = MaterializedPostgreSQL(" ) - assert ( - result[-59:] - == "\\'postgres_database\\', \\'postgres\\', \\'mysecretpassword\\')\n" - ) + assert result[-51:] == "\\'postgres_database\\', \\'postgres\\', \\'[HIDDEN]\\')\n" table_name = "postgresql_replica_4" instance.query(f"DETACH TABLE test_database.{table_name} PERMANENTLY") @@ -363,8 +357,12 @@ def test_database_with_single_non_default_schema(started_cluster): f"INSERT INTO {clickhouse_postgres_db}.postgresql_replica_{altered_table} SELECT number, number, number from numbers(5000, 1000)" ) - assert instance.wait_for_log_line(f"Table postgresql_replica_{altered_table} is skipped from replication stream") - instance.query(f"DETACH TABLE test_database.postgresql_replica_{altered_table} PERMANENTLY") + assert instance.wait_for_log_line( + f"Table postgresql_replica_{altered_table} is skipped from replication stream" + ) + instance.query( + f"DETACH TABLE test_database.postgresql_replica_{altered_table} PERMANENTLY" + ) assert not instance.contains_in_log( "from publication, because table does not exist in PostgreSQL" ) @@ -464,8 +462,12 @@ def test_database_with_multiple_non_default_schemas_1(started_cluster): f"INSERT INTO {clickhouse_postgres_db}.postgresql_replica_{altered_table} SELECT number, number, number from numbers(5000, 1000)" ) - assert instance.wait_for_log_line(f"Table test_schema.postgresql_replica_{altered_table} is skipped from replication stream") - altered_materialized_table = f"{materialized_db}.`test_schema.postgresql_replica_{altered_table}`" + assert instance.wait_for_log_line( + f"Table test_schema.postgresql_replica_{altered_table} is skipped from replication stream" + ) + altered_materialized_table = ( + f"{materialized_db}.`test_schema.postgresql_replica_{altered_table}`" + ) instance.query(f"DETACH TABLE {altered_materialized_table} PERMANENTLY") assert not instance.contains_in_log( "from publication, because table does not exist in PostgreSQL" @@ -572,9 +574,13 @@ def test_database_with_multiple_non_default_schemas_2(started_cluster): f"INSERT INTO clickhouse_postgres_db{altered_schema}.postgresql_replica_{altered_table} SELECT number, number, number from numbers(1000 * {insert_counter}, 1000)" ) - assert instance.wait_for_log_line(f"Table schema{altered_schema}.postgresql_replica_{altered_table} is skipped from replication stream") + assert instance.wait_for_log_line( + f"Table schema{altered_schema}.postgresql_replica_{altered_table} is skipped from replication stream" + ) - altered_materialized_table = f"{materialized_db}.`schema{altered_schema}.postgresql_replica_{altered_table}`" + altered_materialized_table = ( + f"{materialized_db}.`schema{altered_schema}.postgresql_replica_{altered_table}`" + ) instance.query(f"DETACH TABLE {altered_materialized_table} PERMANENTLY") assert not instance.contains_in_log( "from publication, because table does not exist in PostgreSQL" From 9cca571777e8a0659d4c463fdc19862682b5d660 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 16 Feb 2023 17:58:35 +0000 Subject: [PATCH 335/717] Fix constants in the result of MergingSortedAlgorithm. --- src/Processors/Merges/Algorithms/MergedData.h | 14 ++++++++++- ...ing_constants_and_distinct_crash.reference | 4 +++ ...2_sorting_constants_and_distinct_crash.sql | 25 +++++++++++++++++++ 3 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/02662_sorting_constants_and_distinct_crash.reference create mode 100644 tests/queries/0_stateless/02662_sorting_constants_and_distinct_crash.sql diff --git a/src/Processors/Merges/Algorithms/MergedData.h b/src/Processors/Merges/Algorithms/MergedData.h index 24b83013aee..f4ef0b77c53 100644 --- a/src/Processors/Merges/Algorithms/MergedData.h +++ b/src/Processors/Merges/Algorithms/MergedData.h @@ -60,7 +60,19 @@ public: throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot insert to MergedData from Chunk because MergedData is not empty."); UInt64 num_rows = chunk.getNumRows(); - columns = chunk.mutateColumns(); + UInt64 num_columns = chunk.getNumColumns(); + auto chunk_columns = chunk.mutateColumns(); + + /// Here is a special code for constant columns. + /// Currently, 'columns' will contain constants, but 'chunk_columns' will not. + /// We want to keep constants in the result, so just re-create them carefully. + for (size_t i = 0; i < num_columns; ++i) + { + if (isColumnConst(*columns[i])) + columns[i] = columns[i]->cloneResized(num_rows); + else + columns[i] = std::move(chunk_columns[i]); + } if (rows_size < num_rows) { diff --git a/tests/queries/0_stateless/02662_sorting_constants_and_distinct_crash.reference b/tests/queries/0_stateless/02662_sorting_constants_and_distinct_crash.reference new file mode 100644 index 00000000000..bd95f23c026 --- /dev/null +++ b/tests/queries/0_stateless/02662_sorting_constants_and_distinct_crash.reference @@ -0,0 +1,4 @@ +constant_1 250000 +constant_1 2000000 +constant_1 test_value_1 +constant_1 test_value_2 diff --git a/tests/queries/0_stateless/02662_sorting_constants_and_distinct_crash.sql b/tests/queries/0_stateless/02662_sorting_constants_and_distinct_crash.sql new file mode 100644 index 00000000000..31df0bf378b --- /dev/null +++ b/tests/queries/0_stateless/02662_sorting_constants_and_distinct_crash.sql @@ -0,0 +1,25 @@ +drop table if exists test_table; +CREATE TABLE test_table (string_value String) ENGINE = MergeTree ORDER BY string_value; +insert into test_table select * from ( + select 'test_value_1' + from numbers_mt(250000) + union all + select 'test_value_2' + from numbers_mt(2000000) +) +order by rand(); + +select distinct + 'constant_1' as constant_value, + count(*) over(partition by constant_value, string_value) as value_cnt +from ( + select string_value + from test_table +); + +select distinct + 'constant_1' as constant_value, * + from (select string_value from test_table) + ORDER BY constant_value, string_value settings max_threads=1; + +drop table test_table; From 23d173a53caba8a0bbb535a3beaba88f6fe4aa87 Mon Sep 17 00:00:00 2001 From: Joanna Hulboj Date: Fri, 10 Feb 2023 18:30:51 +0000 Subject: [PATCH 336/717] Refactor FunctionArrayMapped to allow for a fixed number of extra positional arguments --- src/Functions/array/FunctionArrayMapped.h | 156 +++++++++++++++------- 1 file changed, 109 insertions(+), 47 deletions(-) diff --git a/src/Functions/array/FunctionArrayMapped.h b/src/Functions/array/FunctionArrayMapped.h index 89599edd9d1..0bbde77e40d 100644 --- a/src/Functions/array/FunctionArrayMapped.h +++ b/src/Functions/array/FunctionArrayMapped.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -75,6 +76,10 @@ const IColumn::Offsets & getOffsets(const T & column) * arrayMap(x1,...,xn -> expression, array1,...,arrayn) - apply the expression to each element of the array (or set of parallel arrays). * arrayFilter(x -> predicate, array) - leave in the array only the elements for which the expression is true. * + * It is possible for the functions to require fixed number of positional arguments: + * arrayPartialSort(limit, arr) + * arrayPartialSort(x -> predicate, limit, arr) + * * For some functions arrayCount, arrayExists, arrayAll, an overload of the form f(array) is available, * which works in the same way as f(x -> x, array). * @@ -88,12 +93,13 @@ public: static constexpr bool is_argument_type_map = std::is_same_v; static constexpr bool is_argument_type_array = std::is_same_v; static constexpr auto argument_type_name = is_argument_type_map ? "Map" : "Array"; + + static constexpr bool has_num_fixed_params = requires(const Impl &) { Impl::num_fixed_params; }; + static constexpr size_t num_fixed_params = []{ if constexpr (has_num_fixed_params) return Impl::num_fixed_params; else return 0; }(); + static FunctionPtr create(ContextPtr) { return std::make_shared(); } - String getName() const override - { - return name; - } + String getName() const override { return name; } bool isVariadic() const override { return true; } size_t getNumberOfArguments() const override { return 0; } @@ -104,30 +110,41 @@ public: void getLambdaArgumentTypes(DataTypes & arguments) const override { if (arguments.empty()) - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, - "Function {} needs at least one argument, passed {}", getName(), arguments.size()); + throw Exception( + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Function {} needs at least one argument, passed {}", + getName(), + arguments.size()); - if (arguments.size() == 1) - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, - "Function {} needs at least one argument with data", getName()); + if (arguments.size() < 1 + num_fixed_params) + throw Exception( + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Function {} needs at least {} argument{} with data", + getName(), + num_fixed_params + 1, + (num_fixed_params + 1 == 1) ? "" : "s"); - if (arguments.size() > 2 && Impl::needOneArray()) - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, - "Function {} needs one argument with data", getName()); + if (arguments.size() > 2 + num_fixed_params && Impl::needOneArray()) + throw Exception( + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Function {} needs {} argument{} with data", + getName(), + num_fixed_params + 1, + (num_fixed_params + 1 == 1) ? "" : "s"); - size_t nested_types_count = is_argument_type_map ? (arguments.size() - 1) * 2 : (arguments.size() - 1); + size_t nested_types_count = (arguments.size() - num_fixed_params - 1) * (is_argument_type_map ? 2 : 1); DataTypes nested_types(nested_types_count); - for (size_t i = 0; i < arguments.size() - 1; ++i) + for (size_t i = 0; i < arguments.size() - 1 - num_fixed_params; ++i) { - const auto * array_type = checkAndGetDataType(&*arguments[i + 1]); + const auto * array_type = checkAndGetDataType(&*arguments[i + 1 + num_fixed_params]); if (!array_type) throw Exception( ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Argument {} of function {} must be {}. Found {} instead", - toString(i + 2), + i + 2 + num_fixed_params, getName(), argument_type_name, - arguments[i + 1]->getName()); + arguments[i + 1 + num_fixed_params]->getName()); if constexpr (is_argument_type_map) { nested_types[2 * i] = recursiveRemoveLowCardinality(array_type->getKeyType()); @@ -144,32 +161,54 @@ public: throw Exception( ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "First argument for this overload of {} must be a function with {} arguments, found {} instead", - getName(), nested_types.size(), arguments[0]->getName()); + getName(), + nested_types.size(), + arguments[0]->getName()); arguments[0] = std::make_shared(nested_types); } DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { - size_t min_args = Impl::needExpression() ? 2 : 1; + size_t min_args = (num_fixed_params + Impl::needExpression()) ? 2 : 1; if (arguments.size() < min_args) - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, - "Function {} needs at least {} argument, passed {}", - getName(), min_args, arguments.size()); + throw Exception( + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Function {} needs at least {} argument{}, passed {}", + getName(), + min_args, + (min_args > 1 ? "s" : ""), + arguments.size()); - if ((arguments.size() == 1) && is_argument_type_array) + if ((arguments.size() == 1 + num_fixed_params) && is_argument_type_array) { - const auto * data_type = checkAndGetDataType(arguments[0].type.get()); + const auto * data_type = checkAndGetDataType(arguments[num_fixed_params].type.get()); if (!data_type) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "The only argument for function {} must be array. " - "Found {} instead", getName(), arguments[0].type->getName()); + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "The {}{}{} argument for function {} must be array. Found {} instead", + num_fixed_params + 1, + getOrdinalSuffix(num_fixed_params + 1), + (num_fixed_params == 0 ? " and only" : ""), + getName(), + arguments[num_fixed_params].type->getName()); + + if constexpr (num_fixed_params) + Impl::checkArguments( + std::span(std::begin(arguments), num_fixed_params), getName()); DataTypePtr nested_type = data_type->getNestedType(); if (Impl::needBoolean() && !isUInt8(nested_type)) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "The only argument for function {} must be array of UInt8. " - "Found {} instead", getName(), arguments[0].type->getName()); + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "The {}{}{} argument for function {} must be array of UInt8. Found {} instead", + num_fixed_params + 1, + getOrdinalSuffix(num_fixed_params + 1), + (num_fixed_params == 0 ? " and only" : ""), + getName(), + arguments[num_fixed_params].type->getName()); if constexpr (is_argument_type_array) return Impl::getReturnType(nested_type, nested_type); @@ -178,17 +217,22 @@ public: } else { - if (arguments.size() > 2 && Impl::needOneArray()) + if (arguments.size() > 2 + num_fixed_params && Impl::needOneArray()) throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} needs one argument with data", getName()); const auto * data_type_function = checkAndGetDataType(arguments[0].type.get()); if (!data_type_function) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "First argument for function {} must be a function. Actual {}", getName(), arguments[0].type->getName()); + if constexpr (num_fixed_params) + Impl::checkArguments( + std::span(std::begin(arguments) + 1, num_fixed_params), getName()); + /// The types of the remaining arguments are already checked in getLambdaArgumentTypes. DataTypePtr return_type = removeLowCardinality(data_type_function->getReturnType()); @@ -199,21 +243,24 @@ public: /// - lambda may return Nothing or Nullable(Nothing) because of default implementation of functions /// for these types. In this case we will just create UInt8 const column full of 0. if (Impl::needBoolean() && !isUInt8(removeNullable(return_type)) && !isNothing(removeNullable(return_type))) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "Expression for function {} must return UInt8 or Nullable(UInt8), found {}", - getName(), return_type->getName()); + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Expression for function {} must return UInt8 or Nullable(UInt8), found {}", + getName(), + return_type->getName()); static_assert(is_argument_type_map || is_argument_type_array, "unsupported type"); - if (arguments.size() < 2) + if (arguments.size() < 2 + num_fixed_params) { throw DB::Exception(ErrorCodes::LOGICAL_ERROR, "Incorrect number of arguments: {}", arguments.size()); } - const auto * first_array_type = checkAndGetDataType(arguments[1].type.get()); + const auto * first_array_type = checkAndGetDataType(arguments[1 + num_fixed_params].type.get()); if (!first_array_type) - throw DB::Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Unsupported type {}", arguments[1].type->getName()); + throw DB::Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Unsupported type {}", arguments[1 + num_fixed_params].type->getName()); if constexpr (is_argument_type_array) return Impl::getReturnType(return_type, first_array_type->getNestedType()); @@ -227,9 +274,9 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { - if (arguments.size() == 1) + if (arguments.size() == 1 + num_fixed_params) { - ColumnPtr column_array_ptr = arguments[0].column; + ColumnPtr column_array_ptr = arguments[num_fixed_params].column; const auto * column_array = checkAndGetColumn(column_array_ptr.get()); if (!column_array) @@ -237,21 +284,30 @@ public: const ColumnConst * column_const_array = checkAndGetColumnConst(column_array_ptr.get()); if (!column_const_array) throw Exception( - ErrorCodes::ILLEGAL_COLUMN, - "Expected {} column, found {}", - argument_type_name, - column_array_ptr->getName()); + ErrorCodes::ILLEGAL_COLUMN, "Expected {} column, found {}", argument_type_name, column_array_ptr->getName()); column_array_ptr = column_const_array->convertToFullColumn(); column_array = assert_cast(column_array_ptr.get()); } if constexpr (std::is_same_v) { - return Impl::execute(*column_array, column_array->getNestedColumn().getDataPtr()); + if constexpr (num_fixed_params) + return Impl::execute( + *column_array, + column_array->getNestedColumn().getDataPtr(), + std::span(std::begin(arguments), num_fixed_params)); + else + return Impl::execute(*column_array, column_array->getNestedColumn().getDataPtr()); } else { - return Impl::execute(*column_array, column_array->getDataPtr()); + if constexpr (num_fixed_params) + return Impl::execute( + *column_array, + column_array->getDataPtr(), + std::span(std::begin(arguments), num_fixed_params)); + else + return Impl::execute(*column_array, column_array->getDataPtr()); } } else @@ -274,7 +330,7 @@ public: ColumnsWithTypeAndName arrays; arrays.reserve(arguments.size() - 1); - for (size_t i = 1; i < arguments.size(); ++i) + for (size_t i = 1 + num_fixed_params; i < arguments.size(); ++i) { const auto & array_with_type_and_name = arguments[i]; @@ -314,7 +370,7 @@ public: getName()); } - if (i == 1) + if (i == 1 + num_fixed_params) { column_first_array_ptr = column_array_ptr; column_first_array = column_array; @@ -380,7 +436,13 @@ public: } } - return Impl::execute(*column_first_array, lambda_result.column); + if constexpr (num_fixed_params) + return Impl::execute( + *column_first_array, + lambda_result.column, + std::span(std::begin(arguments) + 1, num_fixed_params)); + else + return Impl::execute(*column_first_array, lambda_result.column); } } }; From 7bb7ea69130a04365ad8bccdb2de9e3591cea210 Mon Sep 17 00:00:00 2001 From: Joanna Hulboj Date: Fri, 10 Feb 2023 18:31:37 +0000 Subject: [PATCH 337/717] Added arrayPartialSort and arraySort functions --- src/Functions/array/arraySort.cpp | 130 ++++++++++++++++-- .../0_stateless/00390_array_sort.reference | 38 +++++ .../queries/0_stateless/00390_array_sort.sql | 32 +++++ 3 files changed, 190 insertions(+), 10 deletions(-) diff --git a/src/Functions/array/arraySort.cpp b/src/Functions/array/arraySort.cpp index 34c2881a413..3743e8e7f21 100644 --- a/src/Functions/array/arraySort.cpp +++ b/src/Functions/array/arraySort.cpp @@ -1,19 +1,21 @@ #include "FunctionArrayMapped.h" -#include #include +#include namespace DB { /** Sort arrays, by values of its elements, or by values of corresponding elements of calculated expression (known as "schwartzsort"). */ -template +template struct ArraySortImpl { using column_type = ColumnArray; using data_type = DataTypeArray; + static constexpr auto num_fixed_params = is_partial; + static bool needBoolean() { return false; } static bool needExpression() { return false; } static bool needOneArray() { return false; } @@ -27,7 +29,7 @@ struct ArraySortImpl { const IColumn & column; - explicit Less(const IColumn & column_) : column(column_) {} + explicit Less(const IColumn & column_) : column(column_) { } bool operator()(size_t lhs, size_t rhs) const { @@ -38,8 +40,35 @@ struct ArraySortImpl } }; - static ColumnPtr execute(const ColumnArray & array, ColumnPtr mapped) + static void checkArguments(std::span arguments, const String & name) + requires(num_fixed_params) { + if (arguments.size() != 1) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} needs limit argument", name); + + WhichDataType which(arguments[0].type.get()); + if (!which.isUInt() && !which.isInt()) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of limit argument of function {} (must be UInt or Int)", + arguments[0].column->getName(), + name); + } + + static ColumnPtr execute( + const ColumnArray & array, + ColumnPtr mapped, + std::span arguments [[maybe_unused]] = {}) + { + [[maybe_unused]] const auto limit = [&]() -> size_t + { + if constexpr (is_partial) + { + return arguments[0].column.get()->getUInt(0); + } + return 0; + }(); + const ColumnArray::Offsets & offsets = array.getOffsets(); size_t size = offsets.size(); @@ -53,7 +82,18 @@ struct ArraySortImpl for (size_t i = 0; i < size; ++i) { auto next_offset = offsets[i]; - ::sort(&permutation[current_offset], &permutation[next_offset], Less(*mapped)); + if constexpr (is_partial) + { + if (limit) + { + const auto effective_limit = std::min(limit, next_offset - current_offset); + ::partial_sort(&permutation[current_offset], &permutation[current_offset + effective_limit], &permutation[next_offset], Less(*mapped)); + } + else + ::sort(&permutation[current_offset], &permutation[next_offset], Less(*mapped)); + } + else + ::sort(&permutation[current_offset], &permutation[next_offset], Less(*mapped)); current_offset = next_offset; } @@ -61,17 +101,87 @@ struct ArraySortImpl } }; -struct NameArraySort { static constexpr auto name = "arraySort"; }; -struct NameArrayReverseSort { static constexpr auto name = "arrayReverseSort"; }; +struct NameArraySort +{ + static constexpr auto name = "arraySort"; +}; +struct NameArrayReverseSort +{ + static constexpr auto name = "arrayReverseSort"; +}; +struct NameArrayPartialSort +{ + static constexpr auto name = "arrayPartialSort"; +}; +struct NameArrayPartialReverseSort +{ + static constexpr auto name = "arrayPartialReverseSort"; +}; -using FunctionArraySort = FunctionArrayMapped, NameArraySort>; -using FunctionArrayReverseSort = FunctionArrayMapped, NameArrayReverseSort>; +using FunctionArraySort = FunctionArrayMapped, NameArraySort>; +using FunctionArrayReverseSort = FunctionArrayMapped, NameArrayReverseSort>; +using FunctionArrayPartialSort = FunctionArrayMapped, NameArrayPartialSort>; +using FunctionArrayPartialReverseSort = FunctionArrayMapped, NameArrayPartialReverseSort>; REGISTER_FUNCTION(ArraySort) { factory.registerFunction(); factory.registerFunction(); + + factory.registerFunction({ + R"( +Returns an array of the same size as the original array where elements in range `[1..limit]` +are sorted in ascending order. Remaining elements `(limit..N]` shall contain elements in unspecified order. +[example:simple_int] +[example:simple_string] + +If one wishes to retain only the sorted elements it can be achieved with: +[example:retain_sorted] + +If the `func` function is specified, sorting order is determined by the result of the `func` +function applied to the elements of the array. +[example:lambda_simple] + +If `func` accepts multiple arguments, the `arrayPartialSort` function is passed several arrays +that the arguments of `func` will correspond to. +[example:lambda_complex] + +For more details see documentation of `arraySort`. +)", + Documentation::Examples{ + {"simple_int", "SELECT arrayPartialSort(2, [5, 9, 1, 3])"}, + {"simple_string", "SELECT arrayPartialSort(2, ['expenses','lasso','embolism','gladly'])"}, + {"retain_sorted", "SELECT arrayResize(arrayPartialSort(2, [5, 9, 1, 3]), 2)"}, + {"lambda_simple", "SELECT arrayPartialSort((x) -> -x, 2, [5, 9, 1, 3])"}, + {"lambda_complex", "SELECT arrayPartialSort((x, y) -> -y, 1, [0, 1, 2], [1, 2, 3]) as res"}}, + Documentation::Categories{"Array"}}); + factory.registerFunction({ + R"( +Returns an array of the same size as the original array where elements in range `[1..limit]` +are sorted in descending order. Remaining elements `(limit..N]` shall contain elements in unspecified order. +[example:simple_int] +[example:simple_string] + +If one wishes to retain only the sorted elements it can be achieved with: +[example:retain_sorted] + +If the `func` function is specified, sorting order is determined by the result of the `func` +function applied to the elements of the array. +[example:lambda_simple] + +If `func` accepts multiple arguments, the `arrayPartialSort` function is passed several arrays +that the arguments of `func` will correspond to. +[example:lambda_complex] + +For more details see documentation of `arraySort`. +)", + Documentation::Examples{ + {"simple_int", "SELECT arrayPartialReverseSort(2, [5, 9, 1, 3])"}, + {"simple_string", "SELECT arrayPartialReverseSort(2, ['expenses','lasso','embolism','gladly'])"}, + {"retain_sorted", "SELECT arrayResize(arrayPartialReverseSort(2, [5, 9, 1, 3]), 2)"}, + {"lambda_simple", "SELECT arrayPartialReverseSort((x) -> -x, 2, [5, 9, 1, 3])"}, + {"lambda_complex", "SELECT arrayPartialReverseSort((x, y) -> -y, 1, [0, 1, 2], [1, 2, 3]) as res"}}, + Documentation::Categories{"Array"}}); } } - diff --git a/tests/queries/0_stateless/00390_array_sort.reference b/tests/queries/0_stateless/00390_array_sort.reference index 732ab75930a..bfc4de067b6 100644 --- a/tests/queries/0_stateless/00390_array_sort.reference +++ b/tests/queries/0_stateless/00390_array_sort.reference @@ -25,3 +25,41 @@ [7,6,5,4,3,2,1,0] [] ['world','hello'] +[9,4,8,10,5,2,3,7,1,6] 4 [10,9,8,7] [10,9,8,7] +[9,4,8,10,5,2,3,7,1,6] 4 [10,9,8,7] [10,9,8,7] +['9','4','8','10','5','2','3','7','1','6'] 4 ['9','8','7','6'] ['10','1','2','3'] +['9','4','8','10','5','2','3','7','1','6'] 4 ['9','8','7','6'] ['10','1','2','3'] +[[0,1,2,3],[0],[0,1],[0,1,2]] 2 [[0],[0,1]] [[0,1,2,3],[0,1,2]] [[0,1,2,3],[0,1,2]] +[[0,1,2,3],[0],[0,1],[0,1,2]] 2 [[0],[0,1]] [[0,1,2,3],[0,1,2]] [[0,1,2,3],[0,1,2]] +['476118317','873','1381',''] 3 ['','1381','476118317'] ['','873','1381'] +['1','577349846663553','72'] 3 ['1','577349846663553','72'] ['1','72','577349846663553'] +['181981357172','4167749'] 3 ['181981357172','4167749',''] ['4167749','181981357172',''] +['962446486456','415994'] 3 ['415994','962446486456',''] ['415994','962446486456',''] +['77667','936175','7','26','8'] 3 ['26','7','77667'] ['7','8','26'] +['152285784','9','6979435',''] 3 ['','152285784','6979435'] ['','9','6979435'] +['12742','4333384','853','32'] 3 ['12742','32','4333384'] ['32','853','12742'] +['1336581123286','26','488'] 3 ['1336581123286','26','488'] ['26','488','1336581123286'] +['117','','','3455837413562',''] 3 ['','',''] ['','',''] +['441693','539393268817'] 3 ['441693','539393268817',''] ['441693','539393268817',''] +[] +[] +[1,0] +[] +[3,2] +[] +[5,4] +[] +[7,6] +[] +['wrongly','ignore','relocate'] +2 [1,2] +2 ['1','10'] +2 [10,9] +2 ['9','8'] +4 [-inf,-4,1,2] +4 [inf,3,2,1] +10 [-inf,-4,1,2,3,inf,nan,nan,NULL,NULL] +10 [-inf,-4,1,2,3,inf,nan,nan,NULL,NULL] +3 [[1,2],[-10,-20],[10,20],[0,0],[-1.5,1]] [[-10,-20],[-1.5,1],[0,0]] [[10,20],[1,2],[0,0]] [[-10,-20],[-1.5,1],[0,0]] +0 [NULL,9,4,8,10,5,2,3,7,1,6] [1,2,3,4,5,6,7,8,9,10,NULL] [10,9,8,7,6,5,4,3,2,1,NULL] [10,9,8,7,6,5,4,3,2,1,NULL] +10 [NULL,9,4,8,10,5,2,3,7,1,6] [1,2,3,4,5,6,7,8,9,10,NULL] [10,9,8,7,6,5,4,3,2,1,NULL] [10,9,8,7,6,5,4,3,2,1,NULL] diff --git a/tests/queries/0_stateless/00390_array_sort.sql b/tests/queries/0_stateless/00390_array_sort.sql index e3417ca331c..b568c819dfa 100644 --- a/tests/queries/0_stateless/00390_array_sort.sql +++ b/tests/queries/0_stateless/00390_array_sort.sql @@ -12,3 +12,35 @@ SELECT splitByChar('0', toString(intHash64(number))) AS arr, arraySort(arr) AS s SELECT arrayReverseSort(number % 2 ? emptyArrayUInt64() : range(number)) FROM system.numbers LIMIT 10; SELECT arraySort((x, y) -> y, ['hello', 'world'], [2, 1]); + +-- Using arrayResize to trim the unsorted bit of the array that is normally left in unspecified order +SELECT [9,4,8,10,5,2,3,7,1,6] AS arr, 4 AS lim, arrayResize(arrayPartialReverseSort(lim, arr), lim), arrayResize(arrayPartialSort(x -> -x, lim, arr), lim); +SELECT materialize([9,4,8,10,5,2,3,7,1,6]) AS arr, 4 AS lim, arrayResize(arrayPartialReverseSort(lim, arr), lim), arrayResize(arrayPartialSort(x -> -x, lim, arr), lim); + +SELECT arrayMap(x -> toString(x), [9,4,8,10,5,2,3,7,1,6]) AS arr, 4 AS lim, arrayResize(arrayPartialReverseSort(lim, arr), lim), arrayResize(arrayPartialSort(x -> reverse(x), lim, arr), lim); +SELECT arrayMap(x -> toString(x), materialize([9,4,8,10,5,2,3,7,1,6])) AS arr, 4 AS lim, arrayResize(arrayPartialReverseSort(lim, arr), lim), arrayResize(arrayPartialSort(x -> reverse(x), lim, arr), lim); + +SELECT arrayMap(x -> range(x), [4,1,2,3]) AS arr, 2 AS lim, arrayResize(arrayPartialSort(lim, arr), lim), arrayResize(arrayPartialReverseSort(lim, arr), lim), arrayResize(arrayPartialSort(x -> -length(x), lim, arr), lim); +SELECT arrayMap(x -> range(x), materialize([4,1,2,3])) AS arr, 2 AS lim, arrayResize(arrayPartialSort(lim, arr), lim), arrayResize(arrayPartialReverseSort(lim, arr), lim), arrayResize(arrayPartialSort(x -> -length(x), lim, arr), lim); + +SELECT splitByChar('0', toString(intHash64(number))) AS arr, 3 AS lim, arrayResize(arrayPartialSort(lim, arr), lim) AS sorted, arrayResize(arrayPartialSort(x -> toUInt64OrZero(x), lim, arr), lim) AS sorted_nums FROM system.numbers LIMIT 10; + +SELECT res FROM (SELECT arrayPartialReverseSort(2, number % 2 ? emptyArrayUInt64() : range(number)) AS arr, arrayResize(arr, if(empty(arr), 0, 2)) AS res FROM system.numbers LIMIT 10); + +SELECT arrayResize(arrayPartialSort((x, y) -> y, 3, ['directly','ignore','wrongly','relocate','upright'], [4,2,1,3,5]), 3); + +SELECT 2 as nelems, arrayResize(arrayPartialSort(nelems, [NULL,9,4,8,10,5,2,3,7,1,6]), nelems); +SELECT 2 as nelems, arrayResize(arrayPartialSort(nelems, [NULL,'9','4','8','10','5','2','3','7','1','6']), nelems); +SELECT 2 as nelems, arrayResize(arrayPartialReverseSort(nelems, [NULL,9,4,8,10,5,2,3,7,1,6]), nelems); +SELECT 2 as nelems, arrayResize(arrayPartialReverseSort(nelems, [NULL,'9','4','8','10','5','2','3','7','1','6']), nelems); + + +SELECT 4 as nelems, arrayResize(arrayPartialSort(nelems, [1, nan, 2, NULL, 3, nan, -4, NULL, inf, -inf]), nelems); +SELECT 4 as nelems, arrayResize(arrayPartialSort((x) -> -x, nelems, [1, nan, 2, NULL, 3, nan, -4, NULL, inf, -inf]), nelems); +SELECT 10 as nelems, arrayResize(arrayPartialSort(nelems, [1, nan, 2, NULL, 3, nan, -4, NULL, inf, -inf]), nelems); +SELECT 10 as nelems, arrayResize(arrayPartialSort(nelems, [1, nan, 2, NULL, 3, nan, -4, NULL, inf, -inf]), nelems); + +SELECT 3 as nelems, [[1,2],[-10,-20],[10,20],[0,0],[-1.5,1]] as arr, arrayResize(arrayPartialSort(nelems, arr), nelems), arrayResize(arrayPartialReverseSort(nelems, arr), nelems), arrayResize(arrayPartialSort((x) -> arraySum(x), nelems, arr), nelems); + +SELECT 0 as nelems, [NULL,9,4,8,10,5,2,3,7,1,6] AS arr, arrayPartialSort(nelems, arr), arrayPartialReverseSort(nelems, arr), arrayPartialSort((x) -> -x, nelems, arr); +SELECT 10 as nelems, [NULL,9,4,8,10,5,2,3,7,1,6] AS arr, arrayPartialSort(nelems, arr), arrayPartialReverseSort(nelems, arr), arrayPartialSort((x) -> -x, nelems, arr); From 4ee665b158d579f8c44f5ff90cfdd3b8d65e4877 Mon Sep 17 00:00:00 2001 From: Joanna Hulboj Date: Sat, 11 Feb 2023 11:38:31 +0000 Subject: [PATCH 338/717] FIXUP: Missing ErrorCodes --- src/Functions/array/arraySort.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Functions/array/arraySort.cpp b/src/Functions/array/arraySort.cpp index 3743e8e7f21..f2bab58144e 100644 --- a/src/Functions/array/arraySort.cpp +++ b/src/Functions/array/arraySort.cpp @@ -6,6 +6,12 @@ namespace DB { +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + /** Sort arrays, by values of its elements, or by values of corresponding elements of calculated expression (known as "schwartzsort"). */ template From 0900fd683234691b1dfa41aa26fcebf542566956 Mon Sep 17 00:00:00 2001 From: Joanna Hulboj Date: Sat, 11 Feb 2023 21:56:30 +0000 Subject: [PATCH 339/717] FIXUP: Add arrayPartialSort and arrayPartialReverseSort to fuzzer corpus --- tests/fuzz/all.dict | 2 ++ tests/fuzz/dictionaries/functions.dict | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tests/fuzz/all.dict b/tests/fuzz/all.dict index 17ef7d2ab1e..8ded66cbd0d 100644 --- a/tests/fuzz/all.dict +++ b/tests/fuzz/all.dict @@ -72,7 +72,9 @@ "arrayMap" "arrayMax" "arrayMin" +"arrayPartialReverseSort" "arrayPartialShuffle" +"arrayPartialSort" "arrayPopBack" "arrayPopFront" "arrayProduct" diff --git a/tests/fuzz/dictionaries/functions.dict b/tests/fuzz/dictionaries/functions.dict index 877b2679846..dfadb645f3f 100644 --- a/tests/fuzz/dictionaries/functions.dict +++ b/tests/fuzz/dictionaries/functions.dict @@ -334,6 +334,8 @@ "wordShingleSimHash" "arrayCumSum" "arraySort" +"arrayPartialSort" +"arrayPartialReverseSort" "dumpColumnStructure" "multiSearchFirstIndex" "arrayReverseSplit" From 221e4942b232a9215ea64b57587bad5bae08537e Mon Sep 17 00:00:00 2001 From: Joanna Hulboj Date: Sun, 12 Feb 2023 10:58:59 +0000 Subject: [PATCH 340/717] FIXUP: check in FunctionArrayMapped refactor --- src/Functions/array/FunctionArrayMapped.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/array/FunctionArrayMapped.h b/src/Functions/array/FunctionArrayMapped.h index 0bbde77e40d..7c8ac2d855d 100644 --- a/src/Functions/array/FunctionArrayMapped.h +++ b/src/Functions/array/FunctionArrayMapped.h @@ -116,7 +116,7 @@ public: getName(), arguments.size()); - if (arguments.size() < 1 + num_fixed_params) + if (arguments.size() <= 1 + num_fixed_params) throw Exception( ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} needs at least {} argument{} with data", From 5e32e20abb5322341b29f211cb1bc8e2391235a5 Mon Sep 17 00:00:00 2001 From: Joanna Hulboj Date: Mon, 13 Feb 2023 08:25:53 +0000 Subject: [PATCH 341/717] FIXUP: arrayPartialSort arg check and few tests --- src/Functions/array/arraySort.cpp | 2 +- tests/queries/0_stateless/00390_array_sort.sql | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Functions/array/arraySort.cpp b/src/Functions/array/arraySort.cpp index f2bab58144e..202a2adaa5b 100644 --- a/src/Functions/array/arraySort.cpp +++ b/src/Functions/array/arraySort.cpp @@ -57,7 +57,7 @@ struct ArraySortImpl throw Exception( ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of limit argument of function {} (must be UInt or Int)", - arguments[0].column->getName(), + arguments[0].type->getName(), name); } diff --git a/tests/queries/0_stateless/00390_array_sort.sql b/tests/queries/0_stateless/00390_array_sort.sql index b568c819dfa..5959df39c58 100644 --- a/tests/queries/0_stateless/00390_array_sort.sql +++ b/tests/queries/0_stateless/00390_array_sort.sql @@ -44,3 +44,8 @@ SELECT 3 as nelems, [[1,2],[-10,-20],[10,20],[0,0],[-1.5,1]] as arr, arrayResize SELECT 0 as nelems, [NULL,9,4,8,10,5,2,3,7,1,6] AS arr, arrayPartialSort(nelems, arr), arrayPartialReverseSort(nelems, arr), arrayPartialSort((x) -> -x, nelems, arr); SELECT 10 as nelems, [NULL,9,4,8,10,5,2,3,7,1,6] AS arr, arrayPartialSort(nelems, arr), arrayPartialReverseSort(nelems, arr), arrayPartialSort((x) -> -x, nelems, arr); + + +SELECT arrayPartialSort(arraySort([1,2,3]), [1,2,3]); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT arrayMap(x -> range(x), [4, 1, 2, 3]) AS arr, 100 AS lim, arrayResize(arrayPartialSort(arrayPartialSort(lim, arr), arr), lim), arrayResize(arrayPartialReverseSort(lim, arr), lim), arrayResize(arrayPartialSort(x -> (-length(x)), lim, arr), lim); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT arrayPartialReverseSort(arraySort((x, y) -> y, [NULL, NULL], [NULL, NULL]), arr), arrayMap(x -> toString(x), [257, -9223372036854775807, 2, -2147483648, 2147483648, NULL, 65536, -2147483648, 2, 65535]) AS arr, NULL, 100 AS lim, 65536, arrayResize(arrayPartialSort(x -> reverse(x), lim, arr), lim) GROUP BY [NULL, 1023, -2, NULL, 255, '0', NULL, 9223372036854775806] WITH ROLLUP; -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } \ No newline at end of file From 9b1e48c06e8975f7fd3a0c4a059f955367dcabba Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Thu, 16 Feb 2023 19:24:56 +0100 Subject: [PATCH 342/717] Some more corner cases --- .../02665_regexp_extract.reference | 2 + .../0_stateless/02665_regexp_extract.sql | 85 ++++++++++--------- 2 files changed, 47 insertions(+), 40 deletions(-) diff --git a/tests/queries/0_stateless/02665_regexp_extract.reference b/tests/queries/0_stateless/02665_regexp_extract.reference index 812ed7ac8f5..42a6b4a5c9c 100644 --- a/tests/queries/0_stateless/02665_regexp_extract.reference +++ b/tests/queries/0_stateless/02665_regexp_extract.reference @@ -1,6 +1,7 @@ 100 100 200 +100-200 100 \N @@ -28,3 +29,4 @@ \N \N \N +100-200 diff --git a/tests/queries/0_stateless/02665_regexp_extract.sql b/tests/queries/0_stateless/02665_regexp_extract.sql index 9966bda50f4..3fdf132e48a 100644 --- a/tests/queries/0_stateless/02665_regexp_extract.sql +++ b/tests/queries/0_stateless/02665_regexp_extract.sql @@ -1,40 +1,45 @@ -select regexpExtract('100-200', '(\\d+)-(\\d+)', 1); -select regexpExtract('100-200', '(\\d+)-(\\d+)'); -select regexpExtract('100-200', '(\\d+)-(\\d+)', 2); -select regexpExtract('100-200', '(\\d+).*', 1); -select regexpExtract('100-200', '([a-z])', 1); -select regexpExtract(null, '([a-z])', 1); -select regexpExtract('100-200', null, 1); -select regexpExtract('100-200', '([a-z])', null); - -select regexpExtract(materialize('100-200'), '(\\d+)-(\\d+)'); -select regexpExtract(materialize('100-200'), '(\\d+)-(\\d+)', 1); -select regexpExtract(materialize('100-200'), '(\\d+)-(\\d+)', 2); -select regexpExtract(materialize('100-200'), '(\\d+).*', 1); -select regexpExtract(materialize('100-200'), '([a-z])', 1); -select regexpExtract(materialize(null), '([a-z])', 1); -select regexpExtract(materialize('100-200'), null, 1); -select regexpExtract(materialize('100-200'), '([a-z])', null); - -select regexpExtract('100-200', '(\\d+)-(\\d+)', materialize(1)); -select regexpExtract('100-200', '(\\d+)-(\\d+)', materialize(2)); -select regexpExtract('100-200', '(\\d+).*', materialize(1)); -select regexpExtract('100-200', '([a-z])', materialize(1)); -select regexpExtract(null, '([a-z])', materialize(1)); -select regexpExtract('100-200', null, materialize(1)); -select regexpExtract('100-200', '([a-z])', materialize(null)); - -select regexpExtract(materialize('100-200'), '(\\d+)-(\\d+)', materialize(1)); -select regexpExtract(materialize('100-200'), '(\\d+)-(\\d+)', materialize(2)); -select regexpExtract(materialize('100-200'), '(\\d+).*', materialize(1)); -select regexpExtract(materialize('100-200'), '([a-z])', materialize(1)); -select regexpExtract(materialize(null), '([a-z])', materialize(1)); -select regexpExtract(materialize('100-200'), null, materialize(1)); -select regexpExtract(materialize('100-200'), '([a-z])', materialize(null)); - - -select regexpExtract('100-200'); -- { serverError 42 } -select regexpExtract('100-200', '(\\d+)-(\\d+)', 1, 2); -- { serverError 42 } -select regexpExtract(cast('100-200' as FixedString(10)), '(\\d+)-(\\d+)', 1); -- { serverError 43 } -select regexpExtract('100-200', cast('(\\d+)-(\\d+)' as FixedString(20)), 1); -- { serverError 43 } -select regexpExtract('100-200', materialize('(\\d+)-(\\d+)'), 1); -- { serverError 44 } +select regexpExtract('100-200', '(\\d+)-(\\d+)', 1); +select regexpExtract('100-200', '(\\d+)-(\\d+)'); +select regexpExtract('100-200', '(\\d+)-(\\d+)', 2); +select regexpExtract('100-200', '(\\d+)-(\\d+)', 0); +select regexpExtract('100-200', '(\\d+).*', 1); +select regexpExtract('100-200', '([a-z])', 1); +select regexpExtract(null, '([a-z])', 1); +select regexpExtract('100-200', null, 1); +select regexpExtract('100-200', '([a-z])', null); + +select regexpExtract(materialize('100-200'), '(\\d+)-(\\d+)'); +select regexpExtract(materialize('100-200'), '(\\d+)-(\\d+)', 1); +select regexpExtract(materialize('100-200'), '(\\d+)-(\\d+)', 2); +select regexpExtract(materialize('100-200'), '(\\d+).*', 1); +select regexpExtract(materialize('100-200'), '([a-z])', 1); +select regexpExtract(materialize(null), '([a-z])', 1); +select regexpExtract(materialize('100-200'), null, 1); +select regexpExtract(materialize('100-200'), '([a-z])', null); + +select regexpExtract('100-200', '(\\d+)-(\\d+)', materialize(1)); +select regexpExtract('100-200', '(\\d+)-(\\d+)', materialize(2)); +select regexpExtract('100-200', '(\\d+).*', materialize(1)); +select regexpExtract('100-200', '([a-z])', materialize(1)); +select regexpExtract(null, '([a-z])', materialize(1)); +select regexpExtract('100-200', null, materialize(1)); +select regexpExtract('100-200', '([a-z])', materialize(null)); + +select regexpExtract(materialize('100-200'), '(\\d+)-(\\d+)', materialize(1)); +select regexpExtract(materialize('100-200'), '(\\d+)-(\\d+)', materialize(2)); +select regexpExtract(materialize('100-200'), '(\\d+).*', materialize(1)); +select regexpExtract(materialize('100-200'), '([a-z])', materialize(1)); +select regexpExtract(materialize(null), '([a-z])', materialize(1)); +select regexpExtract(materialize('100-200'), null, materialize(1)); +select regexpExtract(materialize('100-200'), '([a-z])', materialize(null)); + + +select regexpExtract('100-200'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +select regexpExtract('100-200', '(\\d+)-(\\d+)', 1, 2); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +select regexpExtract(cast('100-200' as FixedString(10)), '(\\d+)-(\\d+)', 1); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +select regexpExtract('100-200', cast('(\\d+)-(\\d+)' as FixedString(20)), 1); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +select regexpExtract('100-200', materialize('(\\d+)-(\\d+)'), 1); -- { serverError ILLEGAL_COLUMN } +select regexpExtract('100-200', '(\\d+)-(\\d+)', 3); -- { serverError INDEX_OF_POSITIONAL_ARGUMENT_IS_OUT_OF_RANGE } +select regexpExtract('100-200', '(\\d+)-(\\d+)', -1); -- { serverError INDEX_OF_POSITIONAL_ARGUMENT_IS_OUT_OF_RANGE } +select regexpExtract('100-200', '\\d+-\\d+', 0); +select regexpExtract('100-200', '\\d+-\\d+', 1);-- { serverError INDEX_OF_POSITIONAL_ARGUMENT_IS_OUT_OF_RANGE } From 7b4219e23a62148300ddbc44664e1f813b48c398 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Thu, 16 Feb 2023 19:26:10 +0100 Subject: [PATCH 343/717] Added documentation field --- src/Functions/regexpExtract.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Functions/regexpExtract.cpp b/src/Functions/regexpExtract.cpp index bcf7cd98be3..72aa8c834b7 100644 --- a/src/Functions/regexpExtract.cpp +++ b/src/Functions/regexpExtract.cpp @@ -8,6 +8,7 @@ #include #include #include +#include namespace DB { @@ -266,7 +267,8 @@ namespace REGISTER_FUNCTION(RegexpExtract) { - factory.registerFunction(); + factory.registerFunction( + Documentation{"Extracts the first string in haystack that matches the regexp pattern and corresponds to the regex group index."}); /// For Spark compatibility. factory.registerAlias("REGEXP_EXTRACT", "regexpExtract", FunctionFactory::CaseInsensitive); From e0cc1b9ae408f45c2cf009f40581f4aa0c1c2eb0 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Thu, 16 Feb 2023 19:29:26 +0100 Subject: [PATCH 344/717] Fix assertion when index argument is not passed --- src/Functions/regexpExtract.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/Functions/regexpExtract.cpp b/src/Functions/regexpExtract.cpp index 72aa8c834b7..3cbf8398735 100644 --- a/src/Functions/regexpExtract.cpp +++ b/src/Functions/regexpExtract.cpp @@ -78,18 +78,17 @@ namespace const ColumnPtr column_index = arguments.size() > 2 ? arguments[2].column : nullptr; /// Check if the second argument is const column - const ColumnConst * col_pattern = typeid_cast(&*column_pattern); + const ColumnConst * col_pattern = typeid_cast(column_pattern.get()); if (!col_pattern) throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Second argument of function {} must be constant string", getName()); - /// Check if the first argument is string column(const or not) - const ColumnConst * col_const = typeid_cast(&*column); + const ColumnConst * col_const = typeid_cast(column.get()); const ColumnString * col = nullptr; if (col_const) col = typeid_cast(&col_const->getDataColumn()); else - col = typeid_cast(&*column); + col = typeid_cast(column.get()); if (!col) throw Exception( ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of argument of function {}", arguments[0].column->getName(), getName()); @@ -102,7 +101,7 @@ namespace constantVector(col_const->getValue(), col_pattern->getValue(), column_index, vec_res, offsets_res); else if (!column_index || isColumnConst(*column_index)) { - const auto * col_const_index = typeid_cast(&*column_index); + const auto * col_const_index = typeid_cast(column_index.get()); ssize_t index = !col_const_index ? 1 : col_const_index->getInt(0); vectorConstant(col->getChars(), col->getOffsets(), col_pattern->getValue(), index, vec_res, offsets_res); } From 11965e59e2036c17ace6fc7d349000fb778798f4 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Thu, 16 Feb 2023 19:46:24 +0100 Subject: [PATCH 345/717] Analyzer Planner disable by default --- src/Core/Settings.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 25eb1d4a75a..a23fdb6bf14 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -306,7 +306,7 @@ class IColumn; M(Float, opentelemetry_start_trace_probability, 0., "Probability to start an OpenTelemetry trace for an incoming query.", 0) \ M(Bool, opentelemetry_trace_processors, false, "Collect OpenTelemetry spans for processors.", 0) \ M(Bool, prefer_column_name_to_alias, false, "Prefer using column names instead of aliases if possible.", 0) \ - M(Bool, allow_experimental_analyzer, true, "Allow experimental analyzer", 0) \ + M(Bool, allow_experimental_analyzer, false, "Allow experimental analyzer", 0) \ M(Bool, prefer_global_in_and_join, false, "If enabled, all IN/JOIN operators will be rewritten as GLOBAL IN/JOIN. It's useful when the to-be-joined tables are only available on the initiator and we need to always scatter their data on-the-fly during distributed processing with the GLOBAL keyword. It's also useful to reduce the need to access the external sources joining external tables.", 0) \ \ \ From ba6b7b161064c5ccd99fcd3c4e4516eb4d678dab Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 16 Feb 2023 16:01:25 -0300 Subject: [PATCH 346/717] fix test_alias_column --- src/Storages/MergeTree/MergeTreeData.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 69401587e41..fe352c59ead 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -6427,9 +6427,9 @@ std::optional MergeTreeData::getQueryProcessingStageWithAgg // Projections don't support grouping sets yet. if (original_select_query->group_by_with_grouping_sets - || select_query->group_by_with_totals - || select_query->group_by_with_rollup - || select_query->group_by_with_cube) + || original_select_query->group_by_with_totals + || original_select_query->group_by_with_rollup + || original_select_query->group_by_with_cube) return std::nullopt; auto query_options = SelectQueryOptions( @@ -6439,7 +6439,7 @@ std::optional MergeTreeData::getQueryProcessingStageWithAgg ).ignoreProjections().ignoreAlias(); InterpreterSelectQuery select( - query_ptr, + original_query_ptr, query_context, query_options, query_info.prepared_sets); From 3a635e428a48b93d4b99f52a263f8ad6a58d71e0 Mon Sep 17 00:00:00 2001 From: HarryLeeIBM Date: Thu, 16 Feb 2023 11:03:41 -0800 Subject: [PATCH 347/717] Fix xxhash endian issue for s390x --- src/Functions/FunctionsHashing.h | 106 ++++++++++++++++++++++++++----- 1 file changed, 91 insertions(+), 15 deletions(-) diff --git a/src/Functions/FunctionsHashing.h b/src/Functions/FunctionsHashing.h index 69c3a299eea..6bf1a2db3ac 100644 --- a/src/Functions/FunctionsHashing.h +++ b/src/Functions/FunctionsHashing.h @@ -55,7 +55,7 @@ #include #include #include - +#include namespace DB { @@ -1025,17 +1025,58 @@ private: if constexpr (Impl::use_int_hash_for_pods) { - if constexpr (std::is_same_v) - h = IntHash64Impl::apply(bit_cast(vec_from[i])); + if constexpr (std::endian::native == std::endian::little) + { + if constexpr (std::is_same_v) + h = IntHash64Impl::apply(bit_cast(vec_from[i])); + else + h = IntHash32Impl::apply(bit_cast(vec_from[i])); + } else - h = IntHash32Impl::apply(bit_cast(vec_from[i])); + { + if constexpr (std::is_same_v) + { + UInt64 v = bit_cast(vec_from[i]); + v = __builtin_bswap64(v); + h = IntHash64Impl::apply(v); + } + else + { + UInt32 v = bit_cast(vec_from[i]); + v = __builtin_bswap32(v); + h = IntHash32Impl::apply(v); + } + } } else { - if (std::is_same_v) - h = JavaHashImpl::apply(vec_from[i]); + if constexpr (std::endian::native == std::endian::little) + { + if (std::is_same_v) + h = JavaHashImpl::apply(vec_from[i]); + else + h = apply(key, reinterpret_cast(&vec_from[i]), sizeof(vec_from[i])); + } else - h = apply(key, reinterpret_cast(&vec_from[i]), sizeof(vec_from[i])); + { + if (std::is_same_v) + h = JavaHashImpl::apply(vec_from[i]); + else + { + if constexpr (std::is_same_v) + { + UInt64 v = bit_cast(vec_from[i]); + v = __builtin_bswap64(v); + h = apply(key, reinterpret_cast(&v), sizeof(vec_from[i])); + } + else + { + UInt32 v = bit_cast(vec_from[i]); + v = __builtin_bswap32(v); + h = apply(key, reinterpret_cast(&v), sizeof(vec_from[i])); + } + } + } } if constexpr (first) @@ -1048,11 +1089,28 @@ private: { auto value = col_from_const->template getValue(); ToType hash; - if constexpr (std::is_same_v) - hash = IntHash64Impl::apply(bit_cast(value)); + if constexpr (std::endian::native == std::endian::little) + { + if constexpr (std::is_same_v) + hash = IntHash64Impl::apply(bit_cast(value)); + else + hash = IntHash32Impl::apply(bit_cast(value)); + } else - hash = IntHash32Impl::apply(bit_cast(value)); - + { + if constexpr (std::is_same_v) + { + UInt64 v = bit_cast(value); + v = __builtin_bswap64(v); + hash = IntHash64Impl::apply(v); + } + else + { + UInt32 v = bit_cast(value); + v = __builtin_bswap32(v); + hash = IntHash32Impl::apply(bit_cast(v)); + } + } size_t size = vec_to.size(); if constexpr (first) { @@ -1080,8 +1138,17 @@ private: size_t size = vec_from.size(); for (size_t i = 0; i < size; ++i) { - ToType h = apply(key, reinterpret_cast(&vec_from[i]), sizeof(vec_from[i])); - + ToType h; + if constexpr (std::endian::native == std::endian::little) + { + h = apply(key, reinterpret_cast(&vec_from[i]), sizeof(vec_from[i])); + } + else + { + char tmp_buffer[sizeof(vec_from[i])]; + reverseMemcpy(tmp_buffer, &vec_from[i], sizeof(vec_from[i])); + h = apply(key, reinterpret_cast(tmp_buffer), sizeof(vec_from[i])); + } if constexpr (first) vec_to[i] = h; else @@ -1092,8 +1159,17 @@ private: { auto value = col_from_const->template getValue(); - ToType h = apply(key, reinterpret_cast(&value), sizeof(value)); - + ToType h; + if constexpr (std::endian::native == std::endian::little) + { + h = apply(key, reinterpret_cast(&value), sizeof(value)); + } + else + { + char tmp_buffer[sizeof(value)]; + reverseMemcpy(tmp_buffer, &value, sizeof(value)); + h = apply(key, reinterpret_cast(tmp_buffer), sizeof(value)); + } size_t size = vec_to.size(); if constexpr (first) { From d4795ed34b01ff2da67d000e906e4d10be25923d Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 16 Feb 2023 21:16:19 +0100 Subject: [PATCH 348/717] Update 02662_sorting_constants_and_distinct_crash.sql --- .../0_stateless/02662_sorting_constants_and_distinct_crash.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/02662_sorting_constants_and_distinct_crash.sql b/tests/queries/0_stateless/02662_sorting_constants_and_distinct_crash.sql index 31df0bf378b..9b117773b9b 100644 --- a/tests/queries/0_stateless/02662_sorting_constants_and_distinct_crash.sql +++ b/tests/queries/0_stateless/02662_sorting_constants_and_distinct_crash.sql @@ -1,5 +1,6 @@ drop table if exists test_table; CREATE TABLE test_table (string_value String) ENGINE = MergeTree ORDER BY string_value; +system stop merges test_table; insert into test_table select * from ( select 'test_value_1' from numbers_mt(250000) @@ -22,4 +23,5 @@ select distinct from (select string_value from test_table) ORDER BY constant_value, string_value settings max_threads=1; +system start merges test_table; drop table test_table; From 8747e23dbb1705bb60342087dff5e27cc8fe67e2 Mon Sep 17 00:00:00 2001 From: Joanna Hulboj Date: Thu, 16 Feb 2023 17:46:20 +0000 Subject: [PATCH 349/717] FIXUP: PR comments --- src/Functions/array/FunctionArrayMapped.h | 5 ++--- src/Functions/array/arraySort.cpp | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/Functions/array/FunctionArrayMapped.h b/src/Functions/array/FunctionArrayMapped.h index 7c8ac2d855d..42b7ae69a40 100644 --- a/src/Functions/array/FunctionArrayMapped.h +++ b/src/Functions/array/FunctionArrayMapped.h @@ -94,8 +94,7 @@ public: static constexpr bool is_argument_type_array = std::is_same_v; static constexpr auto argument_type_name = is_argument_type_map ? "Map" : "Array"; - static constexpr bool has_num_fixed_params = requires(const Impl &) { Impl::num_fixed_params; }; - static constexpr size_t num_fixed_params = []{ if constexpr (has_num_fixed_params) return Impl::num_fixed_params; else return 0; }(); + static constexpr size_t num_fixed_params = []{ if constexpr (requires { Impl::num_fixed_params; }) return Impl::num_fixed_params; else return 0; }(); static FunctionPtr create(ContextPtr) { return std::make_shared(); } @@ -170,7 +169,7 @@ public: DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { - size_t min_args = (num_fixed_params + Impl::needExpression()) ? 2 : 1; + size_t min_args = (Impl::needExpression() ? 2 : 1) + num_fixed_params ; if (arguments.size() < min_args) throw Exception( ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, diff --git a/src/Functions/array/arraySort.cpp b/src/Functions/array/arraySort.cpp index 202a2adaa5b..916923b1f01 100644 --- a/src/Functions/array/arraySort.cpp +++ b/src/Functions/array/arraySort.cpp @@ -141,7 +141,7 @@ are sorted in ascending order. Remaining elements `(limit..N]` shall contain ele [example:simple_int] [example:simple_string] -If one wishes to retain only the sorted elements it can be achieved with: +To retain only the sorted elements use `arrayResize`: [example:retain_sorted] If the `func` function is specified, sorting order is determined by the result of the `func` @@ -168,7 +168,7 @@ are sorted in descending order. Remaining elements `(limit..N]` shall contain el [example:simple_int] [example:simple_string] -If one wishes to retain only the sorted elements it can be achieved with: +To retain only the sorted elements use `arrayResize`: [example:retain_sorted] If the `func` function is specified, sorting order is determined by the result of the `func` From 9798d9cbca637a401d2522afe388e662f81a918b Mon Sep 17 00:00:00 2001 From: Joanna Hulboj Date: Thu, 16 Feb 2023 20:17:32 +0000 Subject: [PATCH 350/717] FIXUP: Few more tests --- tests/queries/0_stateless/00390_array_sort.sql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/queries/0_stateless/00390_array_sort.sql b/tests/queries/0_stateless/00390_array_sort.sql index 5959df39c58..563dfb4ed1b 100644 --- a/tests/queries/0_stateless/00390_array_sort.sql +++ b/tests/queries/0_stateless/00390_array_sort.sql @@ -46,6 +46,9 @@ SELECT 0 as nelems, [NULL,9,4,8,10,5,2,3,7,1,6] AS arr, arrayPartialSort(nelems, SELECT 10 as nelems, [NULL,9,4,8,10,5,2,3,7,1,6] AS arr, arrayPartialSort(nelems, arr), arrayPartialReverseSort(nelems, arr), arrayPartialSort((x) -> -x, nelems, arr); +SELECT arrayPartialSort([1,2,3]); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +SELECT arrayPartialSort(2, [1,2,3], [1]); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT arrayPartialSort(2, [1,2,3], 3); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } SELECT arrayPartialSort(arraySort([1,2,3]), [1,2,3]); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } SELECT arrayMap(x -> range(x), [4, 1, 2, 3]) AS arr, 100 AS lim, arrayResize(arrayPartialSort(arrayPartialSort(lim, arr), arr), lim), arrayResize(arrayPartialReverseSort(lim, arr), lim), arrayResize(arrayPartialSort(x -> (-length(x)), lim, arr), lim); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } SELECT arrayPartialReverseSort(arraySort((x, y) -> y, [NULL, NULL], [NULL, NULL]), arr), arrayMap(x -> toString(x), [257, -9223372036854775807, 2, -2147483648, 2147483648, NULL, 65536, -2147483648, 2, 65535]) AS arr, NULL, 100 AS lim, 65536, arrayResize(arrayPartialSort(x -> reverse(x), lim, arr), lim) GROUP BY [NULL, 1023, -2, NULL, 255, '0', NULL, 9223372036854775806] WITH ROLLUP; -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } \ No newline at end of file From d48bdcc7549af1aedd5e8bb4915d8bad56929aef Mon Sep 17 00:00:00 2001 From: Joanna Hulboj Date: Thu, 16 Feb 2023 17:51:40 +0000 Subject: [PATCH 351/717] FIXUP: Into FunctionArrayMapped: Simplify the dispatch in case of positional params --- src/Common/tuple.h | 19 ++++++++ src/Functions/array/FunctionArrayMapped.h | 54 ++++++++++++----------- src/Functions/array/arraySort.cpp | 22 +++++---- 3 files changed, 60 insertions(+), 35 deletions(-) create mode 100644 src/Common/tuple.h diff --git a/src/Common/tuple.h b/src/Common/tuple.h new file mode 100644 index 00000000000..f17de09ee25 --- /dev/null +++ b/src/Common/tuple.h @@ -0,0 +1,19 @@ +#pragma once + +#include +#include + +namespace detail +{ +template +auto to_tuple_impl(C && c, std::index_sequence) +{ + return std::tie(c[Is]...); +} +} + +template +auto to_tuple(C && c) +{ + return detail::to_tuple_impl(c, std::make_index_sequence()); +} diff --git a/src/Functions/array/FunctionArrayMapped.h b/src/Functions/array/FunctionArrayMapped.h index 42b7ae69a40..7a201df4638 100644 --- a/src/Functions/array/FunctionArrayMapped.h +++ b/src/Functions/array/FunctionArrayMapped.h @@ -1,6 +1,5 @@ #pragma once -#include #include #include @@ -11,6 +10,7 @@ #include #include +#include #include #include @@ -79,6 +79,7 @@ const IColumn::Offsets & getOffsets(const T & column) * It is possible for the functions to require fixed number of positional arguments: * arrayPartialSort(limit, arr) * arrayPartialSort(x -> predicate, limit, arr) + * See the arraySort.cpp for details. * * For some functions arrayCount, arrayExists, arrayAll, an overload of the form f(array) is available, * which works in the same way as f(x -> x, array). @@ -194,8 +195,13 @@ public: arguments[num_fixed_params].type->getName()); if constexpr (num_fixed_params) - Impl::checkArguments( - std::span(std::begin(arguments), num_fixed_params), getName()); + { + // Arguments are (f0, f1, ..., arr) + // Delegate checking of (f0, f1, ...) to Impl + std::apply( + [this](auto &&... args) { Impl::checkArguments(getName(), std::forward(args)...); }, + to_tuple(std::begin(arguments))); + } DataTypePtr nested_type = data_type->getNestedType(); @@ -229,8 +235,13 @@ public: arguments[0].type->getName()); if constexpr (num_fixed_params) - Impl::checkArguments( - std::span(std::begin(arguments) + 1, num_fixed_params), getName()); + { + // Arguments are (lambda, f0, f1, ..., arr0, arr1,...) + // Delegate checking of (f0, f1, ...) to Impl + std::apply( + [this](auto &&... args) { Impl::checkArguments(getName(), std::forward(args)...); }, + to_tuple(std::begin(arguments) + 1)); + } /// The types of the remaining arguments are already checked in getLambdaArgumentTypes. @@ -290,23 +301,17 @@ public: if constexpr (std::is_same_v) { - if constexpr (num_fixed_params) - return Impl::execute( - *column_array, - column_array->getNestedColumn().getDataPtr(), - std::span(std::begin(arguments), num_fixed_params)); - else - return Impl::execute(*column_array, column_array->getNestedColumn().getDataPtr()); + // Invoke Impl::execute, add the fixed arguments as trailing ones if applicable + return std::apply( + [&](auto &&... args) { return Impl::execute(*column_array, column_array->getNestedColumn().getDataPtr(), std::forward(args)...); }, + to_tuple(std::begin(arguments))); } else { - if constexpr (num_fixed_params) - return Impl::execute( - *column_array, - column_array->getDataPtr(), - std::span(std::begin(arguments), num_fixed_params)); - else - return Impl::execute(*column_array, column_array->getDataPtr()); + // Invoke Impl::execute, add the fixed arguments as trailing ones if applicable + return std::apply( + [&](auto &&... args) { return Impl::execute(*column_array, column_array->getDataPtr(), std::forward(args)...); }, + to_tuple(std::begin(arguments))); } } else @@ -435,13 +440,10 @@ public: } } - if constexpr (num_fixed_params) - return Impl::execute( - *column_first_array, - lambda_result.column, - std::span(std::begin(arguments) + 1, num_fixed_params)); - else - return Impl::execute(*column_first_array, lambda_result.column); + // Invoke Impl::execute, add the fixed arguments as trailing ones if applicable + return std::apply( + [&](auto &&... args) { return Impl::execute(*column_first_array, lambda_result.column, std::forward(args)...); }, + to_tuple(std::begin(arguments) + 1)); } } }; diff --git a/src/Functions/array/arraySort.cpp b/src/Functions/array/arraySort.cpp index 916923b1f01..6671afc544a 100644 --- a/src/Functions/array/arraySort.cpp +++ b/src/Functions/array/arraySort.cpp @@ -46,35 +46,39 @@ struct ArraySortImpl } }; - static void checkArguments(std::span arguments, const String & name) + static void checkArguments(const String & name, const ColumnWithTypeAndName& arg) requires(num_fixed_params) { - if (arguments.size() != 1) - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} needs limit argument", name); - - WhichDataType which(arguments[0].type.get()); + WhichDataType which(arg.type.get()); if (!which.isUInt() && !which.isInt()) throw Exception( ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of limit argument of function {} (must be UInt or Int)", - arguments[0].type->getName(), + arg.type->getName(), name); } static ColumnPtr execute( const ColumnArray & array, ColumnPtr mapped, - std::span arguments [[maybe_unused]] = {}) + const ColumnWithTypeAndName & limit_arg) { - [[maybe_unused]] const auto limit = [&]() -> size_t + const auto limit = [&]() -> size_t { if constexpr (is_partial) { - return arguments[0].column.get()->getUInt(0); + return limit_arg.column.get()->getUInt(0); } return 0; }(); + return execute(array, mapped, limit); + } + static ColumnPtr execute( + const ColumnArray & array, + ColumnPtr mapped, + size_t limit [[maybe_unused]] = 0) + { const ColumnArray::Offsets & offsets = array.getOffsets(); size_t size = offsets.size(); From 1370f9811ac1469559a2bc13d91c65a2d0d12c13 Mon Sep 17 00:00:00 2001 From: DanRoscigno Date: Thu, 16 Feb 2023 16:21:51 -0500 Subject: [PATCH 352/717] Add docs for standalone ClickHouse Keeper install. --- docs/en/getting-started/install.md | 32 +++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/docs/en/getting-started/install.md b/docs/en/getting-started/install.md index 0e0f54a62fe..4e94682ec40 100644 --- a/docs/en/getting-started/install.md +++ b/docs/en/getting-started/install.md @@ -15,15 +15,7 @@ You have three options for getting up and running with ClickHouse: ## ClickHouse Cloud -The quickest and easiest way to get up and running with ClickHouse is to create a new service in [ClickHouse Cloud](https://clickhouse.cloud/): - -

- -![Create a ClickHouse Cloud service](@site/docs/en/_snippets/images/createservice1.png) -
- -Once your Cloud service is provisioned, you will be able to [connect to it](/docs/en/integrations/connect-a-client.md) and start [inserting data](/docs/en/integrations/data-ingestion.md). - +The quickest and easiest way to get up and running with ClickHouse is to create a new service in [ClickHouse Cloud](https://clickhouse.cloud/). ## Self-Managed Install @@ -73,6 +65,7 @@ The [Quick Start](/docs/en/quick-start.mdx/#step-1-get-clickhouse) walks through It is recommended to use official pre-compiled `deb` packages for Debian or Ubuntu. Run these commands to install packages: +#### Setup the Debian repository ``` bash sudo apt-get install -y apt-transport-https ca-certificates dirmngr sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D754 @@ -80,9 +73,16 @@ sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D7 echo "deb https://packages.clickhouse.com/deb stable main" | sudo tee \ /etc/apt/sources.list.d/clickhouse.list sudo apt-get update +``` +#### Install ClickHouse server and client +```bash sudo apt-get install -y clickhouse-server clickhouse-client +``` +#### Start ClickHouse server + +```bash sudo service clickhouse-server start clickhouse-client # or "clickhouse-client --password" if you've set up a password. ``` @@ -128,12 +128,26 @@ You can replace `stable` with `lts` to use different [release kinds](/docs/en/fa You can also download and install packages manually from [here](https://packages.clickhouse.com/deb/pool/main/c/). +#### Install ClickHouse Keeper +```bash +sudo apt-get install -y clickhouse-keeper +``` + +#### Enable and start ClickHouse Keeper + +```bash +sudo systemctl enable clickhouse-keeper +sudo systemctl start clickhouse-keeper +sudo systemctl status clickhouse-keeper +``` + #### Packages {#packages} - `clickhouse-common-static` — Installs ClickHouse compiled binary files. - `clickhouse-server` — Creates a symbolic link for `clickhouse-server` and installs the default server configuration. - `clickhouse-client` — Creates a symbolic link for `clickhouse-client` and other client-related tools. and installs client configuration files. - `clickhouse-common-static-dbg` — Installs ClickHouse compiled binary files with debug info. +- `clickhouse-keeper` - Used to install ClickHouse Keeper on dedicated ClickHouse Keeper nodes. If you are running ClickHouse Keeper on the same server as ClickHouse server, then you do not need to install this package. Installs ClickHouse Keeper and the default ClickHouse Keeper configuration files. :::info If you need to install specific version of ClickHouse you have to install all packages with the same version: From 52d831cb8781d8b6080a0048404434bdc5ff3fd9 Mon Sep 17 00:00:00 2001 From: DanRoscigno Date: Thu, 16 Feb 2023 16:32:12 -0500 Subject: [PATCH 353/717] add text about standalone Keeper --- docs/en/getting-started/install.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/en/getting-started/install.md b/docs/en/getting-started/install.md index 4e94682ec40..7ae25e7f779 100644 --- a/docs/en/getting-started/install.md +++ b/docs/en/getting-started/install.md @@ -128,7 +128,13 @@ You can replace `stable` with `lts` to use different [release kinds](/docs/en/fa You can also download and install packages manually from [here](https://packages.clickhouse.com/deb/pool/main/c/). -#### Install ClickHouse Keeper +#### Install standalone ClickHouse Keeper + +:::tip +If you are going to run ClickHouse Keeper on the same server as ClickHouse server you +do not need to install ClickHouse Keeper as it is included with ClickHouse server. This command is only needed on standalone ClickHouse Keeper servers. +::: + ```bash sudo apt-get install -y clickhouse-keeper ``` From 5a5f99a27e446e5729c4ac9f65ddd900f12c4457 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 16 Feb 2023 21:08:59 +0100 Subject: [PATCH 354/717] Add CPU flamegraphs for perf tests In Real profiler there is too much noise (polling, cond vars, ...) Signed-off-by: Azat Khuzhin --- docker/test/performance-comparison/perf.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docker/test/performance-comparison/perf.py b/docker/test/performance-comparison/perf.py index c2d61ae0f89..65bf49c2914 100755 --- a/docker/test/performance-comparison/perf.py +++ b/docker/test/performance-comparison/perf.py @@ -366,6 +366,7 @@ for query_index in queries_to_run: settings={ "max_execution_time": args.prewarm_max_query_seconds, "query_profiler_real_time_period_ns": 10000000, + "query_profiler_cpu_time_period_ns": 10000000, "memory_profiler_step": "4Mi", }, ) @@ -497,7 +498,10 @@ for query_index in queries_to_run: res = c.execute( q, query_id=run_id, - settings={"query_profiler_real_time_period_ns": 10000000}, + settings={ + "query_profiler_real_time_period_ns": 10000000, + "query_profiler_cpu_time_period_ns": 10000000, + }, ) print( f"profile\t{query_index}\t{run_id}\t{conn_index}\t{c.last_query.elapsed}" From 3771d97936ebaad19e8b075dcfc633680235123a Mon Sep 17 00:00:00 2001 From: Joanna Hulboj Date: Fri, 17 Feb 2023 07:40:54 +0000 Subject: [PATCH 355/717] FIXUP: Unused error code --- src/Functions/array/arraySort.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Functions/array/arraySort.cpp b/src/Functions/array/arraySort.cpp index 6671afc544a..93c0b294168 100644 --- a/src/Functions/array/arraySort.cpp +++ b/src/Functions/array/arraySort.cpp @@ -9,7 +9,6 @@ namespace DB namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } /** Sort arrays, by values of its elements, or by values of corresponding elements of calculated expression (known as "schwartzsort"). From a39f6f419b3f758e03d14d730593034624668aa1 Mon Sep 17 00:00:00 2001 From: flynn Date: Fri, 17 Feb 2023 08:24:25 +0000 Subject: [PATCH 356/717] refactor --- src/Storages/IStorageDataLake.h | 103 +++++++++++++------------------- src/Storages/StorageDeltaLake.h | 3 +- src/Storages/StorageHudi.h | 3 +- src/Storages/StorageIceberg.h | 4 +- src/Storages/StorageS3.cpp | 12 ++++ src/Storages/StorageS3.h | 17 +++--- 6 files changed, 69 insertions(+), 73 deletions(-) diff --git a/src/Storages/IStorageDataLake.h b/src/Storages/IStorageDataLake.h index 3f17596f9b4..5638db32242 100644 --- a/src/Storages/IStorageDataLake.h +++ b/src/Storages/IStorageDataLake.h @@ -5,95 +5,60 @@ #if USE_AWS_S3 # include - -# include - # include -# include + +# include # include +# include + namespace DB { -template -class IStorageDataLake : public IStorage +template +class IStorageDataLake : public Storage { public: - using Configuration = StorageS3::Configuration; + using Configuration = typename Storage::Configuration; // 1. Parses internal file structure of table // 2. Finds out parts with latest version // 3. Creates url for underlying StorageS3 enigne to handle reads IStorageDataLake( - const StorageS3::Configuration & configuration_, + const typename Storage::Configuration & configuration_, const StorageID & table_id_, ColumnsDescription columns_, const ConstraintsDescription & constraints_, const String & comment, ContextPtr context_, std::optional format_settings_) - : IStorage(table_id_) - , base_configuration{configuration_} - , log(&Poco::Logger::get(fmt::format("Storage{}({})", name, table_id_.table_name))) - { - StorageInMemoryMetadata storage_metadata; - StorageS3::updateS3Configuration(context_, base_configuration); - - auto new_configuration = getAdjustedS3Configuration(context_, base_configuration, log); - - if (columns_.empty()) - { - columns_ = StorageS3::getTableStructureFromData(new_configuration, format_settings_, context_, nullptr); - storage_metadata.setColumns(columns_); - } - else - storage_metadata.setColumns(columns_); - - storage_metadata.setConstraints(constraints_); - storage_metadata.setComment(comment); - setInMemoryMetadata(storage_metadata); - - s3engine = std::make_shared( - new_configuration, + : Storage( + getAdjustedConfiguration( + context_, Storage::updateConfiguration(context_, configuration_), &Poco::Logger::get("Storage" + String(name))), table_id_, columns_, constraints_, comment, context_, - format_settings_, - /* distributed_processing_ */ false, - nullptr); + format_settings_) + { } static constexpr auto name = Name::name; String getName() const override { return name; } - // Reads latest version of Lake Table - Pipe read( - const Names & column_names, - const StorageSnapshotPtr & storage_snapshot, - SelectQueryInfo & query_info, - ContextPtr context, - QueryProcessingStage::Enum processed_stage, - size_t max_block_size, - size_t num_streams) override - { - StorageS3::updateS3Configuration(context, base_configuration); - - return s3engine->read(column_names, storage_snapshot, query_info, context, processed_stage, max_block_size, num_streams); - } - static ColumnsDescription getTableStructureFromData( - StorageS3::Configuration & configuration, const std::optional & format_settings, ContextPtr ctx) + typename Storage::Configuration & configuration, const std::optional & format_settings, ContextPtr ctx) { - StorageS3::updateS3Configuration(ctx, configuration); - auto new_configuration = getAdjustedS3Configuration(ctx, configuration, &Poco::Logger::get("Storage" + String(name))); + Storage::updateConfiguration(ctx, configuration); - return StorageS3::getTableStructureFromData(new_configuration, format_settings, ctx, /*object_infos*/ nullptr); + auto new_configuration = getAdjustedConfiguration(ctx, configuration, &Poco::Logger::get("Storage" + String(name))); + + return Storage::getTableStructureFromData(new_configuration, format_settings, ctx, /*object_infos*/ nullptr); } - static StorageS3::Configuration - getAdjustedS3Configuration(const ContextPtr & context, StorageS3::Configuration & configuration, Poco::Logger * log) + static typename Storage::Configuration + getAdjustedConfiguration(const ContextPtr & context, const typename Storage::Configuration & configuration, Poco::Logger * log) { MetadataParser parser{configuration, context}; @@ -101,7 +66,9 @@ public: String new_uri = std::filesystem::path(configuration.url.uri.toString()) / Name::data_directory_prefix / MetadataParser::generateQueryFromKeys(keys, configuration.format); - StorageS3::Configuration new_configuration(configuration); + typename Storage::Configuration new_configuration(configuration); + + /// The only S3 related stuff remain: S3::URI new_configuration.url = S3::URI(new_uri); LOG_DEBUG(log, "Table path: {}, new uri: {}", configuration.url.key, new_uri); @@ -109,10 +76,9 @@ public: return new_configuration; } - - static StorageS3::Configuration getConfiguration(ASTs & engine_args, ContextPtr local_context) + static typename Storage::Configuration getConfiguration(ASTs & engine_args, ContextPtr local_context) { - auto configuration = StorageS3::getConfiguration(engine_args, local_context, false /* get_format_from_file */); + auto configuration = Storage::getConfiguration(engine_args, local_context, false /* get_format_from_file */); if (configuration.format == "auto") configuration.format = "Parquet"; @@ -120,10 +86,21 @@ public: return configuration; } -private: - StorageS3::Configuration base_configuration; - std::shared_ptr s3engine; - Poco::Logger * log; + SinkToStoragePtr write(const ASTPtr & /*query*/, const StorageMetadataPtr & /*metadata_snapshot*/, ContextPtr /*context*/) override + { + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method write is not supported by storage {}", getName()); + } + + void truncate( + const ASTPtr & /*query*/, + const StorageMetadataPtr & /*metadata_snapshot*/, + ContextPtr /*local_context*/, + TableExclusiveLockHolder &) override + { + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Truncate is not supported by storage {}", getName()); + } + + NamesAndTypesList getVirtuals() const override { return {}; } }; } diff --git a/src/Storages/StorageDeltaLake.h b/src/Storages/StorageDeltaLake.h index bae473e96bc..19225638aa7 100644 --- a/src/Storages/StorageDeltaLake.h +++ b/src/Storages/StorageDeltaLake.h @@ -56,7 +56,8 @@ struct StorageDeltaLakeName static constexpr auto data_directory_prefix = ""; }; -using StorageDeltaLake = IStorageDataLake>; +using StorageDeltaLake + = IStorageDataLake>; } #endif diff --git a/src/Storages/StorageHudi.h b/src/Storages/StorageHudi.h index fd7e97919fb..2bdd5126dc3 100644 --- a/src/Storages/StorageHudi.h +++ b/src/Storages/StorageHudi.h @@ -35,7 +35,8 @@ struct StorageHudiName static constexpr auto data_directory_prefix = ""; }; -using StorageHudi = IStorageDataLake>; +using StorageHudi + = IStorageDataLake>; } #endif diff --git a/src/Storages/StorageIceberg.h b/src/Storages/StorageIceberg.h index 50bd40aff1b..20afde337c6 100644 --- a/src/Storages/StorageIceberg.h +++ b/src/Storages/StorageIceberg.h @@ -6,6 +6,7 @@ # include # include +# include namespace DB { @@ -43,7 +44,8 @@ struct StorageIcebergName static constexpr auto data_directory_prefix = "data"; }; -using StorageIceberg = IStorageDataLake>; +using StorageIceberg + = IStorageDataLake>; } #endif diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 49340afb49b..9e37aa81387 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -1217,6 +1217,18 @@ void StorageS3::truncate(const ASTPtr & /* query */, const StorageMetadataPtr &, } +StorageS3::Configuration StorageS3::updateConfiguration(ContextPtr local_context, const StorageS3::Configuration & configuration) +{ + StorageS3::Configuration new_configuration(configuration); + updateS3Configuration(local_context, new_configuration); + return new_configuration; +} + +void StorageS3::updateConfiguration(ContextPtr local_context, StorageS3::Configuration & configuration) +{ + updateS3Configuration(local_context, configuration); +} + void StorageS3::updateS3Configuration(ContextPtr ctx, StorageS3::Configuration & upd) { auto settings = ctx->getStorageS3Settings().getSettings(upd.url.uri.toString()); diff --git a/src/Storages/StorageS3.h b/src/Storages/StorageS3.h index 9199d0066b5..ab719de28f6 100644 --- a/src/Storages/StorageS3.h +++ b/src/Storages/StorageS3.h @@ -284,21 +284,22 @@ public: bool supportsPartitionBy() const override; - static StorageS3::Configuration getConfiguration(ASTs & engine_args, ContextPtr local_context, bool get_format_from_file = true); - using ObjectInfos = StorageS3Source::ObjectInfos; + static void processNamedCollectionResult(StorageS3::Configuration & configuration, const NamedCollection & collection); + + static SchemaCache & getSchemaCache(const ContextPtr & ctx); + + static StorageS3::Configuration getConfiguration(ASTs & engine_args, ContextPtr local_context, bool get_format_from_file = true); + static ColumnsDescription getTableStructureFromData( StorageS3::Configuration & configuration, const std::optional & format_settings, ContextPtr ctx, ObjectInfos * object_infos = nullptr); - static void processNamedCollectionResult(StorageS3::Configuration & configuration, const NamedCollection & collection); - - static SchemaCache & getSchemaCache(const ContextPtr & ctx); - - static void updateS3Configuration(ContextPtr, Configuration &); + static StorageS3::Configuration updateConfiguration(ContextPtr local_context, const Configuration & configuration); + static void updateConfiguration(ContextPtr local_context, Configuration & configuration); private: friend class StorageS3Cluster; @@ -319,6 +320,8 @@ private: ObjectInfos object_infos; + static void updateS3Configuration(ContextPtr, Configuration &); + static std::shared_ptr createFileIterator( const Configuration & s3_configuration, const std::vector & keys, From 561b5751118cd0bb50b5c150d753c9a5180ca0c7 Mon Sep 17 00:00:00 2001 From: flynn Date: Fri, 17 Feb 2023 08:35:38 +0000 Subject: [PATCH 357/717] fix style --- src/Storages/IStorageDataLake.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Storages/IStorageDataLake.h b/src/Storages/IStorageDataLake.h index 5638db32242..3b0824383d1 100644 --- a/src/Storages/IStorageDataLake.h +++ b/src/Storages/IStorageDataLake.h @@ -16,6 +16,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int NOT_IMPLEMENTED; +} + template class IStorageDataLake : public Storage { From 1d5b7ebc73df3725befdc4a0c5981460b9101926 Mon Sep 17 00:00:00 2001 From: flynn Date: Fri, 17 Feb 2023 09:01:13 +0000 Subject: [PATCH 358/717] fix --- src/Storages/StorageS3.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Storages/StorageS3.h b/src/Storages/StorageS3.h index ab719de28f6..a68009bc99e 100644 --- a/src/Storages/StorageS3.h +++ b/src/Storages/StorageS3.h @@ -298,6 +298,7 @@ public: ContextPtr ctx, ObjectInfos * object_infos = nullptr); +protected: static StorageS3::Configuration updateConfiguration(ContextPtr local_context, const Configuration & configuration); static void updateConfiguration(ContextPtr local_context, Configuration & configuration); From 1a352b9021895c63946544301bff1011b7ba8ba0 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 17 Feb 2023 10:21:52 +0100 Subject: [PATCH 359/717] Fix flakiness of test_backup_restore_on_cluster/test_disallow_concurrency The problem is that if you will run first test_concurrent_restores_on_different_node and after test_concurrent_restores_on_same_node, like in [1], the first one will leave the RESTORE and the second will fail. [1]: https://s3.amazonaws.com/clickhouse-test-reports/45282/5af2967f65c3293b277872a002cb5d570200c008/integration_tests__asan__[3/6].html Signed-off-by: Azat Khuzhin --- .../test_disallow_concurrency.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py b/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py index 43e7682ec1d..d689e44e4d4 100644 --- a/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py +++ b/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py @@ -229,3 +229,9 @@ def test_concurrent_restores_on_different_node(): assert "Concurrent restores not supported" in nodes[1].query_and_get_error( f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_name}" ) + + assert_eq_with_retry( + nodes[0], + f"SELECT status FROM system.backups WHERE status == 'RESTORED'", + "RESTORED", + ) From bae85c50e9538afb602e7a49ce34c446d144ba64 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 17 Feb 2023 12:40:17 +0100 Subject: [PATCH 360/717] Fix stress test --- .../MergeTree/MergeTreeBlockReadUtils.cpp | 11 +++++++++++ .../MergeTree/MergeTreeBlockReadUtils.h | 2 ++ .../MergeTree/MergeTreePrefetchedReadPool.cpp | 18 ------------------ .../MergeTree/MergeTreePrefetchedReadPool.h | 2 -- 4 files changed, 13 insertions(+), 20 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp index 1f69fcae8dc..882d0ed90a6 100644 --- a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp +++ b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp @@ -158,6 +158,17 @@ MergeTreeReadTask::MergeTreeReadTask( { } +MergeTreeReadTask::~MergeTreeReadTask() +{ + if (reader.valid()) + reader.wait(); + + for (const auto & pre_reader : pre_reader_for_step) + { + if (pre_reader.valid()) + pre_reader.wait(); + } +} MergeTreeBlockSizePredictor::MergeTreeBlockSizePredictor( const DataPartPtr & data_part_, const Names & columns, const Block & sample_block) diff --git a/src/Storages/MergeTree/MergeTreeBlockReadUtils.h b/src/Storages/MergeTree/MergeTreeBlockReadUtils.h index 91c895c197e..2eb69a57d9c 100644 --- a/src/Storages/MergeTree/MergeTreeBlockReadUtils.h +++ b/src/Storages/MergeTree/MergeTreeBlockReadUtils.h @@ -91,6 +91,8 @@ struct MergeTreeReadTask int64_t priority_ = 0, std::future reader_ = {}, std::vector> && pre_reader_for_step_ = {}); + + ~MergeTreeReadTask(); }; diff --git a/src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp b/src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp index 7ef1e436ec8..0d8f144acca 100644 --- a/src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp +++ b/src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp @@ -540,24 +540,6 @@ MergeTreePrefetchedReadPool::ThreadsTasks MergeTreePrefetchedReadPool::createThr return result_threads_tasks; } -MergeTreePrefetchedReadPool::~MergeTreePrefetchedReadPool() -{ - for (const auto & [_, thread_tasks] : threads_tasks) - { - for (const auto & task : thread_tasks) - { - if (task->reader.valid()) - task->reader.wait(); - - for (const auto & pre_reader : task->pre_reader_for_step) - { - if (pre_reader.valid()) - pre_reader.wait(); - } - } - } -} - std::string MergeTreePrefetchedReadPool::dumpTasks(const ThreadsTasks & tasks) { WriteBufferFromOwnString result; diff --git a/src/Storages/MergeTree/MergeTreePrefetchedReadPool.h b/src/Storages/MergeTree/MergeTreePrefetchedReadPool.h index 8f46593a3c6..09c9974502a 100644 --- a/src/Storages/MergeTree/MergeTreePrefetchedReadPool.h +++ b/src/Storages/MergeTree/MergeTreePrefetchedReadPool.h @@ -34,8 +34,6 @@ public: bool is_remote_read_, const MergeTreeSettings & storage_settings_); - ~MergeTreePrefetchedReadPool() override; - MergeTreeReadTaskPtr getTask(size_t thread) override; void profileFeedback(ReadBufferFromFileBase::ProfileInfo) override {} From 4b1d997b82fa89304d261c8cdc0a9277c36a913c Mon Sep 17 00:00:00 2001 From: flynn Date: Fri, 17 Feb 2023 12:27:53 +0000 Subject: [PATCH 361/717] fix --- src/Storages/IStorageDataLake.h | 20 +++++++++----------- src/Storages/StorageS3.h | 4 ++++ 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/Storages/IStorageDataLake.h b/src/Storages/IStorageDataLake.h index 3b0824383d1..9e322377fbd 100644 --- a/src/Storages/IStorageDataLake.h +++ b/src/Storages/IStorageDataLake.h @@ -30,7 +30,7 @@ public: // 2. Finds out parts with latest version // 3. Creates url for underlying StorageS3 enigne to handle reads IStorageDataLake( - const typename Storage::Configuration & configuration_, + const Configuration & configuration_, const StorageID & table_id_, ColumnsDescription columns_, const ConstraintsDescription & constraints_, @@ -53,7 +53,7 @@ public: String getName() const override { return name; } static ColumnsDescription getTableStructureFromData( - typename Storage::Configuration & configuration, const std::optional & format_settings, ContextPtr ctx) + Configuration & configuration, const std::optional & format_settings, ContextPtr ctx) { Storage::updateConfiguration(ctx, configuration); @@ -62,26 +62,24 @@ public: return Storage::getTableStructureFromData(new_configuration, format_settings, ctx, /*object_infos*/ nullptr); } - static typename Storage::Configuration - getAdjustedConfiguration(const ContextPtr & context, const typename Storage::Configuration & configuration, Poco::Logger * log) + static Configuration + getAdjustedConfiguration(const ContextPtr & context, const Configuration & configuration, Poco::Logger * log) { MetadataParser parser{configuration, context}; auto keys = parser.getFiles(); - String new_uri = std::filesystem::path(configuration.url.uri.toString()) / Name::data_directory_prefix - / MetadataParser::generateQueryFromKeys(keys, configuration.format); - typename Storage::Configuration new_configuration(configuration); + Configuration new_configuration(configuration); - /// The only S3 related stuff remain: S3::URI - new_configuration.url = S3::URI(new_uri); + new_configuration.appendToPath( + std::filesystem::path(Name::data_directory_prefix) / MetadataParser::generateQueryFromKeys(keys, configuration.format)); - LOG_DEBUG(log, "Table path: {}, new uri: {}", configuration.url.key, new_uri); + LOG_DEBUG(log, "Table path: {}, new uri: {}", configuration.url.key, configuration.getPath()); return new_configuration; } - static typename Storage::Configuration getConfiguration(ASTs & engine_args, ContextPtr local_context) + static Configuration getConfiguration(ASTs & engine_args, ContextPtr local_context) { auto configuration = Storage::getConfiguration(engine_args, local_context, false /* get_format_from_file */); diff --git a/src/Storages/StorageS3.h b/src/Storages/StorageS3.h index a68009bc99e..bfc8dd2bbe2 100644 --- a/src/Storages/StorageS3.h +++ b/src/Storages/StorageS3.h @@ -249,6 +249,10 @@ public: bool static_configuration = true; /// Headers from ast is a part of static configuration. HTTPHeaderEntries headers_from_ast; + + void appendToPath(const String & suffix) { url = S3::URI{std::filesystem::path(url.uri.toString()) / suffix}; } + + String getPath() const { return url.uri.toString(); } /// For logging }; StorageS3( From 706db7cca2cd073414c1edc1416c8ab74185f328 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Fri, 17 Feb 2023 13:39:32 +0100 Subject: [PATCH 362/717] Some more test cases --- tests/queries/0_stateless/02665_regexp_extract.reference | 6 ++++++ tests/queries/0_stateless/02665_regexp_extract.sql | 8 ++++++++ 2 files changed, 14 insertions(+) diff --git a/tests/queries/0_stateless/02665_regexp_extract.reference b/tests/queries/0_stateless/02665_regexp_extract.reference index 42a6b4a5c9c..7afc20547e5 100644 --- a/tests/queries/0_stateless/02665_regexp_extract.reference +++ b/tests/queries/0_stateless/02665_regexp_extract.reference @@ -9,6 +9,12 @@ \N 100 100 +100-200 +0123456789 +012345678 +9 +100 +100 200 100 diff --git a/tests/queries/0_stateless/02665_regexp_extract.sql b/tests/queries/0_stateless/02665_regexp_extract.sql index 3fdf132e48a..830651376f8 100644 --- a/tests/queries/0_stateless/02665_regexp_extract.sql +++ b/tests/queries/0_stateless/02665_regexp_extract.sql @@ -8,6 +8,14 @@ select regexpExtract(null, '([a-z])', 1); select regexpExtract('100-200', null, 1); select regexpExtract('100-200', '([a-z])', null); +select REGEXP_EXTRACT('100-200', '(\\d+)-(\\d+)', 1); +select REGEXP_EXTRACT('100-200', '(\\d+)-(\\d+)'); +select REGEXP_EXTRACT('100-200', '(\\d+)-(\\d+)', 0); + +select regexpExtract('0123456789', '(\d+)(\d+)', 0); +select regexpExtract('0123456789', '(\d+)(\d+)', 1); +select regexpExtract('0123456789', '(\d+)(\d+)', 2); + select regexpExtract(materialize('100-200'), '(\\d+)-(\\d+)'); select regexpExtract(materialize('100-200'), '(\\d+)-(\\d+)', 1); select regexpExtract(materialize('100-200'), '(\\d+)-(\\d+)', 2); From 8b6b86eb8ce6bff5482f2341ad3bf37d52c8ad45 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Fri, 17 Feb 2023 13:41:38 +0100 Subject: [PATCH 363/717] echoOn --- .../02665_regexp_extract.reference | 47 +++++++++++++++++++ .../0_stateless/02665_regexp_extract.sql | 2 + 2 files changed, 49 insertions(+) diff --git a/tests/queries/0_stateless/02665_regexp_extract.reference b/tests/queries/0_stateless/02665_regexp_extract.reference index 7afc20547e5..08482caff8b 100644 --- a/tests/queries/0_stateless/02665_regexp_extract.reference +++ b/tests/queries/0_stateless/02665_regexp_extract.reference @@ -1,38 +1,85 @@ +-- { echoOn } +select regexpExtract('100-200', '(\\d+)-(\\d+)', 1); 100 +select regexpExtract('100-200', '(\\d+)-(\\d+)'); 100 +select regexpExtract('100-200', '(\\d+)-(\\d+)', 2); 200 +select regexpExtract('100-200', '(\\d+)-(\\d+)', 0); 100-200 +select regexpExtract('100-200', '(\\d+).*', 1); 100 +select regexpExtract('100-200', '([a-z])', 1); +select regexpExtract(null, '([a-z])', 1); \N +select regexpExtract('100-200', null, 1); \N +select regexpExtract('100-200', '([a-z])', null); \N +select REGEXP_EXTRACT('100-200', '(\\d+)-(\\d+)', 1); 100 +select REGEXP_EXTRACT('100-200', '(\\d+)-(\\d+)'); 100 +select REGEXP_EXTRACT('100-200', '(\\d+)-(\\d+)', 0); 100-200 +select regexpExtract('0123456789', '(\d+)(\d+)', 0); 0123456789 +select regexpExtract('0123456789', '(\d+)(\d+)', 1); 012345678 +select regexpExtract('0123456789', '(\d+)(\d+)', 2); 9 +select regexpExtract(materialize('100-200'), '(\\d+)-(\\d+)'); 100 +select regexpExtract(materialize('100-200'), '(\\d+)-(\\d+)', 1); 100 +select regexpExtract(materialize('100-200'), '(\\d+)-(\\d+)', 2); 200 +select regexpExtract(materialize('100-200'), '(\\d+).*', 1); 100 +select regexpExtract(materialize('100-200'), '([a-z])', 1); +select regexpExtract(materialize(null), '([a-z])', 1); \N +select regexpExtract(materialize('100-200'), null, 1); \N +select regexpExtract(materialize('100-200'), '([a-z])', null); \N +select regexpExtract('100-200', '(\\d+)-(\\d+)', materialize(1)); 100 +select regexpExtract('100-200', '(\\d+)-(\\d+)', materialize(2)); 200 +select regexpExtract('100-200', '(\\d+).*', materialize(1)); 100 +select regexpExtract('100-200', '([a-z])', materialize(1)); +select regexpExtract(null, '([a-z])', materialize(1)); \N +select regexpExtract('100-200', null, materialize(1)); \N +select regexpExtract('100-200', '([a-z])', materialize(null)); \N +select regexpExtract(materialize('100-200'), '(\\d+)-(\\d+)', materialize(1)); 100 +select regexpExtract(materialize('100-200'), '(\\d+)-(\\d+)', materialize(2)); 200 +select regexpExtract(materialize('100-200'), '(\\d+).*', materialize(1)); 100 +select regexpExtract(materialize('100-200'), '([a-z])', materialize(1)); +select regexpExtract(materialize(null), '([a-z])', materialize(1)); \N +select regexpExtract(materialize('100-200'), null, materialize(1)); \N +select regexpExtract(materialize('100-200'), '([a-z])', materialize(null)); \N +select regexpExtract('100-200'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +select regexpExtract('100-200', '(\\d+)-(\\d+)', 1, 2); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +select regexpExtract(cast('100-200' as FixedString(10)), '(\\d+)-(\\d+)', 1); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +select regexpExtract('100-200', cast('(\\d+)-(\\d+)' as FixedString(20)), 1); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +select regexpExtract('100-200', materialize('(\\d+)-(\\d+)'), 1); -- { serverError ILLEGAL_COLUMN } +select regexpExtract('100-200', '(\\d+)-(\\d+)', 3); -- { serverError INDEX_OF_POSITIONAL_ARGUMENT_IS_OUT_OF_RANGE } +select regexpExtract('100-200', '(\\d+)-(\\d+)', -1); -- { serverError INDEX_OF_POSITIONAL_ARGUMENT_IS_OUT_OF_RANGE } +select regexpExtract('100-200', '\\d+-\\d+', 0); 100-200 +select regexpExtract('100-200', '\\d+-\\d+', 1);-- { serverError INDEX_OF_POSITIONAL_ARGUMENT_IS_OUT_OF_RANGE } diff --git a/tests/queries/0_stateless/02665_regexp_extract.sql b/tests/queries/0_stateless/02665_regexp_extract.sql index 830651376f8..362e8e34d93 100644 --- a/tests/queries/0_stateless/02665_regexp_extract.sql +++ b/tests/queries/0_stateless/02665_regexp_extract.sql @@ -1,3 +1,4 @@ +-- { echoOn } select regexpExtract('100-200', '(\\d+)-(\\d+)', 1); select regexpExtract('100-200', '(\\d+)-(\\d+)'); select regexpExtract('100-200', '(\\d+)-(\\d+)', 2); @@ -51,3 +52,4 @@ select regexpExtract('100-200', '(\\d+)-(\\d+)', 3); -- { serverError INDEX_OF_P select regexpExtract('100-200', '(\\d+)-(\\d+)', -1); -- { serverError INDEX_OF_POSITIONAL_ARGUMENT_IS_OUT_OF_RANGE } select regexpExtract('100-200', '\\d+-\\d+', 0); select regexpExtract('100-200', '\\d+-\\d+', 1);-- { serverError INDEX_OF_POSITIONAL_ARGUMENT_IS_OUT_OF_RANGE } +-- { echoOff } From b3a946866175a71d4e60c8ce52d42c2594522d15 Mon Sep 17 00:00:00 2001 From: flynn Date: Fri, 17 Feb 2023 12:42:24 +0000 Subject: [PATCH 364/717] fix --- src/Storages/StorageS3.cpp | 19 +++++++------------ src/Storages/StorageS3.h | 4 +--- src/Storages/StorageS3Cluster.cpp | 4 ++-- 3 files changed, 10 insertions(+), 17 deletions(-) diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 9e37aa81387..ed290c38c1f 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -954,7 +954,7 @@ StorageS3::StorageS3( context_->getGlobalContext()->getRemoteHostFilter().checkURL(s3_configuration.url.uri); StorageInMemoryMetadata storage_metadata; - updateS3Configuration(context_, s3_configuration); + updateConfiguration(context_, s3_configuration); if (columns_.empty()) { auto columns = getTableStructureFromDataImpl( @@ -1040,7 +1040,7 @@ Pipe StorageS3::read( if (partition_by && has_wildcards) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Reading from a partitioned S3 storage is not implemented yet"); - updateS3Configuration(local_context, s3_configuration); + updateConfiguration(local_context, s3_configuration); Pipes pipes; @@ -1115,7 +1115,7 @@ Pipe StorageS3::read( SinkToStoragePtr StorageS3::write(const ASTPtr & query, const StorageMetadataPtr & metadata_snapshot, ContextPtr local_context) { - updateS3Configuration(local_context, s3_configuration); + updateConfiguration(local_context, s3_configuration); auto sample_block = metadata_snapshot->getSampleBlock(); auto chosen_compression_method = chooseCompressionMethod(keys.back(), compression_method); @@ -1185,7 +1185,7 @@ SinkToStoragePtr StorageS3::write(const ASTPtr & query, const StorageMetadataPtr void StorageS3::truncate(const ASTPtr & /* query */, const StorageMetadataPtr &, ContextPtr local_context, TableExclusiveLockHolder &) { - updateS3Configuration(local_context, s3_configuration); + updateConfiguration(local_context, s3_configuration); if (is_key_with_globs) throw Exception(ErrorCodes::DATABASE_ACCESS_DENIED, @@ -1220,16 +1220,11 @@ void StorageS3::truncate(const ASTPtr & /* query */, const StorageMetadataPtr &, StorageS3::Configuration StorageS3::updateConfiguration(ContextPtr local_context, const StorageS3::Configuration & configuration) { StorageS3::Configuration new_configuration(configuration); - updateS3Configuration(local_context, new_configuration); + updateConfiguration(local_context, new_configuration); return new_configuration; } -void StorageS3::updateConfiguration(ContextPtr local_context, StorageS3::Configuration & configuration) -{ - updateS3Configuration(local_context, configuration); -} - -void StorageS3::updateS3Configuration(ContextPtr ctx, StorageS3::Configuration & upd) +void StorageS3::updateConfiguration(ContextPtr ctx, StorageS3::Configuration & upd) { auto settings = ctx->getStorageS3Settings().getSettings(upd.url.uri.toString()); upd.request_settings = settings.request_settings; @@ -1384,7 +1379,7 @@ ColumnsDescription StorageS3::getTableStructureFromData( ContextPtr ctx, ObjectInfos * object_infos) { - updateS3Configuration(ctx, configuration); + updateConfiguration(ctx, configuration); return getTableStructureFromDataImpl( configuration.format, configuration, configuration.compression_method, configuration.url.key.find_first_of("*?{") != std::string::npos, format_settings, ctx, object_infos); diff --git a/src/Storages/StorageS3.h b/src/Storages/StorageS3.h index bfc8dd2bbe2..b4f95d8d10d 100644 --- a/src/Storages/StorageS3.h +++ b/src/Storages/StorageS3.h @@ -304,7 +304,7 @@ public: protected: static StorageS3::Configuration updateConfiguration(ContextPtr local_context, const Configuration & configuration); - static void updateConfiguration(ContextPtr local_context, Configuration & configuration); + static void updateConfiguration(ContextPtr, Configuration &); private: friend class StorageS3Cluster; @@ -325,8 +325,6 @@ private: ObjectInfos object_infos; - static void updateS3Configuration(ContextPtr, Configuration &); - static std::shared_ptr createFileIterator( const Configuration & s3_configuration, const std::vector & keys, diff --git a/src/Storages/StorageS3Cluster.cpp b/src/Storages/StorageS3Cluster.cpp index 6be0bb1fbd3..c7535bb4550 100644 --- a/src/Storages/StorageS3Cluster.cpp +++ b/src/Storages/StorageS3Cluster.cpp @@ -56,7 +56,7 @@ StorageS3Cluster::StorageS3Cluster( { context_->getGlobalContext()->getRemoteHostFilter().checkURL(configuration_.url.uri); StorageInMemoryMetadata storage_metadata; - StorageS3::updateS3Configuration(context_, s3_configuration); + StorageS3::updateConfiguration(context_, s3_configuration); if (columns_.empty()) { @@ -96,7 +96,7 @@ Pipe StorageS3Cluster::read( size_t /*max_block_size*/, size_t /*num_streams*/) { - StorageS3::updateS3Configuration(context, s3_configuration); + StorageS3::updateConfiguration(context, s3_configuration); auto cluster = getCluster(context); auto extension = getTaskIteratorExtension(query_info.query, context); From 729c4b74d1d1592f3b798583bffbceda9eb0929e Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 17 Feb 2023 13:47:45 +0100 Subject: [PATCH 365/717] Fix black check --- .../test_postgresql_replica_database_engine_1/test.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_postgresql_replica_database_engine_1/test.py b/tests/integration/test_postgresql_replica_database_engine_1/test.py index f7d33143a82..019617cdc39 100644 --- a/tests/integration/test_postgresql_replica_database_engine_1/test.py +++ b/tests/integration/test_postgresql_replica_database_engine_1/test.py @@ -406,7 +406,9 @@ def test_table_schema_changes(started_cluster): altered_idx = random.randint(0, 4) altered_table = f"postgresql_replica_{altered_idx}" - prev_count = int(instance.query(f"SELECT count() FROM test_database.{altered_table}")) + prev_count = int( + instance.query(f"SELECT count() FROM test_database.{altered_table}") + ) cursor.execute(f"ALTER TABLE {altered_table} DROP COLUMN value2") for i in range(NUM_TABLES): @@ -415,7 +417,9 @@ def test_table_schema_changes(started_cluster): assert instance.wait_for_log_line( f"Table postgresql_replica_{altered_idx} is skipped from replication stream" ) - assert prev_count == int(instance.query(f"SELECT count() FROM test_database.{altered_table}")) + assert prev_count == int( + instance.query(f"SELECT count() FROM test_database.{altered_table}") + ) def test_many_concurrent_queries(started_cluster): From 73df45dcaa535bc6c20713507850ee4456b77b4b Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Fri, 17 Feb 2023 14:08:22 +0100 Subject: [PATCH 366/717] More negative cases --- tests/queries/0_stateless/02665_regexp_extract.reference | 3 +++ tests/queries/0_stateless/02665_regexp_extract.sql | 3 +++ 2 files changed, 6 insertions(+) diff --git a/tests/queries/0_stateless/02665_regexp_extract.reference b/tests/queries/0_stateless/02665_regexp_extract.reference index 08482caff8b..7f8cf179a91 100644 --- a/tests/queries/0_stateless/02665_regexp_extract.reference +++ b/tests/queries/0_stateless/02665_regexp_extract.reference @@ -77,6 +77,9 @@ select regexpExtract('100-200'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MAT select regexpExtract('100-200', '(\\d+)-(\\d+)', 1, 2); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } select regexpExtract(cast('100-200' as FixedString(10)), '(\\d+)-(\\d+)', 1); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } select regexpExtract('100-200', cast('(\\d+)-(\\d+)' as FixedString(20)), 1); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +select regexpExtract('100-200', '(\\d+)-(\\d+)', 'a'); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +select regexpExtract(100, '(\\d+)-(\\d+)', 1); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +select regexpExtract('100-200', 1, 1); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } select regexpExtract('100-200', materialize('(\\d+)-(\\d+)'), 1); -- { serverError ILLEGAL_COLUMN } select regexpExtract('100-200', '(\\d+)-(\\d+)', 3); -- { serverError INDEX_OF_POSITIONAL_ARGUMENT_IS_OUT_OF_RANGE } select regexpExtract('100-200', '(\\d+)-(\\d+)', -1); -- { serverError INDEX_OF_POSITIONAL_ARGUMENT_IS_OUT_OF_RANGE } diff --git a/tests/queries/0_stateless/02665_regexp_extract.sql b/tests/queries/0_stateless/02665_regexp_extract.sql index 362e8e34d93..db34aa0715a 100644 --- a/tests/queries/0_stateless/02665_regexp_extract.sql +++ b/tests/queries/0_stateless/02665_regexp_extract.sql @@ -47,6 +47,9 @@ select regexpExtract('100-200'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MAT select regexpExtract('100-200', '(\\d+)-(\\d+)', 1, 2); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } select regexpExtract(cast('100-200' as FixedString(10)), '(\\d+)-(\\d+)', 1); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } select regexpExtract('100-200', cast('(\\d+)-(\\d+)' as FixedString(20)), 1); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +select regexpExtract('100-200', '(\\d+)-(\\d+)', 'a'); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +select regexpExtract(100, '(\\d+)-(\\d+)', 1); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +select regexpExtract('100-200', 1, 1); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } select regexpExtract('100-200', materialize('(\\d+)-(\\d+)'), 1); -- { serverError ILLEGAL_COLUMN } select regexpExtract('100-200', '(\\d+)-(\\d+)', 3); -- { serverError INDEX_OF_POSITIONAL_ARGUMENT_IS_OUT_OF_RANGE } select regexpExtract('100-200', '(\\d+)-(\\d+)', -1); -- { serverError INDEX_OF_POSITIONAL_ARGUMENT_IS_OUT_OF_RANGE } From 8e5110d729dec3031e0802173a6199494af79464 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Fri, 17 Feb 2023 14:28:12 +0100 Subject: [PATCH 367/717] More cases --- .../0_stateless/02665_regexp_extract.reference | 18 ++++++++++++++++++ .../0_stateless/02665_regexp_extract.sql | 5 ++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02665_regexp_extract.reference b/tests/queries/0_stateless/02665_regexp_extract.reference index 7f8cf179a91..15534814c92 100644 --- a/tests/queries/0_stateless/02665_regexp_extract.reference +++ b/tests/queries/0_stateless/02665_regexp_extract.reference @@ -73,6 +73,24 @@ select regexpExtract(materialize('100-200'), null, materialize(1)); \N select regexpExtract(materialize('100-200'), '([a-z])', materialize(null)); \N +select regexpExtract('100-200', '(\\d+)-(\\d+)', number) from numbers(3); +100-200 +100 +200 +select regexpExtract(materialize('100-200'), '(\\d+)-(\\d+)', number) from numbers(3); +100-200 +100 +200 +select regexpExtract(number::String || '-' || (2*number)::String, '(\\d+)-(\\d+)', 1) from numbers(3); +0 +1 +2 +select regexpExtract(number::String || '-' || (2*number)::String, '(\\d+)-(\\d+)', number%3) from numbers(5); +0-0 +1 +4 +3-6 +4 select regexpExtract('100-200'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } select regexpExtract('100-200', '(\\d+)-(\\d+)', 1, 2); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } select regexpExtract(cast('100-200' as FixedString(10)), '(\\d+)-(\\d+)', 1); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } diff --git a/tests/queries/0_stateless/02665_regexp_extract.sql b/tests/queries/0_stateless/02665_regexp_extract.sql index db34aa0715a..cccec89ff08 100644 --- a/tests/queries/0_stateless/02665_regexp_extract.sql +++ b/tests/queries/0_stateless/02665_regexp_extract.sql @@ -41,7 +41,10 @@ select regexpExtract(materialize('100-200'), '([a-z])', materialize(1)); select regexpExtract(materialize(null), '([a-z])', materialize(1)); select regexpExtract(materialize('100-200'), null, materialize(1)); select regexpExtract(materialize('100-200'), '([a-z])', materialize(null)); - +select regexpExtract('100-200', '(\\d+)-(\\d+)', number) from numbers(3); +select regexpExtract(materialize('100-200'), '(\\d+)-(\\d+)', number) from numbers(3); +select regexpExtract(number::String || '-' || (2*number)::String, '(\\d+)-(\\d+)', 1) from numbers(3); +select regexpExtract(number::String || '-' || (2*number)::String, '(\\d+)-(\\d+)', number%3) from numbers(5); select regexpExtract('100-200'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } select regexpExtract('100-200', '(\\d+)-(\\d+)', 1, 2); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } From bcb196a11f59c68478471410e66bc8012aff4dcf Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Fri, 17 Feb 2023 14:42:37 +0100 Subject: [PATCH 368/717] Extracted common logic into saveMatch() --- src/Functions/regexpExtract.cpp | 85 ++++++++++++++------------------- 1 file changed, 35 insertions(+), 50 deletions(-) diff --git a/src/Functions/regexpExtract.cpp b/src/Functions/regexpExtract.cpp index 3cbf8398735..3c176693e51 100644 --- a/src/Functions/regexpExtract.cpp +++ b/src/Functions/regexpExtract.cpp @@ -112,6 +112,31 @@ namespace } private: + template + static void saveMatch( + const OptimizedRegularExpression::MatchVec& matches, + size_t match_index, + const InputString & data, + size_t data_offset, + ColumnString::Chars & res_data, + ColumnString::Offsets & res_offsets, + size_t & res_offset) + { + if (match_index < matches.size() && matches[match_index].offset != std::string::npos) + { + const auto & match = matches[match_index]; + res_data.resize(res_offset + match.length + 1); + memcpySmallAllowReadWriteOverflow15(&res_data[res_offset], &data[data_offset + match.offset], match.length); + res_offset += match.length; + } + else + res_data.resize(res_offset + 1); + + res_data[res_offset] = 0; + ++res_offset; + res_offsets.push_back(res_offset); + } + static void vectorConstant( const ColumnString::Chars & data, const ColumnString::Offsets & offsets, @@ -133,32 +158,18 @@ namespace matches.reserve(index + 1); res_data.reserve(data.size() / 5); - res_offsets.resize(offsets.size()); + res_offsets.reserve(offsets.size()); size_t prev_offset = 0; size_t res_offset = 0; - for (size_t i = 0; i < offsets.size(); ++i) + for (size_t cur_offset : offsets) { - size_t cur_offset = offsets[i]; - unsigned count = regexp.match( + regexp.match( reinterpret_cast(&data[prev_offset]), cur_offset - prev_offset - 1, matches, static_cast(index + 1)); - if (count == index + 1 && matches[index].offset != std::string::npos) - { - const auto & match = matches[index]; - res_data.resize(res_offset + match.length + 1); - memcpySmallAllowReadWriteOverflow15(&res_data[res_offset], &data[prev_offset + match.offset], match.length); - res_offset += match.length; - } - else - res_data.resize(res_offset + 1); - - res_data[res_offset] = 0; - ++res_offset; - res_offsets[i] = res_offset; - + saveMatch(matches, index, data, prev_offset, res_data, res_offsets, res_offset); prev_offset = cur_offset; } } @@ -172,7 +183,7 @@ namespace ColumnString::Offsets & res_offsets) { res_data.reserve(data.size() / 5); - res_offsets.resize(offsets.size()); + res_offsets.reserve(offsets.size()); const Regexps::Regexp regexp = Regexps::createRegexp(pattern); unsigned capture = regexp.getNumberOfSubpatterns(); @@ -193,26 +204,13 @@ namespace index, capture + 1); - unsigned count = regexp.match( + regexp.match( reinterpret_cast(&data[prev_offset]), cur_offset - prev_offset - 1, matches, static_cast(index + 1)); - if (count == index + 1 && matches[index].offset != std::string::npos) - { - const auto & match = matches[index]; - res_data.resize(res_offset + match.length + 1); - memcpySmallAllowReadWriteOverflow15(&res_data[res_offset], &data[prev_offset + match.offset], match.length); - res_offset += match.length; - } - else - res_data.resize(res_offset + 1); - - res_data[res_offset] = 0; - ++res_offset; - res_offsets[i] = res_offset; - + saveMatch(matches, index, data, prev_offset, res_data, res_offsets, res_offset); prev_offset = cur_offset; } } @@ -226,14 +224,13 @@ namespace { size_t rows = column_index->size(); res_data.reserve(str.size() / 5); - res_offsets.resize(rows); + res_offsets.reserve(rows); const Regexps::Regexp regexp = Regexps::createRegexp(pattern); unsigned capture = regexp.getNumberOfSubpatterns(); OptimizedRegularExpression::MatchVec matches; matches.reserve(capture + 1); - unsigned count = regexp.match(str.data(), str.size(), matches, static_cast(capture + 1)); - bool found = count == capture + 1; + regexp.match(str.data(), str.size(), matches, static_cast(capture + 1)); size_t res_offset = 0; for (size_t i = 0; i < rows; ++i) @@ -246,19 +243,7 @@ namespace index, capture + 1); - if (found && matches[index].offset != std::string::npos) - { - const auto & match = matches[index]; - res_data.resize(res_offset + match.length + 1); - memcpySmallAllowReadWriteOverflow15(&res_data[res_offset], str.data() + match.offset, match.length); - res_offset += match.length; - } - else - res_data.resize(res_offset + 1); - - res_data[res_offset] = 0; - ++res_offset; - res_offsets[i] = res_offset; + saveMatch(matches, index, str, 0, res_data, res_offsets, res_offset); } } }; From 7a1aae6189674f853de204202a6763e5a2c509d1 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 17 Feb 2023 14:03:28 +0000 Subject: [PATCH 369/717] Automatic style fix --- tests/ci/workflow_approve_rerun_lambda/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/workflow_approve_rerun_lambda/app.py b/tests/ci/workflow_approve_rerun_lambda/app.py index b563a9786c4..fb14dfd2258 100644 --- a/tests/ci/workflow_approve_rerun_lambda/app.py +++ b/tests/ci/workflow_approve_rerun_lambda/app.py @@ -123,7 +123,7 @@ TRUSTED_CONTRIBUTORS = { "BoloniniD", # Seasoned contributor, HSE "tonickkozlov", # Cloudflare "tylerhannan", # ClickHouse Employee - "myrrc", # Mike Kot, DoubleCloud + "myrrc", # Mike Kot, DoubleCloud ] } From 64dfc9f8ce2a3e7fd9a5636d2f3b2f56c01fa309 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Fri, 17 Feb 2023 14:39:20 +0000 Subject: [PATCH 370/717] Fix clang-analyzer warning --- .../QueryPlan/Optimizations/removeRedundantDistinct.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp index 57684cc23a3..02725dc3122 100644 --- a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp +++ b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp @@ -88,7 +88,7 @@ namespace chassert(!node->children.empty()); node = node->children.front(); } - if (node->type != ActionsDAG::ActionType::INPUT) + if (node && node->type != ActionsDAG::ActionType::INPUT) return nullptr; return node; From 165e2dd18e4ae27d19239623a128449459011c62 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 17 Feb 2023 15:36:03 +0000 Subject: [PATCH 371/717] Automatic style fix --- tests/ci/workflow_approve_rerun_lambda/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/workflow_approve_rerun_lambda/app.py b/tests/ci/workflow_approve_rerun_lambda/app.py index b563a9786c4..fb14dfd2258 100644 --- a/tests/ci/workflow_approve_rerun_lambda/app.py +++ b/tests/ci/workflow_approve_rerun_lambda/app.py @@ -123,7 +123,7 @@ TRUSTED_CONTRIBUTORS = { "BoloniniD", # Seasoned contributor, HSE "tonickkozlov", # Cloudflare "tylerhannan", # ClickHouse Employee - "myrrc", # Mike Kot, DoubleCloud + "myrrc", # Mike Kot, DoubleCloud ] } From 6c1436f78e3a18a95279e1c022c0be7fd6106544 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Fri, 17 Feb 2023 16:50:46 +0100 Subject: [PATCH 372/717] Fixed tests --- src/Functions/nested.cpp | 8 +++++++- src/Storages/LiveView/StorageLiveView.cpp | 3 ++- tests/queries/0_stateless/00490_with_select.sql | 2 ++ .../01934_constexpr_aggregate_function_parameters.sql | 4 +++- tests/queries/0_stateless/02483_elapsed_time.sh | 2 +- 5 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/Functions/nested.cpp b/src/Functions/nested.cpp index f3012fde282..7617951784f 100644 --- a/src/Functions/nested.cpp +++ b/src/Functions/nested.cpp @@ -165,7 +165,13 @@ private: REGISTER_FUNCTION(Nested) { - factory.registerFunction(); + factory.registerFunction({ + R"( +Returns the array of tuples from multiple arrays. +)", + Documentation::Examples{{"nested", "SELECT nested(['keys', 'values'], ['key_1', 'key_2'], ['value_1','value_2'])"}}, + Documentation::Categories{"OtherFunctions"} + }); } } diff --git a/src/Storages/LiveView/StorageLiveView.cpp b/src/Storages/LiveView/StorageLiveView.cpp index 0e9b35fe720..5f398f39dcc 100644 --- a/src/Storages/LiveView/StorageLiveView.cpp +++ b/src/Storages/LiveView/StorageLiveView.cpp @@ -78,6 +78,7 @@ SelectQueryDescription buildSelectQueryDescription(const ASTPtr & select_query, { ASTPtr inner_query = select_query; std::optional dependent_table_storage_id; + bool allow_experimental_analyzer = context->getSettingsRef().allow_experimental_analyzer; while (true) { @@ -100,7 +101,7 @@ SelectQueryDescription buildSelectQueryDescription(const ASTPtr & select_query, if (auto db_and_table = getDatabaseAndTable(*inner_select_query, 0)) { const auto * table_expression = getTableExpression(*inner_select_query, 0); - if (table_expression->database_and_table_name->tryGetAlias().empty()) + if (allow_experimental_analyzer && table_expression->database_and_table_name->tryGetAlias().empty()) table_expression->database_and_table_name->setAlias("__dependent_table"); String select_database_name = db_and_table->database; diff --git a/tests/queries/0_stateless/00490_with_select.sql b/tests/queries/0_stateless/00490_with_select.sql index 7fcb89b5b22..c803cf1d3ad 100644 --- a/tests/queries/0_stateless/00490_with_select.sql +++ b/tests/queries/0_stateless/00490_with_select.sql @@ -1,3 +1,5 @@ +SET allow_experimental_analyzer = 1; + with pow(2,2) as four select pow(four, 2), 2 as two, pow(two, 2); select `pow(four, 2)`, `pow(two, 2)` from (with pow(2,2) as four select pow(four, 2), 2 as two, pow(two, 2)); with (select pow(two,2)) as four select pow(four, 2), 2 as two, pow(two, 2); diff --git a/tests/queries/0_stateless/01934_constexpr_aggregate_function_parameters.sql b/tests/queries/0_stateless/01934_constexpr_aggregate_function_parameters.sql index ac7fdafda82..730313f1daa 100644 --- a/tests/queries/0_stateless/01934_constexpr_aggregate_function_parameters.sql +++ b/tests/queries/0_stateless/01934_constexpr_aggregate_function_parameters.sql @@ -1,10 +1,12 @@ +SET allow_experimental_analyzer = 1; + SELECT groupArray(2 + 3)(number) FROM numbers(10); SELECT groupArray('5'::UInt8)(number) FROM numbers(10); SELECT groupArray(NULL)(number) FROM numbers(10); -- { serverError 36 } SELECT groupArray(NULL + NULL)(number) FROM numbers(10); -- { serverError 36 } SELECT groupArray([])(number) FROM numbers(10); -- { serverError 36 } -SELECT groupArray(throwIf(1))(number) FROM numbers(10); -- { serverError 134 } +SELECT groupArray(throwIf(1))(number) FROM numbers(10); -- { serverError 36 } -- Not the best error message, can be improved. SELECT groupArray(number)(number) FROM numbers(10); -- { serverError 36 } diff --git a/tests/queries/0_stateless/02483_elapsed_time.sh b/tests/queries/0_stateless/02483_elapsed_time.sh index f743120175b..608299eb01b 100755 --- a/tests/queries/0_stateless/02483_elapsed_time.sh +++ b/tests/queries/0_stateless/02483_elapsed_time.sh @@ -17,7 +17,7 @@ EXCEPTION_BEFORE_START_QUERY="WITH FROM system.numbers WHERE number IN (sub) ) - SETTINGS enable_global_with_statement = 0" + SETTINGS enable_global_with_statement = 0, allow_experimental_analyzer = 1" # For this query the system.query_log needs to show ExceptionBeforeStart and elapsed seconds <= 1.0 From afb36dd1205205f09426f8eac38de8d4daa5911a Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 17 Feb 2023 16:54:04 +0100 Subject: [PATCH 373/717] Allow disk setting to override storage_policy from config merge tree settings insteaf of exception --- src/Storages/MergeTree/MergeTreeSettings.cpp | 36 +++++-- .../MergeTree/registerStorageMergeTree.cpp | 5 - .../configs/config.d/mergetree_settings.xml | 5 + .../test_disk_configuration/test.py | 99 ++++++++++++++++--- 4 files changed, 119 insertions(+), 26 deletions(-) create mode 100644 tests/integration/test_disk_configuration/configs/config.d/mergetree_settings.xml diff --git a/src/Storages/MergeTree/MergeTreeSettings.cpp b/src/Storages/MergeTree/MergeTreeSettings.cpp index 059b64875a8..e5af0c772ba 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.cpp +++ b/src/Storages/MergeTree/MergeTreeSettings.cpp @@ -49,22 +49,42 @@ void MergeTreeSettings::loadFromQuery(ASTStorage & storage_def, ContextPtr conte { try { + bool found_disk_setting = false; + bool found_storage_policy_setting = false; + auto changes = storage_def.settings->changes; for (auto & [name, value] : changes) { CustomType custom; - if (value.tryGet(custom) && 0 == strcmp(custom.getTypeName(), "AST")) + if (name == "disk") { - auto ast = dynamic_cast(custom.getImpl()).ast; - if (ast && isDiskFunction(ast)) + if (value.tryGet(custom) && 0 == strcmp(custom.getTypeName(), "AST")) { - const auto & ast_function = assert_cast(*ast); - auto disk_name = getOrCreateDiskFromDiskAST(ast_function, context); - LOG_TRACE(&Poco::Logger::get("MergeTreeSettings"), "Created custom disk {}", disk_name); - value = disk_name; - break; + auto ast = dynamic_cast(custom.getImpl()).ast; + if (ast && isDiskFunction(ast)) + { + const auto & ast_function = assert_cast(*ast); + auto disk_name = getOrCreateDiskFromDiskAST(ast_function, context); + LOG_TRACE(&Poco::Logger::get("MergeTreeSettings"), "Created custom disk {}", disk_name); + value = disk_name; + } } + + if (has("storage_policy")) + resetToDefault("storage_policy"); + + found_disk_setting = true; } + else if (name == "storage_policy") + found_storage_policy_setting = true; + + if (found_disk_setting && found_storage_policy_setting) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "MergeTree settings `storage_policy` and `disk` cannot be specified at the same time"); + } + } applyChanges(changes); diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index 9da845c6b9a..1878201dd43 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -590,11 +590,6 @@ static StoragePtr create(const StorageFactory::Arguments & args) storage_settings->loadFromQuery(*args.storage_def, context); - if (storage_settings->disk.changed && storage_settings->storage_policy.changed) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "MergeTree settings `storage_policy` and `disk` cannot be specified at the same time"); - // updates the default storage_settings with settings specified via SETTINGS arg in a query if (args.storage_def->settings) { diff --git a/tests/integration/test_disk_configuration/configs/config.d/mergetree_settings.xml b/tests/integration/test_disk_configuration/configs/config.d/mergetree_settings.xml new file mode 100644 index 00000000000..6b5875a9d6f --- /dev/null +++ b/tests/integration/test_disk_configuration/configs/config.d/mergetree_settings.xml @@ -0,0 +1,5 @@ + + + unknown + + diff --git a/tests/integration/test_disk_configuration/test.py b/tests/integration/test_disk_configuration/test.py index aca072623ed..a01f10a4407 100644 --- a/tests/integration/test_disk_configuration/test.py +++ b/tests/integration/test_disk_configuration/test.py @@ -32,6 +32,16 @@ def start_cluster(): with_minio=True, macros={"replica": "node2", "shard": "shard1"}, ) + cluster.add_instance( + "node3", + main_configs=[ + "configs/config.d/storage_configuration.xml", + "configs/config.d/remote_servers.xml", + "configs/config.d/mergetree_settings.xml", + ], + stay_alive=True, + with_minio=True, + ) cluster.start() yield cluster @@ -43,19 +53,6 @@ def start_cluster(): def test_merge_tree_disk_setting(start_cluster): node1 = cluster.instances["node1"] - assert ( - "MergeTree settings `storage_policy` and `disk` cannot be specified at the same time" - in node1.query_and_get_error( - f""" - DROP TABLE IF EXISTS {TABLE_NAME}; - CREATE TABLE {TABLE_NAME} (a Int32) - ENGINE = MergeTree() - ORDER BY tuple() - SETTINGS disk = 'disk_local', storage_policy = 's3'; - """ - ) - ) - node1.query( f""" DROP TABLE IF EXISTS {TABLE_NAME}; @@ -296,3 +293,79 @@ def test_merge_tree_custom_disk_setting(start_cluster): f"SELECT disks FROM system.storage_policies WHERE policy_name = '{policy2}'" ).strip() ) + + +def test_merge_tree_setting_override(start_cluster): + node = cluster.instances["node3"] + assert ( + "MergeTree settings `storage_policy` and `disk` cannot be specified at the same time" + in node.query_and_get_error( + f""" + DROP TABLE IF EXISTS {TABLE_NAME}; + CREATE TABLE {TABLE_NAME} (a Int32) + ENGINE = MergeTree() + ORDER BY tuple() + SETTINGS disk = 'kek', storage_policy = 's3'; + """ + ) + ) + + assert "Unknown storage policy" in node.query_and_get_error( + f""" + DROP TABLE IF EXISTS {TABLE_NAME}; + CREATE TABLE {TABLE_NAME} (a Int32) + ENGINE = MergeTree() + ORDER BY tuple(); + """ + ) + + assert "Unknown disk" in node.query_and_get_error( + f""" + DROP TABLE IF EXISTS {TABLE_NAME}; + CREATE TABLE {TABLE_NAME} (a Int32) + ENGINE = MergeTree() + ORDER BY tuple() + SETTINGS disk = 'kek'; + """ + ) + + node.query( + f""" + DROP TABLE IF EXISTS {TABLE_NAME} NO DELAY; + CREATE TABLE {TABLE_NAME} (a Int32) + ENGINE = MergeTree() + ORDER BY tuple() + SETTINGS + disk = disk( + type=s3, + endpoint='http://minio1:9001/root/data/', + access_key_id='minio', + secret_access_key='minio123'); + """ + ) + + minio = cluster.minio_client + node.query(f"INSERT INTO {TABLE_NAME} SELECT number FROM numbers(100)") + assert int(node.query(f"SELECT count() FROM {TABLE_NAME}")) == 100 + assert ( + len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) + > 0 + ) + + node.query( + f""" + DROP TABLE IF EXISTS {TABLE_NAME} NO DELAY; + CREATE TABLE {TABLE_NAME} (a Int32) + ENGINE = MergeTree() + ORDER BY tuple() + SETTINGS disk = 's3' + """ + ) + + minio = cluster.minio_client + node.query(f"INSERT INTO {TABLE_NAME} SELECT number FROM numbers(100)") + assert int(node.query(f"SELECT count() FROM {TABLE_NAME}")) == 100 + assert ( + len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) + > 0 + ) From 3457b68dcc6aa3ab8d7ba96377e9bcad4cdc5ba0 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 17 Feb 2023 17:57:49 +0100 Subject: [PATCH 374/717] fix --- programs/local/LocalServer.cpp | 3 ++- programs/server/Server.cpp | 3 ++- src/Interpreters/DatabaseCatalog.cpp | 16 +++++++++++++--- src/Interpreters/DatabaseCatalog.h | 1 + 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 571e3a3a4df..7d7fd8ff32d 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -654,8 +654,9 @@ void LocalServer::processConfig() if (!config().has("only-system-tables")) { - DatabaseCatalog::instance().startupBackgroundCleanup(); + DatabaseCatalog::instance().createBackgroundTasks(); loadMetadata(global_context); + DatabaseCatalog::instance().startupBackgroundCleanup(); } /// For ClickHouse local if path is not set the loader will be disabled. diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index df223317fcc..34d266b6519 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -1656,11 +1656,12 @@ try /// that may execute DROP before loadMarkedAsDroppedTables() in background, /// and so loadMarkedAsDroppedTables() will find it and try to add, and UUID will overlap. database_catalog.loadMarkedAsDroppedTables(); - database_catalog.startupBackgroundCleanup(); + database_catalog.createBackgroundTasks(); /// Then, load remaining databases loadMetadata(global_context, default_database); convertDatabasesEnginesIfNeed(global_context); startupSystemTables(); + database_catalog.startupBackgroundCleanup(); /// After loading validate that default database exists database_catalog.assertDatabaseExists(default_database); /// Load user-defined SQL functions. diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index 24d5f3491fd..6142608306f 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -152,20 +152,30 @@ void DatabaseCatalog::initializeAndLoadTemporaryDatabase() attachDatabase(TEMPORARY_DATABASE, db_for_temporary_and_external_tables); } -void DatabaseCatalog::startupBackgroundCleanup() +void DatabaseCatalog::createBackgroundTasks() { + /// It has to be done before databases are loaded (to avoid a race condition on initalization) if (Context::getGlobalContextInstance()->getApplicationType() == Context::ApplicationType::SERVER && unused_dir_cleanup_period_sec) { auto cleanup_task_holder = getContext()->getSchedulePool().createTask("DatabaseCatalog", [this]() { this->cleanupStoreDirectoryTask(); }); cleanup_task = std::make_unique(std::move(cleanup_task_holder)); + } + + auto task_holder = getContext()->getSchedulePool().createTask("DatabaseCatalog", [this](){ this->dropTableDataTask(); }); + drop_task = std::make_unique(std::move(task_holder)); +} + +void DatabaseCatalog::startupBackgroundCleanup() +{ + /// And it has to be done after all databases are loaded, otherwise cleanup_task may remove something that should not be removed + if (cleanup_task) + { (*cleanup_task)->activate(); /// Do not start task immediately on server startup, it's not urgent. (*cleanup_task)->scheduleAfter(unused_dir_hide_timeout_sec * 1000); } - auto task_holder = getContext()->getSchedulePool().createTask("DatabaseCatalog", [this](){ this->dropTableDataTask(); }); - drop_task = std::make_unique(std::move(task_holder)); (*drop_task)->activate(); std::lock_guard lock{tables_marked_dropped_mutex}; if (!tables_marked_dropped.empty()) diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h index 5526e1496a3..11a855ebd79 100644 --- a/src/Interpreters/DatabaseCatalog.h +++ b/src/Interpreters/DatabaseCatalog.h @@ -135,6 +135,7 @@ public: static DatabaseCatalog & instance(); static void shutdown(); + void createBackgroundTasks(); void initializeAndLoadTemporaryDatabase(); void startupBackgroundCleanup(); void loadMarkedAsDroppedTables(); From 771070a6928fc6351931c930b5874d96a1622ed5 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Fri, 17 Feb 2023 18:32:56 +0100 Subject: [PATCH 375/717] Test for buffer overrun found by fuzzer --- tests/queries/0_stateless/02665_regexp_extract.reference | 2 ++ tests/queries/0_stateless/02665_regexp_extract.sql | 1 + 2 files changed, 3 insertions(+) diff --git a/tests/queries/0_stateless/02665_regexp_extract.reference b/tests/queries/0_stateless/02665_regexp_extract.reference index 15534814c92..58c427087b3 100644 --- a/tests/queries/0_stateless/02665_regexp_extract.reference +++ b/tests/queries/0_stateless/02665_regexp_extract.reference @@ -91,6 +91,8 @@ select regexpExtract(number::String || '-' || (2*number)::String, '(\\d+)-(\\d+) 4 3-6 4 +select regexpExtract('100-200100-200', '(\\d+)-(\\d+)(\\d+)-(\\d+)', materialize(3)); +0 select regexpExtract('100-200'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } select regexpExtract('100-200', '(\\d+)-(\\d+)', 1, 2); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } select regexpExtract(cast('100-200' as FixedString(10)), '(\\d+)-(\\d+)', 1); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } diff --git a/tests/queries/0_stateless/02665_regexp_extract.sql b/tests/queries/0_stateless/02665_regexp_extract.sql index cccec89ff08..1ebc9d46c6f 100644 --- a/tests/queries/0_stateless/02665_regexp_extract.sql +++ b/tests/queries/0_stateless/02665_regexp_extract.sql @@ -45,6 +45,7 @@ select regexpExtract('100-200', '(\\d+)-(\\d+)', number) from numbers(3); select regexpExtract(materialize('100-200'), '(\\d+)-(\\d+)', number) from numbers(3); select regexpExtract(number::String || '-' || (2*number)::String, '(\\d+)-(\\d+)', 1) from numbers(3); select regexpExtract(number::String || '-' || (2*number)::String, '(\\d+)-(\\d+)', number%3) from numbers(5); +select regexpExtract('100-200100-200', '(\\d+)-(\\d+)(\\d+)-(\\d+)', materialize(3)); select regexpExtract('100-200'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } select regexpExtract('100-200', '(\\d+)-(\\d+)', 1, 2); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } From 0b6f632584862b9a5ddaa94f219f846deafbe6e4 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Fri, 17 Feb 2023 18:34:26 +0100 Subject: [PATCH 376/717] Add clickhouse/clickhouse-keeper Dockerfile autoupdate --- utils/list-versions/update-docker-version.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utils/list-versions/update-docker-version.sh b/utils/list-versions/update-docker-version.sh index 429da330a9f..832dcafb746 100755 --- a/utils/list-versions/update-docker-version.sh +++ b/utils/list-versions/update-docker-version.sh @@ -7,4 +7,5 @@ GIT_ROOT=$(git rev-parse --show-cdup) GIT_ROOT=${GIT_ROOT:-.} VERSION=$(sed -e '1 s/^v//; 1 s/-.*//p; d' "$GIT_ROOT"/utils/list-versions/version_date.tsv) -find "$GIT_ROOT/docker/server/" -name 'Dockerfile.*' -print0 | xargs -0 sed -i "/^ARG VERSION=/ s/^.*$/ARG VERSION=\"$VERSION\"/" +find "$GIT_ROOT/docker/keeper/" "$GIT_ROOT/docker/server/" -name 'Dockerfile.*' -print0 | \ + xargs -0 sed -i "/^ARG VERSION=/ s/^.*$/ARG VERSION=\"$VERSION\"/" From 1565f2c23398afdaf664b6d0b3810696b18f7e8f Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Fri, 17 Feb 2023 18:39:09 +0100 Subject: [PATCH 377/717] Do not destroy symlinks with sed --- utils/list-versions/update-docker-version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/list-versions/update-docker-version.sh b/utils/list-versions/update-docker-version.sh index 832dcafb746..497e7851f7f 100755 --- a/utils/list-versions/update-docker-version.sh +++ b/utils/list-versions/update-docker-version.sh @@ -8,4 +8,4 @@ GIT_ROOT=${GIT_ROOT:-.} VERSION=$(sed -e '1 s/^v//; 1 s/-.*//p; d' "$GIT_ROOT"/utils/list-versions/version_date.tsv) find "$GIT_ROOT/docker/keeper/" "$GIT_ROOT/docker/server/" -name 'Dockerfile.*' -print0 | \ - xargs -0 sed -i "/^ARG VERSION=/ s/^.*$/ARG VERSION=\"$VERSION\"/" + xargs -0 sed -i --follow-symlinks "/^ARG VERSION=/ s/^.*$/ARG VERSION=\"$VERSION\"/" From ecf33b9059e5bbfdfc69049fd4b3de81542338b5 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Fri, 17 Feb 2023 18:39:43 +0100 Subject: [PATCH 378/717] Update the VERSION for clickhouse-keeper Dockerfile --- docker/keeper/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/keeper/Dockerfile b/docker/keeper/Dockerfile index 282392bd98a..eda8274edac 100644 --- a/docker/keeper/Dockerfile +++ b/docker/keeper/Dockerfile @@ -29,7 +29,7 @@ RUN arch=${TARGETARCH:-amd64} \ esac ARG REPOSITORY="https://s3.amazonaws.com/clickhouse-builds/22.4/31c367d3cd3aefd316778601ff6565119fe36682/package_release" -ARG VERSION="22.4.1.917" +ARG VERSION="23.1.3.5" ARG PACKAGES="clickhouse-keeper" # user/group precreated explicitly with fixed uid/gid on purpose. From 53b6fd79fb4464de3d24f92dd0abb2e606772761 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 17 Feb 2023 21:18:49 +0300 Subject: [PATCH 379/717] Update DatabaseCatalog.cpp --- src/Interpreters/DatabaseCatalog.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index 6142608306f..2132c6d39f1 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -154,7 +154,7 @@ void DatabaseCatalog::initializeAndLoadTemporaryDatabase() void DatabaseCatalog::createBackgroundTasks() { - /// It has to be done before databases are loaded (to avoid a race condition on initalization) + /// It has to be done before databases are loaded (to avoid a race condition on initialization) if (Context::getGlobalContextInstance()->getApplicationType() == Context::ApplicationType::SERVER && unused_dir_cleanup_period_sec) { auto cleanup_task_holder From 3d9b3c1b767315728ff69dfca12d6fa184f2f631 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Fri, 17 Feb 2023 18:49:44 +0100 Subject: [PATCH 380/717] Fix for read buffer overrun --- src/Functions/regexpExtract.cpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/Functions/regexpExtract.cpp b/src/Functions/regexpExtract.cpp index 3c176693e51..4e5d5819c5c 100644 --- a/src/Functions/regexpExtract.cpp +++ b/src/Functions/regexpExtract.cpp @@ -112,11 +112,10 @@ namespace } private: - template static void saveMatch( - const OptimizedRegularExpression::MatchVec& matches, + const OptimizedRegularExpression::MatchVec & matches, size_t match_index, - const InputString & data, + const ColumnString::Chars & data, size_t data_offset, ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets, @@ -226,11 +225,15 @@ namespace res_data.reserve(str.size() / 5); res_offsets.reserve(rows); + /// Copy data into padded array to be able to use memcpySmallAllowReadWriteOverflow15. + ColumnString::Chars padded_str; + padded_str.insert(str.begin(), str.end()); + const Regexps::Regexp regexp = Regexps::createRegexp(pattern); unsigned capture = regexp.getNumberOfSubpatterns(); OptimizedRegularExpression::MatchVec matches; matches.reserve(capture + 1); - regexp.match(str.data(), str.size(), matches, static_cast(capture + 1)); + regexp.match(reinterpret_cast(padded_str.data()), padded_str.size(), matches, static_cast(capture + 1)); size_t res_offset = 0; for (size_t i = 0; i < rows; ++i) @@ -243,7 +246,7 @@ namespace index, capture + 1); - saveMatch(matches, index, str, 0, res_data, res_offsets, res_offset); + saveMatch(matches, index, padded_str, 0, res_data, res_offsets, res_offset); } } }; From 30748eace8e6aeddfbf0eb34a6fc396f18f2d786 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 17 Feb 2023 21:02:16 +0100 Subject: [PATCH 381/717] Fix black check --- tests/integration/test_disk_configuration/test.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_disk_configuration/test.py b/tests/integration/test_disk_configuration/test.py index a01f10a4407..60d75e4dac1 100644 --- a/tests/integration/test_disk_configuration/test.py +++ b/tests/integration/test_disk_configuration/test.py @@ -348,8 +348,7 @@ def test_merge_tree_setting_override(start_cluster): node.query(f"INSERT INTO {TABLE_NAME} SELECT number FROM numbers(100)") assert int(node.query(f"SELECT count() FROM {TABLE_NAME}")) == 100 assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) - > 0 + len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) > 0 ) node.query( @@ -366,6 +365,5 @@ def test_merge_tree_setting_override(start_cluster): node.query(f"INSERT INTO {TABLE_NAME} SELECT number FROM numbers(100)") assert int(node.query(f"SELECT count() FROM {TABLE_NAME}")) == 100 assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) - > 0 + len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) > 0 ) From 502e240563d5bd557e0c848bbf9735598918e691 Mon Sep 17 00:00:00 2001 From: DanRoscigno Date: Fri, 17 Feb 2023 15:55:55 -0500 Subject: [PATCH 382/717] update RPM install --- docs/en/getting-started/install.md | 31 +++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/docs/en/getting-started/install.md b/docs/en/getting-started/install.md index 7ae25e7f779..a0b4efb3ddd 100644 --- a/docs/en/getting-started/install.md +++ b/docs/en/getting-started/install.md @@ -164,17 +164,46 @@ If you need to install specific version of ClickHouse you have to install all pa It is recommended to use official pre-compiled `rpm` packages for CentOS, RedHat, and all other rpm-based Linux distributions. +#### Setup the RPM repository First, you need to add the official repository: ``` bash sudo yum install -y yum-utils sudo yum-config-manager --add-repo https://packages.clickhouse.com/rpm/clickhouse.repo -sudo yum install -y clickhouse-server clickhouse-client +``` +#### Install ClickHouse server and client + +```bash +sudo yum install -y clickhouse-server clickhouse-client +``` + +#### Start ClickHouse server + +```bash sudo /etc/init.d/clickhouse-server start clickhouse-client # or "clickhouse-client --password" if you set up a password. ``` +#### Install standalone ClickHouse Keeper + +:::tip +If you are going to run ClickHouse Keeper on the same server as ClickHouse server you +do not need to install ClickHouse Keeper as it is included with ClickHouse server. This command is only needed on standalone ClickHouse Keeper servers. +::: + +```bash +sudo yum install -y clickhouse-keeper +``` + +#### Enable and start ClickHouse Keeper + +```bash +sudo systemctl enable clickhouse-keeper +sudo systemctl start clickhouse-keeper +sudo systemctl status clickhouse-keeper +``` + diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index e80155f39b9..5eb4dd19829 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -175,26 +175,69 @@ namespace const auto CLEANUP_TIMEOUT_MS = 3000; const auto MAX_THREAD_WORK_DURATION_MS = 60000; // once per minute leave do reschedule (we can't lock threads in pool forever) - const String CONFIG_PREFIX = "kafka"; - const String CONFIG_TOPIC_TAG = "topic"; + const String CONFIG_KAFKA_TAG = "kafka"; + const String CONFIG_KAFKA_TOPIC_TAG = "kafka_topic"; + const String CONFIG_NAME_TAG = "name"; - /// Read server configuration into cppkafka configuration + /// Read server configuration into cppkafka configuration, used by global configuration and by legacy per-topic configuration void loadFromConfig(cppkafka::Configuration & kafka_config, const Poco::Util::AbstractConfiguration & config, const String & config_prefix) { - Poco::Util::AbstractConfiguration::Keys keys; - config.keys(config_prefix, keys); + /// Read all tags one level below + Poco::Util::AbstractConfiguration::Keys tags; + config.keys(config_prefix, tags); - for (const auto & key : keys) + for (const auto & tag : tags) { - if (key == CONFIG_TOPIC_TAG) - continue; // special case: when loading the global config, cppkafka rightfully complains that "topic" isn't a valid kafka setting - const String key_path = config_prefix + "." + key; - // log_level has valid underscore, rest librdkafka setting use dot.separated.format - // which is not acceptable for XML. - // See also https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md - const String key_name = (key == "log_level") ? key : boost::replace_all_copy(key, "_", "."); - String value = config.getString(key_path); - kafka_config.set(key_name, value); + if (tag.starts_with(CONFIG_KAFKA_TOPIC_TAG)) /// Multiple occurrences given as "kafka_topic", "kafka_topic[1]", etc. + continue; // used by new per-topic configuration, ignore + + const String setting_path = config_prefix + "." + tag; + const String setting_value = config.getString(setting_path); + + // "log_level" has valid underscore, the remaining librdkafka setting use dot.separated.format which isn't acceptable for XML. + // See https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md + const String setting_name_in_kafka_config = (tag == "log_level") ? tag : boost::replace_all_copy(tag, "_", "."); + kafka_config.set(setting_name_in_kafka_config, setting_value); + } + } + + /// Read server configuration into cppkafa configuration, used by new per-topic configuration + void loadTopicConfig(cppkafka::Configuration & kafka_config, const Poco::Util::AbstractConfiguration & config, const String & config_prefix, const String & topic) + { + /// Read all tags one level below + Poco::Util::AbstractConfiguration::Keys tags; + config.keys(config_prefix, tags); + + for (const auto & tag : tags) + { + /// Consider tag . Multiple occurrences given as "kafka_topic", "kafka_topic[1]", etc. + if (tag.starts_with(CONFIG_KAFKA_TOPIC_TAG)) + { + /// Read topic name between ... + const String kafka_topic_path = config_prefix + "." + tag; + const String kafpa_topic_name_path = kafka_topic_path + "." + CONFIG_NAME_TAG; + + const String topic_name = config.getString(kafpa_topic_name_path); + if (topic_name == topic) + { + /// Found it! Now read the per-topic configuration into cppkafka. + Poco::Util::AbstractConfiguration::Keys inner_tags; + config.keys(kafka_topic_path, inner_tags); + for (const auto & inner_tag : inner_tags) + { + if (inner_tag == CONFIG_NAME_TAG) + continue; // ignore + + // "log_level" has valid underscore, the remaining librdkafka setting use dot.separated.format which isn't acceptable for XML. + // See https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md + const String setting_path = kafka_topic_path + "." + inner_tag; + const String setting_value = config.getString(setting_path); + + const String setting_name_in_kafka_config = (inner_tag == "log_level") ? inner_tag : boost::replace_all_copy(inner_tag, "_", "."); + kafka_config.set(setting_name_in_kafka_config, setting_value); + } + } + } } } } @@ -513,8 +556,8 @@ size_t StorageKafka::getPollTimeoutMillisecond() const String StorageKafka::getConfigPrefix() const { if (!collection_name.empty()) - return "named_collections." + collection_name + "." + CONFIG_PREFIX; /// Add one more level to separate librdkafka configuration. - return CONFIG_PREFIX; + return "named_collections." + collection_name + "." + CONFIG_KAFKA_TAG; /// Add one more level to separate librdkafka configuration. + return CONFIG_KAFKA_TAG; } void StorageKafka::updateConfiguration(cppkafka::Configuration & kafka_config) @@ -571,21 +614,24 @@ void StorageKafka::updateConfiguration(cppkafka::Configuration & kafka_config) loadFromConfig(kafka_config, config, topic_config_key); } - // Update consumer topic-specific configuration (new syntax). Example with topic "football": + // Update consumer topic-specific configuration (new syntax). Example with topics "football" and "baseball": // - // + // + // football // 250 - // 100000 - // + // 5000 + // + // + // baseball + // 300 + // 2000 + // // - // Advantages: The period restriction doesn't apply to XML attributes (e.g. works) and everything - // kafka-related is below . + // Advantages: The period restriction no longer applies (e.g. sports.football will work), everything + // Kafka-related is below . for (const auto & topic : topics) - { - const auto topic_config_key = config_prefix + ".topic[@name=" + topic + "]"; - if (config.has(topic_config_key)) - loadFromConfig(kafka_config, config, topic_config_key); - } + if (config.has(config_prefix)) + loadTopicConfig(kafka_config, config, config_prefix, topic); // No need to add any prefix, messages can be distinguished kafka_config.set_log_callback([this](cppkafka::KafkaHandleBase &, int level, const std::string & facility, const std::string & message) diff --git a/tests/integration/test_storage_kafka/configs/kafka.xml b/tests/integration/test_storage_kafka/configs/kafka.xml index f1987aab009..062c98a2ac7 100644 --- a/tests/integration/test_storage_kafka/configs/kafka.xml +++ b/tests/integration/test_storage_kafka/configs/kafka.xml @@ -10,12 +10,13 @@ --> cgrp,consumer,topic,protocol - + + consumer_hang 300 6000 - + diff --git a/tests/integration/test_storage_kerberized_kafka/configs/kafka.xml b/tests/integration/test_storage_kerberized_kafka/configs/kafka.xml index 56cfab2834f..61558181fe6 100644 --- a/tests/integration/test_storage_kerberized_kafka/configs/kafka.xml +++ b/tests/integration/test_storage_kerberized_kafka/configs/kafka.xml @@ -15,12 +15,13 @@ kafkauser/instance@TEST.CLICKHOUSE.TECH security false - + + consumer_hang 300 6000 - + From b3eacf0894d3fc7cb9549818b54f7f66ac525ed7 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Thu, 23 Feb 2023 20:06:24 +0000 Subject: [PATCH 615/717] Whitespace for the win --- docs/en/engines/table-engines/integrations/kafka.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/en/engines/table-engines/integrations/kafka.md b/docs/en/engines/table-engines/integrations/kafka.md index 4fe8a817bf1..524b988ae09 100644 --- a/docs/en/engines/table-engines/integrations/kafka.md +++ b/docs/en/engines/table-engines/integrations/kafka.md @@ -169,12 +169,15 @@ Similar to GraphiteMergeTree, the Kafka engine supports extended configuration u cgrp smallest + + logs 250 100000 + stats 400 @@ -197,10 +200,12 @@ Similar to GraphiteMergeTree, the Kafka engine supports extended configuration u + 250 100000 + 400 50000 From cca0bead445aae71d30b26423e6f4fa5cfd0dec6 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Thu, 23 Feb 2023 20:07:06 +0000 Subject: [PATCH 616/717] Slighly improved example --- docs/en/engines/table-engines/integrations/kafka.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/engines/table-engines/integrations/kafka.md b/docs/en/engines/table-engines/integrations/kafka.md index 524b988ae09..ef422632d3e 100644 --- a/docs/en/engines/table-engines/integrations/kafka.md +++ b/docs/en/engines/table-engines/integrations/kafka.md @@ -165,8 +165,8 @@ If you want to change the target table by using `ALTER`, we recommend disabling Similar to GraphiteMergeTree, the Kafka engine supports extended configuration using the ClickHouse config file. There are two configuration keys that you can use: global (below ``) and topic-level (below ``). The global configuration is applied first, and then the topic-level configuration is applied (if it exists). ``` xml - + cgrp smallest @@ -192,8 +192,8 @@ Similar to GraphiteMergeTree, the Kafka engine supports extended configuration u Example in deprecated syntax ``` xml - + cgrp smallest From d26a12b6b2b01629c5e049a9a10e597ba757b26a Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Thu, 23 Feb 2023 20:27:30 +0000 Subject: [PATCH 617/717] Remove ZSTD version from CMake output - produced wrong output: > ZSTD VERSION 1.5.0 (see any recent local or CI build log), whereas the current version is 1.5.4 - that could be fixed, on the other hand other contrib's also don't print their version (it is given implicitly by ClickHouse's source Git hash), the parsing regex is fragile and it hasn't changed since 2016 so it was probably broken for a long time already - therefore remove it --- contrib/zstd-cmake/CMakeLists.txt | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/contrib/zstd-cmake/CMakeLists.txt b/contrib/zstd-cmake/CMakeLists.txt index f44d5db12c4..a6262178dc7 100644 --- a/contrib/zstd-cmake/CMakeLists.txt +++ b/contrib/zstd-cmake/CMakeLists.txt @@ -30,25 +30,10 @@ # - zstd homepage : http://www.zstd.net/ # ################################################################ -# Get library version based on information from input content (use regular exp) -function(GetLibraryVersion _content _outputVar1 _outputVar2 _outputVar3) - string(REGEX MATCHALL ".*define ZSTD_VERSION_MAJOR+.* ([0-9]+).*define ZSTD_VERSION_MINOR+.* ([0-9]+).*define ZSTD_VERSION_RELEASE+.* ([0-9]+)" VERSION_REGEX "${_content}") - SET(${_outputVar1} ${CMAKE_MATCH_1} PARENT_SCOPE) - SET(${_outputVar2} ${CMAKE_MATCH_2} PARENT_SCOPE) - SET(${_outputVar3} ${CMAKE_MATCH_3} PARENT_SCOPE) -endfunction() - # Define library directory, where sources and header files are located SET(LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/zstd/lib") INCLUDE_DIRECTORIES(BEFORE ${LIBRARY_DIR} "${LIBRARY_DIR}/common") -# Read file content -FILE(READ "${LIBRARY_DIR}/zstd.h" HEADER_CONTENT) - -# Parse version -GetLibraryVersion("${HEADER_CONTENT}" LIBVER_MAJOR LIBVER_MINOR LIBVER_RELEASE) -MESSAGE(STATUS "ZSTD VERSION ${LIBVER_MAJOR}.${LIBVER_MINOR}.${LIBVER_RELEASE}") - # cd contrib/zstd/lib # find . -name '*.c' -or -name '*.S' | grep -vP 'deprecated|legacy' | sort | sed 's/^\./ "${LIBRARY_DIR}/"' SET(Sources From 25a759d8d156be980bd6584e00602234e04a8e35 Mon Sep 17 00:00:00 2001 From: Rich Raposa Date: Thu, 23 Feb 2023 14:41:45 -0700 Subject: [PATCH 618/717] Update insert-into.md Fix the formatting --- docs/en/sql-reference/statements/insert-into.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/insert-into.md b/docs/en/sql-reference/statements/insert-into.md index 609740dec56..03a4ab3453c 100644 --- a/docs/en/sql-reference/statements/insert-into.md +++ b/docs/en/sql-reference/statements/insert-into.md @@ -95,7 +95,7 @@ You can insert data separately from the query by using the command-line client o If table has [constraints](../../sql-reference/statements/create/table.md#constraints), their expressions will be checked for each row of inserted data. If any of those constraints is not satisfied — server will raise an exception containing constraint name and expression, the query will be stopped. -## Inserting the Results of `SELECT` +## Inserting the Results of SELECT **Syntax** From dd3591cb9c28230da59927e8213fe894c7050381 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 23 Feb 2023 22:07:08 +0000 Subject: [PATCH 619/717] Update version_date.tsv and changelogs after v22.11.6.44-stable --- docs/changelogs/v22.11.6.44-stable.md | 37 +++++++++++++++++++++++++++ utils/list-versions/version_date.tsv | 1 + 2 files changed, 38 insertions(+) create mode 100644 docs/changelogs/v22.11.6.44-stable.md diff --git a/docs/changelogs/v22.11.6.44-stable.md b/docs/changelogs/v22.11.6.44-stable.md new file mode 100644 index 00000000000..6e628b85150 --- /dev/null +++ b/docs/changelogs/v22.11.6.44-stable.md @@ -0,0 +1,37 @@ +--- +sidebar_position: 1 +sidebar_label: 2023 +--- + +# 2023 Changelog + +### ClickHouse release v22.11.6.44-stable (73ddf91298f) FIXME as compared to v22.11.5.15-stable (d763e5a9239) + +#### Performance Improvement +* Backported in [#45703](https://github.com/ClickHouse/ClickHouse/issues/45703): Fixed performance of short `SELECT` queries that read from tables with large number of`Array`/`Map`/`Nested` columns. [#45630](https://github.com/ClickHouse/ClickHouse/pull/45630) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#46376](https://github.com/ClickHouse/ClickHouse/issues/46376): Fix too big memory usage for vertical merges on non-remote disk. Respect `max_insert_delayed_streams_for_parallel_write` for the remote disk. [#46275](https://github.com/ClickHouse/ClickHouse/pull/46275) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + +#### Build/Testing/Packaging Improvement +* Backported in [#45977](https://github.com/ClickHouse/ClickHouse/issues/45977): Fix zookeeper downloading, update the version, and optimize the image size. [#44853](https://github.com/ClickHouse/ClickHouse/pull/44853) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Backported in [#46114](https://github.com/ClickHouse/ClickHouse/issues/46114): Remove the dependency on the `adduser` tool from the packages, because we don't use it. This fixes [#44934](https://github.com/ClickHouse/ClickHouse/issues/44934). [#45011](https://github.com/ClickHouse/ClickHouse/pull/45011) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#46483](https://github.com/ClickHouse/ClickHouse/issues/46483): Get rid of unnecessary build for standalone clickhouse-keeper. [#46367](https://github.com/ClickHouse/ClickHouse/pull/46367) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Backported in [#46507](https://github.com/ClickHouse/ClickHouse/issues/46507): Some time ago the ccache compression was changed to `zst`, but `gz` archives are downloaded by default. It fixes it by prioritizing zst archive. [#46490](https://github.com/ClickHouse/ClickHouse/pull/46490) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + +#### Bug Fix (user-visible misbehavior in official stable or prestable release) + +* Backported in [#45903](https://github.com/ClickHouse/ClickHouse/issues/45903): Fixed bug with non-parsable default value for EPHEMERAL column in table metadata. [#44026](https://github.com/ClickHouse/ClickHouse/pull/44026) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Backported in [#46239](https://github.com/ClickHouse/ClickHouse/issues/46239): A couple of seg faults have been reported around `c-ares`. All of the recent stack traces observed fail on inserting into `std::unodered_set<>`. I believe I have found the root cause of this, it seems to be unprocessed queries. Prior to this PR, CH calls `poll` to wait on the file descriptors in the `c-ares` channel. According to the [poll docs](https://man7.org/linux/man-pages/man2/poll.2.html), a negative return value means an error has ocurred. Because of this, we would abort the execution and return failure. The problem is that `poll` will also return a negative value if a system interrupt occurs. A system interrupt does not mean the processing has failed or ended, but we would abort it anyways because we were checking for negative values. Once the execution is aborted, the whole stack is destroyed, which includes the `std::unordered_set` passed to the `void *` parameter of the c-ares callback. Once c-ares completed the request, the callback would be invoked and would access an invalid memory address causing a segfault. [#45629](https://github.com/ClickHouse/ClickHouse/pull/45629) ([Arthur Passos](https://github.com/arthurpassos)). +* Backported in [#46216](https://github.com/ClickHouse/ClickHouse/issues/46216): Fix reading of non existing nested columns with multiple level in compact parts. [#46045](https://github.com/ClickHouse/ClickHouse/pull/46045) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#46444](https://github.com/ClickHouse/ClickHouse/issues/46444): Fix possible `LOGICAL_ERROR` in asynchronous inserts with invalid data sent in format `VALUES`. [#46350](https://github.com/ClickHouse/ClickHouse/pull/46350) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#46676](https://github.com/ClickHouse/ClickHouse/issues/46676): Fix an invalid processing of constant `LowCardinality` argument in function `arrayMap`. This bug could lead to a segfault in release, and logical error `Bad cast` in debug build. [#46569](https://github.com/ClickHouse/ClickHouse/pull/46569) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Another attempt to fix automerge, or at least to have debug footprint [#45476](https://github.com/ClickHouse/ClickHouse/pull/45476) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add check for running workflows to merge_pr.py [#45803](https://github.com/ClickHouse/ClickHouse/pull/45803) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Get rid of progress timestamps in release publishing [#45818](https://github.com/ClickHouse/ClickHouse/pull/45818) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add necessary dependency for sanitizers [#45959](https://github.com/ClickHouse/ClickHouse/pull/45959) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add helping logging to auto-merge script [#46080](https://github.com/ClickHouse/ClickHouse/pull/46080) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix write buffer destruction order for vertical merge. [#46205](https://github.com/ClickHouse/ClickHouse/pull/46205) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Get rid of legacy DocsReleaseChecks [#46665](https://github.com/ClickHouse/ClickHouse/pull/46665) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index 3c0f5e7e72a..d4abbdebee1 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -5,6 +5,7 @@ v23.1.1.3077-stable 2023-01-25 v22.12.3.5-stable 2023-01-10 v22.12.2.25-stable 2023-01-06 v22.12.1.1752-stable 2022-12-15 +v22.11.6.44-stable 2023-02-23 v22.11.5.15-stable 2023-01-29 v22.11.4.3-stable 2023-01-10 v22.11.3.47-stable 2023-01-09 From 2f47046fe1245a18faf21e7c7d1ee007e55bc306 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Thu, 23 Feb 2023 23:05:55 +0000 Subject: [PATCH 620/717] CMake: Add quick-n-dirty check that the build machine isn't too old --- cmake/cpu_features.cmake | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/cmake/cpu_features.cmake b/cmake/cpu_features.cmake index b0ff2349957..ed70beff4c2 100644 --- a/cmake/cpu_features.cmake +++ b/cmake/cpu_features.cmake @@ -60,6 +60,23 @@ elseif (ARCH_AARCH64) set (COMPILER_FLAGS "${COMPILER_FLAGS} -march=armv8.2-a+simd+crypto+dotprod+ssbs -Xclang=-target-feature -Xclang=+ldapr -Wno-unused-command-line-argument") endif () + # Quick-n-dirty check: The build generates and executes intermediate binaries, e.g. protoc and llvm-tablegen. If we build on ARM for ARM + # and the build machine is too old, i.e. doesn't satisfy above modern profile, then these intermediate binaries will not run (dump + # SIGILL). Even if they could run, the build machine wouldn't be able to run the ClickHouse binary. In that case, suggest to run the + # build with the compat profile. + if (OS_LINUX AND CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^(aarch64.*|AARCH64.*|arm64.*|ARM64.*)" AND NOT NO_ARMV81_OR_HIGHER) + # CPU features in /proc/cpuinfo and compiler flags don't align :( ... pick some obvious flags contained in modern but not legacy + # profile (full Graviton 3 /proc/cpuinfo is "fp asimd evtstrm aes pmull sha1 sha2 crc32 atomics fphp asimdhp cpuid asimdrdm jscvt + # fcma lrcpc dcpop sha3 sm3 sm4 asimddp sha512 sve asimdfhm dit uscat ilrcpc flagm ssbs paca pacg dcpodp svei8mm svebf16 i8mm bf16 + # dgh rng") + execute_process( + COMMAND grep -P "^(?=.*atomic)(?=.*ssbs)" /proc/cpuinfo + OUTPUT_VARIABLE FLAGS) + if (NOT FLAGS) + MESSAGE(FATAL_ERROR "The build machine does not satisfy the minimum CPU requirements, try to run cmake with -DNO_ARMV81_OR_HIGHER=1") + endif() + endif() + elseif (ARCH_PPC64LE) # By Default, build for power8 and up, allow building for power9 and up # Note that gcc and clang have support for x86 SSE2 intrinsics when building for PowerPC From 55b3bffe599cf3d3c52b4923fcf65e1a93c17549 Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Thu, 23 Feb 2023 21:15:31 -0400 Subject: [PATCH 621/717] Update settings.md --- docs/en/operations/settings/settings.md | 39 +++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 71c712f73a4..11d0e6c28de 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -3948,3 +3948,42 @@ Default value: `0`. :::note Use this setting only for backward compatibility if your use cases depend on old syntax. ::: + +## final {#final} + +Automatically applies [FINAL](../../sql-reference/statements/select/from/#final-modifier) modifier to all tables in a query, to tables where [FINAL](../../sql-reference/statements/select/from/#final-modifier) is applicable, including joined tables and tables in sub-queries, and +distributed tables. + +Example: + +```sql +CREATE TABLE test +( + key Int64, + some String +) +ENGINE = ReplacingMergeTree +ORDER BY key; + +INSERT INTO test FORMAT Values (1, 'first'); +INSERT INTO test FORMAT Values (1, 'second'); + +SELECT * FROM test; +┌─key─┬─some───┐ +│ 1 │ second │ +└─────┴────────┘ +┌─key─┬─some──┐ +│ 1 │ first │ +└─────┴───────┘ + +SELECT * FROM test SETTINGS final = 1; +┌─key─┬─some───┐ +│ 1 │ second │ +└─────┴────────┘ + +SET final = 1; +SELECT * FROM test; +┌─key─┬─some───┐ +│ 1 │ second │ +└─────┴────────┘ +``` From 06a3c5598ae3fab61e877bdbf5f5c4639a2a4adb Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Thu, 23 Feb 2023 21:21:36 -0400 Subject: [PATCH 622/717] Update from.md --- docs/en/sql-reference/statements/select/from.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/en/sql-reference/statements/select/from.md b/docs/en/sql-reference/statements/select/from.md index b751384cb72..110889e0797 100644 --- a/docs/en/sql-reference/statements/select/from.md +++ b/docs/en/sql-reference/statements/select/from.md @@ -36,6 +36,8 @@ Queries that use `FINAL` are executed slightly slower than similar queries that **In most cases, avoid using `FINAL`.** The common approach is to use different queries that assume the background processes of the `MergeTree` engine have’t happened yet and deal with it by applying aggregation (for example, to discard duplicates). +`FINAL` be applied automatically using [FINAL](../../operations/settings/settings.md) setting to all tables in a query using a session or a user profile. + ## Implementation Details If the `FROM` clause is omitted, data will be read from the `system.one` table. From 8851c268c2cdd6d53aab81c19212b32bb97cdc5a Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Thu, 23 Feb 2023 21:25:00 -0400 Subject: [PATCH 623/717] Update from.md --- docs/en/sql-reference/statements/select/from.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/select/from.md b/docs/en/sql-reference/statements/select/from.md index 110889e0797..35c83ee783a 100644 --- a/docs/en/sql-reference/statements/select/from.md +++ b/docs/en/sql-reference/statements/select/from.md @@ -36,7 +36,7 @@ Queries that use `FINAL` are executed slightly slower than similar queries that **In most cases, avoid using `FINAL`.** The common approach is to use different queries that assume the background processes of the `MergeTree` engine have’t happened yet and deal with it by applying aggregation (for example, to discard duplicates). -`FINAL` be applied automatically using [FINAL](../../operations/settings/settings.md) setting to all tables in a query using a session or a user profile. +`FINAL` be applied automatically using [FINAL](../../../operations/settings/settings.md#final) setting to all tables in a query using a session or a user profile. ## Implementation Details From 828b85033509c545046a51747074ac33fda7dbb4 Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Thu, 23 Feb 2023 21:39:46 -0400 Subject: [PATCH 624/717] Update from.md --- docs/en/sql-reference/statements/select/from.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/select/from.md b/docs/en/sql-reference/statements/select/from.md index 35c83ee783a..fb6c1f94902 100644 --- a/docs/en/sql-reference/statements/select/from.md +++ b/docs/en/sql-reference/statements/select/from.md @@ -36,7 +36,7 @@ Queries that use `FINAL` are executed slightly slower than similar queries that **In most cases, avoid using `FINAL`.** The common approach is to use different queries that assume the background processes of the `MergeTree` engine have’t happened yet and deal with it by applying aggregation (for example, to discard duplicates). -`FINAL` be applied automatically using [FINAL](../../../operations/settings/settings.md#final) setting to all tables in a query using a session or a user profile. +`FINAL` can be applied automatically using [FINAL](../../../operations/settings/settings.md#final) setting to all tables in a query using a session or a user profile. ## Implementation Details From 329abe28ba56ad5fd03b13d6039d298d19d81005 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 24 Feb 2023 09:38:32 +0000 Subject: [PATCH 625/717] Same check for x86 --- cmake/cpu_features.cmake | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/cmake/cpu_features.cmake b/cmake/cpu_features.cmake index ed70beff4c2..551cb483bda 100644 --- a/cmake/cpu_features.cmake +++ b/cmake/cpu_features.cmake @@ -119,6 +119,22 @@ elseif (ARCH_AMD64) SET(ENABLE_AVX512_FOR_SPEC_OP 0) endif() + # Same quick-n-dirty check for x86 as above for ARM, see above. + if (OS_LINUX AND CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "amd64|x86_64" AND NOT NO_SSE3_OR_HIGHER) + # Test for flags in standard profile but not in NO_SSE3_OR_HIGHER profile. + # /proc/cpuid for Intel Xeon 8124: "fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse + # sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon rep_good nopl xtopology nonstop_tsc cpuid aperfmperf + # tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c + # rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx + # avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke"" + execute_process( + COMMAND grep -P "^(?=.*ssse3)(?=.*sse4_1)(?=.*sse4_2)" /proc/cpuinfo + OUTPUT_VARIABLE FLAGS) + if (NOT FLAGS) + MESSAGE(FATAL_ERROR "The build machine does not satisfy the minimum CPU requirements, try to run cmake with -DNO_SSE3_OR_HIGHER=1") + endif() + endif() + # ClickHouse can be cross-compiled (e.g. on an ARM host for x86) but it is also possible to build ClickHouse on x86 w/o AVX for x86 w/ # AVX. We only check that the compiler can emit certain SIMD instructions, we don't care if the host system is able to run the binary. # Therefore, use check_cxx_source_compiles (= does the code compile+link?) instead of check_cxx_source_runs (= does the code From 032dae5d298f679fa35ad63a254eba9411efedf5 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 24 Feb 2023 09:41:02 +0000 Subject: [PATCH 626/717] Quick-n-dirty --> best effort --- cmake/cpu_features.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/cpu_features.cmake b/cmake/cpu_features.cmake index 551cb483bda..ba21cc43b1e 100644 --- a/cmake/cpu_features.cmake +++ b/cmake/cpu_features.cmake @@ -60,7 +60,7 @@ elseif (ARCH_AARCH64) set (COMPILER_FLAGS "${COMPILER_FLAGS} -march=armv8.2-a+simd+crypto+dotprod+ssbs -Xclang=-target-feature -Xclang=+ldapr -Wno-unused-command-line-argument") endif () - # Quick-n-dirty check: The build generates and executes intermediate binaries, e.g. protoc and llvm-tablegen. If we build on ARM for ARM + # Best-effort check: The build generates and executes intermediate binaries, e.g. protoc and llvm-tablegen. If we build on ARM for ARM # and the build machine is too old, i.e. doesn't satisfy above modern profile, then these intermediate binaries will not run (dump # SIGILL). Even if they could run, the build machine wouldn't be able to run the ClickHouse binary. In that case, suggest to run the # build with the compat profile. @@ -119,7 +119,7 @@ elseif (ARCH_AMD64) SET(ENABLE_AVX512_FOR_SPEC_OP 0) endif() - # Same quick-n-dirty check for x86 as above for ARM, see above. + # Same best-effort check for x86 as above for ARM, see above. if (OS_LINUX AND CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "amd64|x86_64" AND NOT NO_SSE3_OR_HIGHER) # Test for flags in standard profile but not in NO_SSE3_OR_HIGHER profile. # /proc/cpuid for Intel Xeon 8124: "fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse From c34b4c8ed119281b1d28ce7ecbd2c9fc91240f2e Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 24 Feb 2023 09:43:02 +0000 Subject: [PATCH 627/717] Fix grammar --- cmake/cpu_features.cmake | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/cpu_features.cmake b/cmake/cpu_features.cmake index ba21cc43b1e..ca68b2ee49a 100644 --- a/cmake/cpu_features.cmake +++ b/cmake/cpu_features.cmake @@ -65,10 +65,10 @@ elseif (ARCH_AARCH64) # SIGILL). Even if they could run, the build machine wouldn't be able to run the ClickHouse binary. In that case, suggest to run the # build with the compat profile. if (OS_LINUX AND CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^(aarch64.*|AARCH64.*|arm64.*|ARM64.*)" AND NOT NO_ARMV81_OR_HIGHER) - # CPU features in /proc/cpuinfo and compiler flags don't align :( ... pick some obvious flags contained in modern but not legacy - # profile (full Graviton 3 /proc/cpuinfo is "fp asimd evtstrm aes pmull sha1 sha2 crc32 atomics fphp asimdhp cpuid asimdrdm jscvt - # fcma lrcpc dcpop sha3 sm3 sm4 asimddp sha512 sve asimdfhm dit uscat ilrcpc flagm ssbs paca pacg dcpodp svei8mm svebf16 i8mm bf16 - # dgh rng") + # CPU features in /proc/cpuinfo and compiler flags don't align :( ... pick some obvious flags contained in the modern but not in the + # legacy profile (full Graviton 3 /proc/cpuinfo is "fp asimd evtstrm aes pmull sha1 sha2 crc32 atomics fphp asimdhp cpuid asimdrdm + # jscvt fcma lrcpc dcpop sha3 sm3 sm4 asimddp sha512 sve asimdfhm dit uscat ilrcpc flagm ssbs paca pacg dcpodp svei8mm svebf16 i8mm + # bf16 dgh rng") execute_process( COMMAND grep -P "^(?=.*atomic)(?=.*ssbs)" /proc/cpuinfo OUTPUT_VARIABLE FLAGS) @@ -119,7 +119,7 @@ elseif (ARCH_AMD64) SET(ENABLE_AVX512_FOR_SPEC_OP 0) endif() - # Same best-effort check for x86 as above for ARM, see above. + # Same best-effort check for x86 as above for ARM. if (OS_LINUX AND CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "amd64|x86_64" AND NOT NO_SSE3_OR_HIGHER) # Test for flags in standard profile but not in NO_SSE3_OR_HIGHER profile. # /proc/cpuid for Intel Xeon 8124: "fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse From 80a31ce6527efc0435f2dd68d9699b82bb660fd1 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 24 Feb 2023 11:17:06 +0100 Subject: [PATCH 628/717] Fix possible flakiness of 02132_client_history_navigation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It is possible to match some escape sequence as "2": send: sending "\r" to { exp4 } expect: does "\u001b[0m\u001b[J\u001b[29G" (spawn_id exp4) match glob pattern "2"? yes expect: set expect_out(0,string) "2" expect: set expect_out(spawn_id) "exp4" expect: set expect_out(buffer) "\u001b[0m\u001b[J\u001b[2" send: sending "exit\r" to { exp4 } [21GSELECT 2[0m[J[29G exit [1G:-] [5Gexit[0m[J[9G [1mSELECT[0m 2[1m AS [0;32mexit[0m Query id: 1105a048-79c8-4429-8452-b6f9c9c720f7 ┌─[1mexit[0m─┐ │ 2 │ └──────┘ CI: https://s3.amazonaws.com/clickhouse-test-reports/46779/fa3aeb6ff9c9409416c983151da73c6b8a2c5d66/stateless_tests_flaky_check__asan_/run.log Signed-off-by: Azat Khuzhin --- .../0_stateless/02132_client_history_navigation.expect | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/queries/0_stateless/02132_client_history_navigation.expect b/tests/queries/0_stateless/02132_client_history_navigation.expect index ec58edab797..d71ed4c4063 100755 --- a/tests/queries/0_stateless/02132_client_history_navigation.expect +++ b/tests/queries/0_stateless/02132_client_history_navigation.expect @@ -23,16 +23,16 @@ spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \ expect ":) " # Make a query -send -- "SELECT 1\r" +send -- "SELECT unique1\r" expect "1" expect ":) " -send -- "SELECT 2" +send -- "SELECT unique2" # NOTE: it does not work for alacritty with TERM=xterm send -- "\033\[A" -expect "SELECT 1" +expect "SELECT unique1" send -- "\033\[B" -expect "SELECT 2" +expect "SELECT unique2" send -- "\r" -expect "2" +expect "unique2" send -- "exit\r" expect eof From 6db867b03d49e225a8d80b3a271d2f34845ffc51 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 24 Feb 2023 11:19:37 +0100 Subject: [PATCH 629/717] Fix 02049_clickhouse_local_merge_tree flakiness Likely if \4 is sent before replxx spawns the interactive prompt it will not be handled: send: sending "\u0004" to { exp4 } 0 rows in set. Elapsed: 0.001 sec. [1Gce90d76fd35f :) [1Gce90d76fd35f :) expect: timed out , result: So let's use plain "exit" instead. CI: https://s3.amazonaws.com/clickhouse-test-reports/46779/fa3aeb6ff9c9409416c983151da73c6b8a2c5d66/stateless_tests_flaky_check__asan_/run.log Signed-off-by: Azat Khuzhin --- .../0_stateless/02049_clickhouse_local_merge_tree.expect | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02049_clickhouse_local_merge_tree.expect b/tests/queries/0_stateless/02049_clickhouse_local_merge_tree.expect index 42510a6f8a6..a1454696253 100755 --- a/tests/queries/0_stateless/02049_clickhouse_local_merge_tree.expect +++ b/tests/queries/0_stateless/02049_clickhouse_local_merge_tree.expect @@ -5,7 +5,7 @@ set basename [file tail $argv0] exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 log_user 0 -set timeout 20 +set timeout 60 match_max 100000 expect_after { @@ -50,5 +50,5 @@ expect "Ok." send -- "drop table t\r" expect "Ok." -send -- "\4" +send -- "exit\r" expect eof From 264b894b168e124613874c6e28cd8ff2dfb45892 Mon Sep 17 00:00:00 2001 From: Vladimir C Date: Fri, 24 Feb 2023 12:03:31 +0100 Subject: [PATCH 630/717] Remove unneeded recursion CrossToInnerJoinPass/findInTableExpression --- src/Analyzer/Passes/CrossToInnerJoinPass.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/Analyzer/Passes/CrossToInnerJoinPass.cpp b/src/Analyzer/Passes/CrossToInnerJoinPass.cpp index bfa518e4415..4f62d6ef068 100644 --- a/src/Analyzer/Passes/CrossToInnerJoinPass.cpp +++ b/src/Analyzer/Passes/CrossToInnerJoinPass.cpp @@ -112,10 +112,6 @@ bool findInTableExpression(const IQueryTreeNode * source, const QueryTreeNodePtr || findInTableExpression(source, join_node->getRightTableExpression()); } - if (const auto * query_node = table_expression->as()) - { - return findInTableExpression(source, query_node->getJoinTree()); - } return false; } From 9fb24341e2b1d5622d793b4889bef04ef7b887ac Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Fri, 24 Feb 2023 12:12:45 +0100 Subject: [PATCH 631/717] Update settings.md --- docs/en/operations/server-configuration-parameters/settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index 75ae6f3d2bc..17d03dfa4ec 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -467,7 +467,7 @@ Port for exchanging data between ClickHouse servers. The hostname that can be used by other servers to access this server. -If omitted, it is defined in the same way as the `hostname-f` command. +If omitted, it is defined in the same way as the `hostname -f` command. Useful for breaking away from a specific network interface. From 52a088f82d320b7919bc1e0cc855f8d63afd476c Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 24 Feb 2023 11:26:38 +0000 Subject: [PATCH 632/717] Fix new start position --- src/Functions/checkHyperscanRegexp.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Functions/checkHyperscanRegexp.cpp b/src/Functions/checkHyperscanRegexp.cpp index e6fbc3baa1a..ff5f33cad7b 100644 --- a/src/Functions/checkHyperscanRegexp.cpp +++ b/src/Functions/checkHyperscanRegexp.cpp @@ -53,7 +53,7 @@ bool SlowWithHyperscanChecker::isSlowOneRepeat(std::string_view regexp) if (searcher_one_repeat.Match(haystack, start_pos, regexp.size(), re2_st::RE2::Anchor::UNANCHORED, matches, 2)) { const auto & match = matches[0]; - start_pos += (matches[0].data() - haystack.data()) + match.length(); // fwd by prefix + match length + start_pos = (matches[0].data() - haystack.data()) + match.length(); // new start pos = prefix before match + match length const auto & submatch = matches[1]; if (isLargerThanFifty({submatch.data(), submatch.size()})) return true; @@ -75,7 +75,7 @@ bool SlowWithHyperscanChecker::isSlowTwoRepeats(std::string_view regexp) if (searcher_two_repeats.Match(haystack, start_pos, regexp.size(), re2_st::RE2::Anchor::UNANCHORED, matches, 3)) { const auto & match = matches[0]; - start_pos += (matches[0].data() - haystack.data()) + match.length(); // fwd by prefix + match length + start_pos = (matches[0].data() - haystack.data()) + match.length(); // new start pos = prefix before match + match length const auto & submatch1 = matches[1]; const auto & submatch2 = matches[2]; if (isLargerThanFifty({submatch1.data(), submatch1.size()}) From d2c49265bb1f9c2551afb41d815bb70be6be3d43 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 24 Feb 2023 11:29:45 +0000 Subject: [PATCH 633/717] Cosmetics --- src/Functions/checkHyperscanRegexp.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Functions/checkHyperscanRegexp.cpp b/src/Functions/checkHyperscanRegexp.cpp index ff5f33cad7b..441e35cc5db 100644 --- a/src/Functions/checkHyperscanRegexp.cpp +++ b/src/Functions/checkHyperscanRegexp.cpp @@ -48,12 +48,12 @@ bool SlowWithHyperscanChecker::isSlowOneRepeat(std::string_view regexp) re2_st::StringPiece haystack(regexp.data(), regexp.size()); re2_st::StringPiece matches[2]; size_t start_pos = 0; - while (start_pos < regexp.size()) + while (start_pos < haystack.size()) { - if (searcher_one_repeat.Match(haystack, start_pos, regexp.size(), re2_st::RE2::Anchor::UNANCHORED, matches, 2)) + if (searcher_one_repeat.Match(haystack, start_pos, haystack.size(), re2_st::RE2::Anchor::UNANCHORED, matches, 2)) { const auto & match = matches[0]; - start_pos = (matches[0].data() - haystack.data()) + match.length(); // new start pos = prefix before match + match length + start_pos = (matches[0].data() - haystack.data()) + match.size(); // new start pos = prefix before match + match length const auto & submatch = matches[1]; if (isLargerThanFifty({submatch.data(), submatch.size()})) return true; @@ -70,12 +70,12 @@ bool SlowWithHyperscanChecker::isSlowTwoRepeats(std::string_view regexp) re2_st::StringPiece haystack(regexp.data(), regexp.size()); re2_st::StringPiece matches[3]; size_t start_pos = 0; - while (start_pos < regexp.size()) + while (start_pos < haystack.size()) { - if (searcher_two_repeats.Match(haystack, start_pos, regexp.size(), re2_st::RE2::Anchor::UNANCHORED, matches, 3)) + if (searcher_two_repeats.Match(haystack, start_pos, haystack.size(), re2_st::RE2::Anchor::UNANCHORED, matches, 3)) { const auto & match = matches[0]; - start_pos = (matches[0].data() - haystack.data()) + match.length(); // new start pos = prefix before match + match length + start_pos = (matches[0].data() - haystack.data()) + match.size(); // new start pos = prefix before match + match length const auto & submatch1 = matches[1]; const auto & submatch2 = matches[2]; if (isLargerThanFifty({submatch1.data(), submatch1.size()}) From 7ff8d2d3d5a5ff77bff2e3c41b57458183bbf623 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Fri, 24 Feb 2023 12:36:29 +0100 Subject: [PATCH 634/717] Don't report ZK node exists to system.errors --- src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index 31fd99f0aa1..035cbdac55e 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -985,7 +985,7 @@ std::vector ReplicatedMergeTreeSinkImpl::commitPart( } else if (Coordination::isUserError(multi_code)) { - String failed_op_path = zkutil::KeeperMultiException(multi_code, ops, responses).getPathForFirstFailedOp(); + String failed_op_path = ops[zkutil::getFailedOpIndex(multi_code, responses)]->getPath(); auto contains = [](const auto & block_ids, const String & path) { @@ -1002,7 +1002,7 @@ std::vector ReplicatedMergeTreeSinkImpl::commitPart( if (multi_code == Coordination::Error::ZNODEEXISTS && deduplicate_block && contains(block_id_path, failed_op_path)) { - /// Block with the same id have just appeared in table (or other replica), rollback thee insertion. + /// Block with the same id have just appeared in table (or other replica), rollback the insertion. LOG_INFO(log, "Block with ID {} already exists (it was just appeared). Renaming part {} back to {}. Will retry write.", toString(block_id), part->name, temporary_part_relative_path); From ad7ec740097d572c09ae7667979a008391da176f Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Fri, 24 Feb 2023 12:05:21 +0100 Subject: [PATCH 635/717] Make autogenerated version independent from tweak by default --- tests/ci/release.py | 37 +++++++++++++++++-------------------- tests/ci/version_helper.py | 7 +++++++ 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/tests/ci/release.py b/tests/ci/release.py index a4fe4046572..7386af595a4 100755 --- a/tests/ci/release.py +++ b/tests/ci/release.py @@ -265,7 +265,11 @@ class Release: f"for {self.release_type} release" ) - def _commit_cmake_contributors(self, version: ClickHouseVersion) -> None: + def _update_cmake_contributors( + self, version: ClickHouseVersion, reset_tweak: bool = True + ) -> None: + if reset_tweak: + version = version.reset_tweak() update_cmake_version(version) update_contributors(raise_error=True) if self.dry_run: @@ -274,9 +278,15 @@ class Release: self.run(f"git diff '{self.CMAKE_PATH}' '{self.CONTRIBUTORS_PATH}'"), ) self.run(f"git checkout '{self.CMAKE_PATH}' '{self.CONTRIBUTORS_PATH}'") + + def _commit_cmake_contributors( + self, version: ClickHouseVersion, reset_tweak: bool = True + ) -> None: + if reset_tweak: + version = version.reset_tweak() self.run( - f"git commit -m 'Update version to {version.string}' " - f"'{self.CMAKE_PATH}' '{self.CONTRIBUTORS_PATH}'", + f"git commit '{self.CMAKE_PATH}' '{self.CONTRIBUTORS_PATH}' " + f"-m 'Update autogenerated version to {version.string} and contributors'", dry_run=self.dry_run, ) @@ -321,27 +331,12 @@ class Release: with self._create_gh_release(False): self.version = self.version.update(self.release_type) self.version.with_description(version_type) - update_cmake_version(self.version) - update_contributors(raise_error=True) - if self.dry_run: - logging.info( - "Dry running, resetting the following changes in the repo:\n%s", - self.run( - f"git diff '{self.CMAKE_PATH}' '{self.CONTRIBUTORS_PATH}'" - ), - ) - self.run(f"git checkout '{self.CMAKE_PATH}' '{self.CONTRIBUTORS_PATH}'") - + self._update_cmake_contributors(self.version) # Checkouting the commit of the branch and not the branch itself, # then we are able to skip rollback with self._checkout(f"{self.release_branch}^0", False): current_commit = self.run("git rev-parse HEAD") - self.run( - f"git commit -m " - f"'Update version to {self.version.string}' " - f"'{self.CMAKE_PATH}' '{self.CONTRIBUTORS_PATH}'", - dry_run=self.dry_run, - ) + self._commit_cmake_contributors(self.version) with self._push( "HEAD", with_rollback_on_fail=False, remote_ref=self.release_branch ): @@ -406,6 +401,7 @@ class Release: if version_type == VersionType.LTS: pr_labels += " --label release-lts" new_version.with_description(version_type) + self._update_cmake_contributors(new_version) self._commit_cmake_contributors(new_version) with self._push(self.release_branch): with self._create_gh_label( @@ -434,6 +430,7 @@ class Release: self.read_version() self.version = self.version.update(self.release_type) self.version.with_description(VersionType.TESTING) + self._update_cmake_contributors(self.version) self._commit_cmake_contributors(self.version) with self._push(helper_branch): body_file = get_abs_path(".github/PULL_REQUEST_TEMPLATE.md") diff --git a/tests/ci/version_helper.py b/tests/ci/version_helper.py index 69cfba64be3..1ede2c90b55 100755 --- a/tests/ci/version_helper.py +++ b/tests/ci/version_helper.py @@ -88,6 +88,13 @@ class ClickHouseVersion: self.major, self.minor, self.patch + 1, self.revision, self._git ) + def reset_tweak(self) -> "ClickHouseVersion": + if self._git is not None: + self._git.update() + return ClickHouseVersion( + self.major, self.minor, self.patch, self.revision, self._git, "1" + ) + @property def major(self) -> int: return self._major From 6fa16b782836d1c3568a6b38821059dfb252b6d8 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Fri, 24 Feb 2023 12:14:17 +0100 Subject: [PATCH 636/717] Add check that major is not issued in the middle of a year --- tests/ci/release.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/ci/release.py b/tests/ci/release.py index 7386af595a4..f7b5155e750 100755 --- a/tests/ci/release.py +++ b/tests/ci/release.py @@ -199,6 +199,10 @@ class Release: raise ValueError( "The relese type must be 'major' for minor versions>=12" ) + if self._version.minor < 12 and self.release_type == "major": + raise ValueError( + "The relese type must be 'minor' for minor versions<12" + ) with self._checkout(self.release_commit, True): # Checkout to the commit, it will provide the correct current version From d49e0af80a0e2ca3f3960df3bf4e7eea32222123 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 24 Feb 2023 13:29:44 +0100 Subject: [PATCH 637/717] Fix 02047_client_exception flakiness Signed-off-by: Azat Khuzhin --- tests/queries/0_stateless/02047_client_exception.expect | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02047_client_exception.expect b/tests/queries/0_stateless/02047_client_exception.expect index 75e3672da8f..4dfdf211ba2 100755 --- a/tests/queries/0_stateless/02047_client_exception.expect +++ b/tests/queries/0_stateless/02047_client_exception.expect @@ -6,7 +6,7 @@ exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 set history_file $env(CLICKHOUSE_TMP)/$basename.history log_user 0 -set timeout 20 +set timeout 60 match_max 100000 expect_after { @@ -31,5 +31,5 @@ expect "Received exception from server" send -- "DROP TABLE test_02047\r" expect "Ok." -send -- "\4" +send -- "exit\r" expect eof From 3e444287c4d664b2738d1e14000021377cf9aef8 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 24 Feb 2023 13:30:47 +0100 Subject: [PATCH 638/717] Improve 02132_client_history_navigation Signed-off-by: Azat Khuzhin --- .../02132_client_history_navigation.expect | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/queries/0_stateless/02132_client_history_navigation.expect b/tests/queries/0_stateless/02132_client_history_navigation.expect index d71ed4c4063..dc7e44b41ee 100755 --- a/tests/queries/0_stateless/02132_client_history_navigation.expect +++ b/tests/queries/0_stateless/02132_client_history_navigation.expect @@ -23,16 +23,20 @@ spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \ expect ":) " # Make a query -send -- "SELECT unique1\r" +send -- "SELECT 'unique1'\r" expect "1" expect ":) " -send -- "SELECT unique2" +send -- "SELECT 'unique2'" # NOTE: it does not work for alacritty with TERM=xterm send -- "\033\[A" -expect "SELECT unique1" +expect "SELECT 'unique1'" send -- "\033\[B" -expect "SELECT unique2" +expect "SELECT 'unique2'" send -- "\r" -expect "unique2" +# First is echoed command +expect "'unique2'" +# Second is the response from the server +expect "'unique2'" + send -- "exit\r" expect eof From 944c24061512fd3b6fadc632829bb50d155bd6a7 Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Fri, 24 Feb 2023 14:09:58 +0100 Subject: [PATCH 639/717] Update history --- src/Core/SettingsChangesHistory.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index 2d346467338..c9d68b94a5e 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -80,12 +80,12 @@ namespace SettingsChangesHistory /// It's used to implement `compatibility` setting (see https://github.com/ClickHouse/ClickHouse/issues/35972) static std::map settings_changes_history = { + {"23.3", {{"input_format_json_ignore_unknown_keys_in_named_tuple", false, true, "Improve parsing JSON objects as named tuples"}}}, {"23.2", {{"output_format_parquet_fixed_string_as_fixed_byte_array", false, true, "Use Parquet FIXED_LENGTH_BYTE_ARRAY type for FixedString by default"}, {"output_format_arrow_fixed_string_as_fixed_byte_array", false, true, "Use Arrow FIXED_SIZE_BINARY type for FixedString by default"}, {"query_plan_remove_redundant_distinct", false, true, "Remove redundant Distinct step in query plan"}, {"optimize_duplicate_order_by_and_distinct", true, false, "Remove duplicate ORDER BY and DISTINCT if it's possible"}, - {"insert_keeper_max_retries", 0, 20, "Enable reconnections to Keeper on INSERT, improve reliability"}, - {"input_format_json_ignore_unknown_keys_in_named_tuple", false, true, "Improve parsing JSON objects as named tuples"}}}, + {"insert_keeper_max_retries", 0, 20, "Enable reconnections to Keeper on INSERT, improve reliability"}}}, {"23.1", {{"input_format_json_read_objects_as_strings", 0, 1, "Enable reading nested json objects as strings while object type is experimental"}, {"input_format_json_defaults_for_missing_elements_in_named_tuple", false, true, "Allow missing elements in JSON objects while reading named tuples by default"}, {"input_format_csv_detect_header", false, true, "Detect header in CSV format by default"}, From 5aa0a3a57e378195dea7a9f9707d2ac1eb90ec80 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 24 Feb 2023 15:38:53 +0100 Subject: [PATCH 640/717] Fix flakiness of 01179_insert_values_semicolon CI: https://s3.amazonaws.com/clickhouse-test-reports/46785/169f864e88b903b27921ab47149d2cd68d3efc52/stateless_tests__tsan__s3_storage__[4/5].html Signed-off-by: Azat Khuzhin --- tests/queries/0_stateless/01179_insert_values_semicolon.expect | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01179_insert_values_semicolon.expect b/tests/queries/0_stateless/01179_insert_values_semicolon.expect index a5f127ec19e..4784fc38a84 100755 --- a/tests/queries/0_stateless/01179_insert_values_semicolon.expect +++ b/tests/queries/0_stateless/01179_insert_values_semicolon.expect @@ -40,5 +40,5 @@ expect "2020-01-01 00:00:00.000\t2" send -- "DROP TABLE test_01179\r" expect "Ok." -send -- "\4" +send -- "exit\r" expect eof From 86e9b131c79c1899b5d3f0c42437a1067c0fbb67 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 24 Feb 2023 14:44:38 +0100 Subject: [PATCH 641/717] Allow PREWHERE for Merge with different DEFAULT expression for column It can be not that special to have Merge engine w/o default expression, while MergeTree will have it. So improve PREWHERE for such kinds, and also allow PREWHERE for DEFAULT and MATERIALIZED (since both are physical columns on disk and this make sense). Signed-off-by: Azat Khuzhin --- src/Storages/StorageMerge.cpp | 36 ++++++++++++++---- ...erge_prewhere_default_expression.reference | 8 ++++ ...2575_merge_prewhere_default_expression.sql | 38 +++++++++++++++++++ ...prewhere_different_default_kind.reference} | 0 ...merge_prewhere_different_default_kind.sql} | 2 + .../02575_merge_prewhere_ephemeral.reference | 11 ++++++ .../02575_merge_prewhere_ephemeral.sql | 38 +++++++++++++++++++ ...2575_merge_prewhere_materialized.reference | 8 ++++ .../02575_merge_prewhere_materialized.sql | 38 +++++++++++++++++++ 9 files changed, 172 insertions(+), 7 deletions(-) create mode 100644 tests/queries/0_stateless/02575_merge_prewhere_default_expression.reference create mode 100644 tests/queries/0_stateless/02575_merge_prewhere_default_expression.sql rename tests/queries/0_stateless/{02570_merge_alias_prewhere.reference => 02575_merge_prewhere_different_default_kind.reference} (100%) rename tests/queries/0_stateless/{02570_merge_alias_prewhere.sql => 02575_merge_prewhere_different_default_kind.sql} (90%) create mode 100644 tests/queries/0_stateless/02575_merge_prewhere_ephemeral.reference create mode 100644 tests/queries/0_stateless/02575_merge_prewhere_ephemeral.sql create mode 100644 tests/queries/0_stateless/02575_merge_prewhere_materialized.reference create mode 100644 tests/queries/0_stateless/02575_merge_prewhere_materialized.sql diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index b1ac2e2ba0f..0ca29e2826a 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -44,6 +44,27 @@ #include +namespace +{ + +using namespace DB; +bool columnIsPhysical(ColumnDefaultKind kind) +{ + return kind == ColumnDefaultKind::Default || kind == ColumnDefaultKind::Materialized; +} +bool columnDefaultKindHasSameType(ColumnDefaultKind lhs, ColumnDefaultKind rhs) +{ + if (lhs == rhs) + return true; + + if (columnIsPhysical(lhs) == columnIsPhysical(rhs)) + return true; + + return false; +} + +} + namespace DB { @@ -172,11 +193,13 @@ std::optional StorageMerge::supportedPrewhereColumns() const NameSet supported_columns; - std::unordered_map>> column_type_default; + std::unordered_map> column_info; for (const auto & name_type : columns.getAll()) { - column_type_default.emplace(name_type.name, std::make_pair( - name_type.type.get(), columns.getDefault(name_type.name))); + const auto & column_default = columns.getDefault(name_type.name).value_or(ColumnDefault{}); + column_info.emplace(name_type.name, std::make_pair( + name_type.type.get(), + column_default.kind)); supported_columns.emplace(name_type.name); } @@ -191,11 +214,10 @@ std::optional StorageMerge::supportedPrewhereColumns() const const auto & table_columns = table_metadata_ptr->getColumns(); for (const auto & column : table_columns.getAll()) { - const auto & root_type_default = column_type_default[column.name]; - const IDataType * root_type = root_type_default.first; - const std::optional & src_default = root_type_default.second; + const auto & column_default = table_columns.getDefault(column.name).value_or(ColumnDefault{}); + const auto & [root_type, src_default_kind] = column_info[column.name]; if ((root_type && !root_type->equals(*column.type)) || - src_default != table_columns.getDefault(column.name)) + !columnDefaultKindHasSameType(src_default_kind, column_default.kind)) { supported_columns.erase(column.name); } diff --git a/tests/queries/0_stateless/02575_merge_prewhere_default_expression.reference b/tests/queries/0_stateless/02575_merge_prewhere_default_expression.reference new file mode 100644 index 00000000000..434384b3d77 --- /dev/null +++ b/tests/queries/0_stateless/02575_merge_prewhere_default_expression.reference @@ -0,0 +1,8 @@ +-- { echoOn } +SELECT * FROM m PREWHERE a = 'OK' ORDER BY a, f; +OK 1 +OK 2 +SELECT * FROM m PREWHERE f = 1 ORDER BY a, f; +OK 1 +SELECT * FROM m WHERE f = 0 SETTINGS optimize_move_to_prewhere=0; +SELECT * FROM m WHERE f = 0 SETTINGS optimize_move_to_prewhere=1; diff --git a/tests/queries/0_stateless/02575_merge_prewhere_default_expression.sql b/tests/queries/0_stateless/02575_merge_prewhere_default_expression.sql new file mode 100644 index 00000000000..83c1d51269d --- /dev/null +++ b/tests/queries/0_stateless/02575_merge_prewhere_default_expression.sql @@ -0,0 +1,38 @@ +-- Allow PREWHERE when Merge() and MergeTree has different DEFAULT expression + +DROP TABLE IF EXISTS m; +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; + +CREATE TABLE m +( + `a` String, + `f` UInt8 DEFAULT 0 +) +ENGINE = Merge(currentDatabase(), '^(t1|t2)$'); + +CREATE TABLE t1 +( + a String, + f UInt8 DEFAULT 1 +) +ENGINE = MergeTree +ORDER BY tuple() +SETTINGS index_granularity = 8192; +INSERT INTO t1 (a) VALUES ('OK'); + +CREATE TABLE t2 +( + a String, + f UInt8 DEFAULT 2 +) +ENGINE = MergeTree +ORDER BY tuple() +SETTINGS index_granularity = 8192; +INSERT INTO t2 (a) VALUES ('OK'); + +-- { echoOn } +SELECT * FROM m PREWHERE a = 'OK' ORDER BY a, f; +SELECT * FROM m PREWHERE f = 1 ORDER BY a, f; +SELECT * FROM m WHERE f = 0 SETTINGS optimize_move_to_prewhere=0; +SELECT * FROM m WHERE f = 0 SETTINGS optimize_move_to_prewhere=1; diff --git a/tests/queries/0_stateless/02570_merge_alias_prewhere.reference b/tests/queries/0_stateless/02575_merge_prewhere_different_default_kind.reference similarity index 100% rename from tests/queries/0_stateless/02570_merge_alias_prewhere.reference rename to tests/queries/0_stateless/02575_merge_prewhere_different_default_kind.reference diff --git a/tests/queries/0_stateless/02570_merge_alias_prewhere.sql b/tests/queries/0_stateless/02575_merge_prewhere_different_default_kind.sql similarity index 90% rename from tests/queries/0_stateless/02570_merge_alias_prewhere.sql rename to tests/queries/0_stateless/02575_merge_prewhere_different_default_kind.sql index 59ca717a418..0f1d582a26e 100644 --- a/tests/queries/0_stateless/02570_merge_alias_prewhere.sql +++ b/tests/queries/0_stateless/02575_merge_prewhere_different_default_kind.sql @@ -1,3 +1,5 @@ +-- Prohibit PREWHERE when Merge and MergeTree has different default type of the column + DROP TABLE IF EXISTS m; DROP TABLE IF EXISTS t1; DROP TABLE IF EXISTS t2; diff --git a/tests/queries/0_stateless/02575_merge_prewhere_ephemeral.reference b/tests/queries/0_stateless/02575_merge_prewhere_ephemeral.reference new file mode 100644 index 00000000000..9f214b8c536 --- /dev/null +++ b/tests/queries/0_stateless/02575_merge_prewhere_ephemeral.reference @@ -0,0 +1,11 @@ +-- { echoOn } +SELECT * FROM m PREWHERE a = 'OK' ORDER BY a; +OK +OK +SELECT * FROM m PREWHERE f = 1 ORDER BY a; -- { serverError ILLEGAL_PREWHERE } +SELECT * FROM m WHERE a = 'OK' SETTINGS optimize_move_to_prewhere=0; +OK +OK +SELECT * FROM m WHERE a = 'OK' SETTINGS optimize_move_to_prewhere=1; +OK +OK diff --git a/tests/queries/0_stateless/02575_merge_prewhere_ephemeral.sql b/tests/queries/0_stateless/02575_merge_prewhere_ephemeral.sql new file mode 100644 index 00000000000..85e03647d62 --- /dev/null +++ b/tests/queries/0_stateless/02575_merge_prewhere_ephemeral.sql @@ -0,0 +1,38 @@ +-- You cannot query EPHEMERAL + +DROP TABLE IF EXISTS m; +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; + +CREATE TABLE m +( + `a` String, + `f` UInt8 EPHEMERAL 0 +) +ENGINE = Merge(currentDatabase(), '^(t1|t2)$'); + +CREATE TABLE t1 +( + a String, + f UInt8 DEFAULT 1 +) +ENGINE = MergeTree +ORDER BY tuple() +SETTINGS index_granularity = 8192; +INSERT INTO t1 (a) VALUES ('OK'); + +CREATE TABLE t2 +( + a String, + f UInt8 DEFAULT 2 +) +ENGINE = MergeTree +ORDER BY tuple() +SETTINGS index_granularity = 8192; +INSERT INTO t2 (a) VALUES ('OK'); + +-- { echoOn } +SELECT * FROM m PREWHERE a = 'OK' ORDER BY a; +SELECT * FROM m PREWHERE f = 1 ORDER BY a; -- { serverError ILLEGAL_PREWHERE } +SELECT * FROM m WHERE a = 'OK' SETTINGS optimize_move_to_prewhere=0; +SELECT * FROM m WHERE a = 'OK' SETTINGS optimize_move_to_prewhere=1; diff --git a/tests/queries/0_stateless/02575_merge_prewhere_materialized.reference b/tests/queries/0_stateless/02575_merge_prewhere_materialized.reference new file mode 100644 index 00000000000..434384b3d77 --- /dev/null +++ b/tests/queries/0_stateless/02575_merge_prewhere_materialized.reference @@ -0,0 +1,8 @@ +-- { echoOn } +SELECT * FROM m PREWHERE a = 'OK' ORDER BY a, f; +OK 1 +OK 2 +SELECT * FROM m PREWHERE f = 1 ORDER BY a, f; +OK 1 +SELECT * FROM m WHERE f = 0 SETTINGS optimize_move_to_prewhere=0; +SELECT * FROM m WHERE f = 0 SETTINGS optimize_move_to_prewhere=1; diff --git a/tests/queries/0_stateless/02575_merge_prewhere_materialized.sql b/tests/queries/0_stateless/02575_merge_prewhere_materialized.sql new file mode 100644 index 00000000000..46794b181a0 --- /dev/null +++ b/tests/queries/0_stateless/02575_merge_prewhere_materialized.sql @@ -0,0 +1,38 @@ +-- Allow PREWHERE when Merge has DEFAULT and MergeTree has MATERIALIZED + +DROP TABLE IF EXISTS m; +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; + +CREATE TABLE m +( + `a` String, + `f` UInt8 DEFAULT 0 +) +ENGINE = Merge(currentDatabase(), '^(t1|t2)$'); + +CREATE TABLE t1 +( + a String, + f UInt8 MATERIALIZED 1 +) +ENGINE = MergeTree +ORDER BY tuple() +SETTINGS index_granularity = 8192; +INSERT INTO t1 (a) VALUES ('OK'); + +CREATE TABLE t2 +( + a String, + f UInt8 DEFAULT 2 +) +ENGINE = MergeTree +ORDER BY tuple() +SETTINGS index_granularity = 8192; +INSERT INTO t2 (a) VALUES ('OK'); + +-- { echoOn } +SELECT * FROM m PREWHERE a = 'OK' ORDER BY a, f; +SELECT * FROM m PREWHERE f = 1 ORDER BY a, f; +SELECT * FROM m WHERE f = 0 SETTINGS optimize_move_to_prewhere=0; +SELECT * FROM m WHERE f = 0 SETTINGS optimize_move_to_prewhere=1; From e8458048a9edba42d173a5453e10e7511b98cfd7 Mon Sep 17 00:00:00 2001 From: HarryLeeIBM Date: Fri, 24 Feb 2023 07:33:57 -0800 Subject: [PATCH 642/717] Fix http xz compression issue for s390x --- contrib/xz-cmake/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/contrib/xz-cmake/CMakeLists.txt b/contrib/xz-cmake/CMakeLists.txt index 9d08adc9c7a..c3a8203c83e 100644 --- a/contrib/xz-cmake/CMakeLists.txt +++ b/contrib/xz-cmake/CMakeLists.txt @@ -94,6 +94,10 @@ if (ARCH_AMD64 OR ARCH_AARCH64) add_compile_definitions(TUKLIB_FAST_UNALIGNED_ACCESS=1) endif () +if (ARCH_S390X) + add_compile_definitions(WORDS_BIGENDIAN) +endif () + find_package(Threads REQUIRED) From e4e19c90e7e3d2803a7487e05e9bb51f150efb8a Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 22 Feb 2023 16:18:34 +0000 Subject: [PATCH 643/717] Do not reanalyze expressions from aggregation in projection --- src/Planner/PlannerActionsVisitor.cpp | 20 ++++++++++++++++++- ...8_analyzer_array_join_calc_twice.reference | 6 ++++++ .../25338_analyzer_array_join_calc_twice.sql | 5 +++++ 3 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/25338_analyzer_array_join_calc_twice.reference create mode 100644 tests/queries/0_stateless/25338_analyzer_array_join_calc_twice.sql diff --git a/src/Planner/PlannerActionsVisitor.cpp b/src/Planner/PlannerActionsVisitor.cpp index 2c5c8ab5e62..614e5c03bc1 100644 --- a/src/Planner/PlannerActionsVisitor.cpp +++ b/src/Planner/PlannerActionsVisitor.cpp @@ -65,6 +65,15 @@ public: return node_name_to_node.find(node_name) != node_name_to_node.end(); } + [[maybe_unused]] bool containsInputNode(const std::string & node_name) + { + const auto * node = tryGetNode(node_name); + if (node && node->type == ActionsDAG::ActionType::INPUT) + return true; + + return false; + } + [[maybe_unused]] const ActionsDAG::Node * tryGetNode(const std::string & node_name) { auto it = node_name_to_node.find(node_name); @@ -421,7 +430,16 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi auto function_node_name = calculateActionNodeName(node, *planner_context, node_to_node_name); - if (function_node.isAggregateFunction() || function_node.isWindowFunction()) + /* Aggregate functions, window functions, and GROUP BY expressions were already analyzed in the previous steps. + * If we have already visited some expression, we don't need to revisit it or its arguments again. + * For example, the expression from the aggregation step is also present in the projection: + * SELECT foo(a, b, c) as x FROM table GROUP BY foo(a, b, c) + * In this case we should not analyze `a`, `b`, `c` again. + * Moreover, it can lead to an error if we have arrayJoin in the arguments because it will be calculated twice. + */ + bool is_input_node = function_node.isAggregateFunction() || function_node.isWindowFunction() + || actions_stack.front().containsInputNode(function_node_name); + if (is_input_node) { size_t actions_stack_size = actions_stack.size(); diff --git a/tests/queries/0_stateless/25338_analyzer_array_join_calc_twice.reference b/tests/queries/0_stateless/25338_analyzer_array_join_calc_twice.reference new file mode 100644 index 00000000000..bc42121fb39 --- /dev/null +++ b/tests/queries/0_stateless/25338_analyzer_array_join_calc_twice.reference @@ -0,0 +1,6 @@ +2 +3 +4 +2 +3 +4 diff --git a/tests/queries/0_stateless/25338_analyzer_array_join_calc_twice.sql b/tests/queries/0_stateless/25338_analyzer_array_join_calc_twice.sql new file mode 100644 index 00000000000..b6bb258db28 --- /dev/null +++ b/tests/queries/0_stateless/25338_analyzer_array_join_calc_twice.sql @@ -0,0 +1,5 @@ +SET allow_experimental_analyzer = 1; + +SELECT 1 + arrayJoin(a) AS m FROM (SELECT [1, 2, 3] AS a) GROUP BY m; + +SELECT 1 + arrayJoin(a) AS m FROM (SELECT [1, 2, 3] AS a) GROUP BY 1 + arrayJoin(a); From b0abeefb74155c33e354e27e35b4c00d82eb5c0e Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 24 Feb 2023 15:52:32 +0000 Subject: [PATCH 644/717] Rectify string literals escaping docs Cf. https://github.com/ClickHouse/ClickHouse/issues/46592#issuecomment-1437059603 --- docs/en/sql-reference/syntax.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/syntax.md b/docs/en/sql-reference/syntax.md index 4c2054274e4..bde37241ed0 100644 --- a/docs/en/sql-reference/syntax.md +++ b/docs/en/sql-reference/syntax.md @@ -83,9 +83,13 @@ Examples: `1`, `10_000_000`, `0xffff_ffff`, `18446744073709551615`, `0xDEADBEEF` ### String -Only string literals in single quotes are supported. The enclosed characters can be backslash-escaped. The following escape sequences have a corresponding special value: `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\a`, `\v`, `\xHH`. In all other cases, escape sequences in the format `\c`, where `c` is any character, are converted to `c`. It means that you can use the sequences `\'`and`\\`. The value will have the [String](../sql-reference/data-types/string.md) type. +String literals must be enclosed in single quotes, double quotes are not supported. +Escaping works either -In string literals, you need to escape at least `'` and `\`. Single quotes can be escaped with the single quote, literals `'It\'s'` and `'It''s'` are equal. +- in SQL-style based on a preceding single quote where the single-quote character `'` (and only this character) can be escaped as `''`, or +- in C-style based on a preceding backslash with the following supported escape sequences: `\\`, `\'`, `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\a`, `\v`, `\xHH`. The backslash loses its special meaning and will be interpreted literally if it precedes characters different than the listed ones. + +In string literals, you need to escape at least `'` and `\` using escape codes `''` (or: `\'`) and `\\`. ### Compound From 6402050125923c278164c17abec7ebf87994ae25 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 24 Feb 2023 16:59:38 +0100 Subject: [PATCH 645/717] Fixup comments --- src/Storages/Kafka/StorageKafka.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index 5eb4dd19829..78a11c56409 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -616,16 +616,16 @@ void StorageKafka::updateConfiguration(cppkafka::Configuration & kafka_config) // Update consumer topic-specific configuration (new syntax). Example with topics "football" and "baseball": // - // + // // football // 250 // 5000 - // - // + // + // // baseball // 300 // 2000 - // + // // // Advantages: The period restriction no longer applies (e.g. sports.football will work), everything // Kafka-related is below . From 9974f45ab0399618a93ae06e8ecbe663ab3ba45e Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Fri, 24 Feb 2023 13:15:35 -0400 Subject: [PATCH 646/717] Update table.md --- .../sql-reference/statements/create/table.md | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md index 119f25d6d00..20efa0eb4e1 100644 --- a/docs/en/sql-reference/statements/create/table.md +++ b/docs/en/sql-reference/statements/create/table.md @@ -127,6 +127,26 @@ Default expressions may be defined as an arbitrary expression from table constan Normal default value. If the INSERT query does not specify the corresponding column, it will be filled in by computing the corresponding expression. +Example: + +```sql +CREATE OR REPLACE TABLE test +( + id UInt64, + updated_at DateTime DEFAULT now(), + updated_at_date Date DEFAULT toDate(updated_at) +) +ENGINE = MergeTree +ORDER BY id; + +INSERT INTO test (id) Values (1); + +SELECT * FROM test; +┌─id─┬──────────updated_at─┬─updated_at_date─┐ +│ 1 │ 2023-02-24 17:06:46 │ 2023-02-24 │ +└────┴─────────────────────┴─────────────────┘ +``` + ### MATERIALIZED `MATERIALIZED expr` @@ -135,6 +155,37 @@ Materialized expression. Such a column can’t be specified for INSERT, because For an INSERT without a list of columns, these columns are not considered. In addition, this column is not substituted when using an asterisk in a SELECT query. This is to preserve the invariant that the dump obtained using `SELECT *` can be inserted back into the table using INSERT without specifying the list of columns. +Example: + +```sql +CREATE OR REPLACE TABLE test +( + id UInt64, + updated_at DateTime MATERIALIZED now(), + updated_at_date Date MATERIALIZED toDate(updated_at) +) +ENGINE = MergeTree +ORDER BY id; + +INSERT INTO test Values (1); + +SELECT * FROM test; +┌─id─┐ +│ 1 │ +└────┘ + + +SELECT id, updated_at, updated_at_date FROM test; +┌─id─┬──────────updated_at─┬─updated_at_date─┐ +│ 1 │ 2023-02-24 17:08:08 │ 2023-02-24 │ +└────┴─────────────────────┴─────────────────┘ + +SELECT * FROM test SETTINGS asterisk_include_materialized_columns=1; +┌─id─┬──────────updated_at─┬─updated_at_date─┐ +│ 1 │ 2023-02-24 17:08:08 │ 2023-02-24 │ +└────┴─────────────────────┴─────────────────┘ +``` + ### EPHEMERAL `EPHEMERAL [expr]` @@ -142,6 +193,34 @@ In addition, this column is not substituted when using an asterisk in a SELECT q Ephemeral column. Such a column isn't stored in the table and cannot be SELECTed, but can be referenced in the defaults of CREATE statement. If `expr` is omitted type for column is required. INSERT without list of columns will skip such column, so SELECT/INSERT invariant is preserved - the dump obtained using `SELECT *` can be inserted back into the table using INSERT without specifying the list of columns. +Example: + +```sql +CREATE OR REPLACE TABLE test +( + id UInt64, + unhexed String EPHEMERAL, + hexed FixedString(4) DEFAULT unhex(unhexed) +) +ENGINE = MergeTree +ORDER BY id + +INSERT INTO test (id, unhexed) Values (1, '5a90b714'); + +SELECT + id, + hexed, + hex(hexed) +FROM test +FORMAT Vertical; + +Row 1: +────── +id: 1 +hexed: Z�� +hex(hexed): 5A90B714 +``` + ### ALIAS `ALIAS expr` @@ -156,6 +235,29 @@ If you add a new column to a table but later change its default expression, the It is not possible to set default values for elements in nested data structures. +```sql +CREATE OR REPLACE TABLE test +( + id UInt64, + size_bytes Int64, + size String Alias formatReadableSize(size_bytes) +) +ENGINE = MergeTree +ORDER BY id; + +INSERT INTO test Values (1, 4678899); + +SELECT id, size_bytes, size FROM test; +┌─id─┬─size_bytes─┬─size─────┐ +│ 1 │ 4678899 │ 4.46 MiB │ +└────┴────────────┴──────────┘ + +SELECT * FROM test SETTINGS asterisk_include_alias_columns=1; +┌─id─┬─size_bytes─┬─size─────┐ +│ 1 │ 4678899 │ 4.46 MiB │ +└────┴────────────┴──────────┘ +``` + ## Primary Key You can define a [primary key](../../../engines/table-engines/mergetree-family/mergetree.md#primary-keys-and-indexes-in-queries) when creating a table. Primary key can be specified in two ways: From 95b701a37caaf4255819adcc4e097570455f0611 Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Fri, 24 Feb 2023 13:17:30 -0400 Subject: [PATCH 647/717] Update table.md --- docs/en/sql-reference/statements/create/table.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md index 20efa0eb4e1..9e66afba613 100644 --- a/docs/en/sql-reference/statements/create/table.md +++ b/docs/en/sql-reference/statements/create/table.md @@ -174,7 +174,6 @@ SELECT * FROM test; │ 1 │ └────┘ - SELECT id, updated_at, updated_at_date FROM test; ┌─id─┬──────────updated_at─┬─updated_at_date─┐ │ 1 │ 2023-02-24 17:08:08 │ 2023-02-24 │ From 1fbe96d20c28afc0611bd5599c942e19798aa61b Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Fri, 24 Feb 2023 18:08:51 +0000 Subject: [PATCH 648/717] fix mysql protocol --- src/Server/MySQLHandler.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/Server/MySQLHandler.cpp b/src/Server/MySQLHandler.cpp index e3467d664e7..d8ea359ce5f 100644 --- a/src/Server/MySQLHandler.cpp +++ b/src/Server/MySQLHandler.cpp @@ -354,9 +354,13 @@ void MySQLHandler::comQuery(ReadBuffer & payload) auto set_result_details = [&with_output](const QueryResultDetails & details) { - if (details.format && *details.format != "MySQLWire") - throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "MySQL protocol does not support custom output formats"); - with_output = true; + if (details.format) + { + if (*details.format != "MySQLWire") + throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "MySQL protocol does not support custom output formats"); + + with_output = true; + } }; executeQuery(should_replace ? replacement : payload, *out, false, query_context, set_result_details, format_settings); From 25eb4053eb9e5aacf127e7276fb184721a42140d Mon Sep 17 00:00:00 2001 From: rfraposa Date: Fri, 24 Feb 2023 11:26:37 -0700 Subject: [PATCH 649/717] Update install.md --- docs/en/getting-started/install.md | 48 +++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/docs/en/getting-started/install.md b/docs/en/getting-started/install.md index 404c6c6f227..6a15c32e9dd 100644 --- a/docs/en/getting-started/install.md +++ b/docs/en/getting-started/install.md @@ -5,6 +5,10 @@ description: Install ClickHouse slug: /en/install --- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; + # Install ClickHouse You have three options for getting up and running with ClickHouse: @@ -19,17 +23,23 @@ The quickest and easiest way to get up and running with ClickHouse is to create ## Self-Managed Install + + + 1. The simplest way to download ClickHouse locally is to run the following command. If your operating system is supported, an appropriate ClickHouse binary will be downloaded and made runnable: + ```bash curl https://clickhouse.com/ | sh ``` 1. Run the `install` command, which defines a collection of useful symlinks along with the files and folders used by ClickHouse - all of which you can see in the output of the install script: + ```bash sudo ./clickhouse install ``` 1. At the end of the install script, you are prompted for a password for the `default` user. Feel free to enter a password, or you can optionally leave it blank: + ```response Creating log directory /var/log/clickhouse-server. Creating data directory /var/lib/clickhouse. @@ -40,6 +50,7 @@ The quickest and easiest way to get up and running with ClickHouse is to create Enter password for default user: ``` You should see the following output: + ```response ClickHouse has been successfully installed. @@ -51,10 +62,45 @@ The quickest and easiest way to get up and running with ClickHouse is to create ``` 1. Run the following command to start the ClickHouse server: + ```bash + sudo clickhouse start + ``` + + + + +1. The simplest way to download ClickHouse locally is to run the following command. If your operating system is supported, an appropriate ClickHouse binary will be downloaded and made runnable: ```bash - sudo clickhouse start + curl https://clickhouse.com/ | sh ``` +1. Run the ClickHouse server: + + ```bash + ./clickhouse server + ``` + +1. Open a new terminal and use the **clickhouse-client** to connect to your service: + + ```bash + ./clickhouse client + ``` + + ```response + ./clickhouse client + ClickHouse client version 23.2.1.1501 (official build). + Connecting to localhost:9000 as user default. + Connected to ClickHouse server version 23.2.1 revision 54461. + + local-host :) + ``` + + You are ready to start sending DDL and SQL commands to ClickHouse! + + + + + :::tip The [Quick Start](/docs/en/quick-start.mdx/#step-1-get-clickhouse) walks through the steps to download and run ClickHouse, connect to it, and insert data. ::: From 7b1b4f1fd6e851090ddd9f8a7a646c29ab2abd3a Mon Sep 17 00:00:00 2001 From: rfraposa Date: Fri, 24 Feb 2023 11:54:12 -0700 Subject: [PATCH 650/717] Update aspell-dict.txt --- utils/check-style/aspell-ignore/en/aspell-dict.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index 00477f0fb8f..ab03da8cc80 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -24,6 +24,7 @@ CapnProto CentOS ClickHouse ClickHouse's +CodeBlock Config ConnectionDetails Contrib @@ -151,6 +152,7 @@ Submodules Subqueries TSVRaw TSan +TabItem TabSeparated TabSeparatedRaw TabSeparatedRawWithNames From 83071164cc8ba5d9b965557a4dbc2d05f62559f6 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 24 Feb 2023 20:13:08 +0100 Subject: [PATCH 651/717] Fix off-by-one error in netloc() Signed-off-by: Azat Khuzhin --- src/Functions/URL/netloc.cpp | 2 +- tests/queries/0_stateless/00398_url_functions.reference | 1 + tests/queries/0_stateless/00398_url_functions.sql.j2 | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Functions/URL/netloc.cpp b/src/Functions/URL/netloc.cpp index 5c23f7cc06d..fe953e5765c 100644 --- a/src/Functions/URL/netloc.cpp +++ b/src/Functions/URL/netloc.cpp @@ -108,7 +108,7 @@ struct ExtractNetloc case '=': case '&': return pos > start_of_host - ? std::string_view(start_of_host, std::min(std::min(pos - 1, question_mark_pos), slash_pos) - start_of_host) + ? std::string_view(start_of_host, std::min(std::min(pos, question_mark_pos), slash_pos) - start_of_host) : std::string_view(); } } diff --git a/tests/queries/0_stateless/00398_url_functions.reference b/tests/queries/0_stateless/00398_url_functions.reference index 39d740e55cd..ff84e20f00f 100644 --- a/tests/queries/0_stateless/00398_url_functions.reference +++ b/tests/queries/0_stateless/00398_url_functions.reference @@ -53,6 +53,7 @@ paul:zozo@example.ru www.example.com www.example.com example.com +foo:foo ====DOMAIN==== com diff --git a/tests/queries/0_stateless/00398_url_functions.sql.j2 b/tests/queries/0_stateless/00398_url_functions.sql.j2 index dd7da2ce6ad..80314b19385 100644 --- a/tests/queries/0_stateless/00398_url_functions.sql.j2 +++ b/tests/queries/0_stateless/00398_url_functions.sql.j2 @@ -41,6 +41,7 @@ SELECT netloc('svn+ssh://paul:zozo@example.ru/?q=hello%20world') AS Netloc; SELECT netloc('//www.example.com') AS Netloc; SELECT netloc('www.example.com') as Netloc; SELECT netloc('example.com') as Netloc; +SELECT netloc('http://foo:foo%@foo.com') as Netloc; SELECT '====DOMAIN===='; SELECT topLevelDomain('http://paul@www.example.com:80/') AS Domain; From 381f0cc5b38efa4a0d863ad4630106b1d8127329 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 24 Feb 2023 20:44:47 +0100 Subject: [PATCH 652/717] Edit the changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e4785892c3..e22377e2332 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,7 @@ * Add two new functions which allow for user-defined keys/seeds with SipHash{64,128}. [#45513](https://github.com/ClickHouse/ClickHouse/pull/45513) ([Salvatore Mesoraca](https://github.com/aiven-sal)). * Allow a three-argument version for table function `format`. close [#45808](https://github.com/ClickHouse/ClickHouse/issues/45808). [#45873](https://github.com/ClickHouse/ClickHouse/pull/45873) ([FFFFFFFHHHHHHH](https://github.com/FFFFFFFHHHHHHH)). * Add `JodaTime` format support for 'x','w','S'. Refer to https://joda-time.sourceforge.net/apidocs/org/joda/time/format/DateTimeFormat.html. [#46073](https://github.com/ClickHouse/ClickHouse/pull/46073) ([zk_kiger](https://github.com/zk-kiger)). -* Support window function `ntile`. +* Support window function `ntile`. ([lgbo](https://github.com/lgbo-ustc)). * Add setting `final` to implicitly apply the `FINAL` modifier to every table. [#40945](https://github.com/ClickHouse/ClickHouse/pull/40945) ([Arthur Passos](https://github.com/arthurpassos)). * Added `arrayPartialSort` and `arrayPartialReverseSort` functions. [#46296](https://github.com/ClickHouse/ClickHouse/pull/46296) ([Joanna Hulboj](https://github.com/jh0x)). * The new http parameter `client_protocol_version` allows setting a client protocol version for HTTP responses using the Native format. [#40397](https://github.com/ClickHouse/ClickHouse/issues/40397). [#46360](https://github.com/ClickHouse/ClickHouse/pull/46360) ([Geoff Genz](https://github.com/genzgd)). From 066389e6ff78fb9d1812691f43b33f507baed70a Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 24 Feb 2023 20:20:49 +0100 Subject: [PATCH 653/717] Relax symbols that are allowed in userinfo in netloc() Signed-off-by: Azat Khuzhin --- src/Functions/URL/netloc.cpp | 28 +++++++++++++++---- .../0_stateless/00398_url_functions.reference | 5 +++- .../0_stateless/00398_url_functions.sql.j2 | 3 ++ 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/src/Functions/URL/netloc.cpp b/src/Functions/URL/netloc.cpp index fe953e5765c..bc34e34a40d 100644 --- a/src/Functions/URL/netloc.cpp +++ b/src/Functions/URL/netloc.cpp @@ -7,6 +7,7 @@ namespace DB { +/// NOTE: Implementation is not RFC3986 compatible struct ExtractNetloc { /// We use the same as domain function @@ -67,6 +68,7 @@ struct ExtractNetloc /// Now pos points to the first byte after scheme (if there is). bool has_identification = false; + Pos hostname_end = end; Pos question_mark_pos = end; Pos slash_pos = end; Pos start_of_host = pos; @@ -90,23 +92,37 @@ struct ExtractNetloc return std::string_view(start_of_host, pos - start_of_host); case '@': /// foo:bar@example.ru has_identification = true; + hostname_end = end; break; + case ';': + case '=': + case '&': + case '~': + case '%': + /// Symbols above are sub-delims in RFC3986 and should be + /// allowed for userinfo (named identification here). + /// + /// NOTE: that those symbols is allowed for reg-name (host) + /// too, but right now host parsing looks more like in + /// RFC1034 (in other words domains that are allowed to be + /// registered). + if (!has_identification) + { + hostname_end = pos; + break; + } + [[fallthrough]]; case ' ': /// restricted symbols in whole URL case '\t': case '<': case '>': - case '%': case '{': case '}': case '|': case '\\': case '^': - case '~': case '[': case ']': - case ';': - case '=': - case '&': return pos > start_of_host ? std::string_view(start_of_host, std::min(std::min(pos, question_mark_pos), slash_pos) - start_of_host) : std::string_view(); @@ -116,7 +132,7 @@ struct ExtractNetloc if (has_identification) return std::string_view(start_of_host, pos - start_of_host); else - return std::string_view(start_of_host, std::min(std::min(pos, question_mark_pos), slash_pos) - start_of_host); + return std::string_view(start_of_host, std::min(std::min(std::min(pos, question_mark_pos), slash_pos), hostname_end) - start_of_host); } static void execute(Pos data, size_t size, Pos & res_data, size_t & res_size) diff --git a/tests/queries/0_stateless/00398_url_functions.reference b/tests/queries/0_stateless/00398_url_functions.reference index ff84e20f00f..c5982afcfaa 100644 --- a/tests/queries/0_stateless/00398_url_functions.reference +++ b/tests/queries/0_stateless/00398_url_functions.reference @@ -53,7 +53,10 @@ paul:zozo@example.ru www.example.com www.example.com example.com -foo:foo +foo:foo%@foo.com +foo:foo%41bar@foo.com +foo:foo%41%42bar@foo.com +foo:foo%41bar@foo ====DOMAIN==== com diff --git a/tests/queries/0_stateless/00398_url_functions.sql.j2 b/tests/queries/0_stateless/00398_url_functions.sql.j2 index 80314b19385..b96b71ca59e 100644 --- a/tests/queries/0_stateless/00398_url_functions.sql.j2 +++ b/tests/queries/0_stateless/00398_url_functions.sql.j2 @@ -42,6 +42,9 @@ SELECT netloc('//www.example.com') AS Netloc; SELECT netloc('www.example.com') as Netloc; SELECT netloc('example.com') as Netloc; SELECT netloc('http://foo:foo%@foo.com') as Netloc; +SELECT netloc('http://foo:foo%41bar@foo.com') as Netloc; +SELECT netloc('http://foo:foo%41%42bar@foo.com') as Netloc; +SELECT netloc('http://foo:foo%41bar@foo%41.com') as Netloc; SELECT '====DOMAIN===='; SELECT topLevelDomain('http://paul@www.example.com:80/') AS Domain; From 78bc48236bd6ab356ce67dac20543b18ea52b23e Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 24 Feb 2023 20:41:05 +0100 Subject: [PATCH 654/717] Relax symbols that are allowed in userinfo in *domain*RFC() Signed-off-by: Azat Khuzhin --- src/Functions/URL/domain.h | 23 +++++++++++++++---- .../0_stateless/00398_url_functions.reference | 10 ++++++-- .../0_stateless/00398_url_functions.sql.j2 | 5 +++- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/src/Functions/URL/domain.h b/src/Functions/URL/domain.h index 64362edf2c3..91f83a457be 100644 --- a/src/Functions/URL/domain.h +++ b/src/Functions/URL/domain.h @@ -75,6 +75,7 @@ exloop: if ((scheme_end - pos) > 2 && *pos == ':' && *(pos + 1) == '/' && *(pos Pos dot_pos = nullptr; Pos colon_pos = nullptr; + bool has_sub_delims = false; bool has_at_symbol = false; bool has_terminator_after_colon = false; const auto * start_of_host = pos; @@ -97,25 +98,35 @@ exloop: if ((scheme_end - pos) > 2 && *pos == ':' && *(pos + 1) == '/' && *(pos case '@': /// myemail@gmail.com if (has_terminator_after_colon) return std::string_view{}; if (has_at_symbol) goto done; + has_sub_delims = false; has_at_symbol = true; start_of_host = pos + 1; break; + case ';': + case '=': + case '&': + case '~': + case '%': + /// Symbols above are sub-delims in RFC3986 and should be + /// allowed for userinfo (named identification here). + /// + /// NOTE: that those symbols is allowed for reg-name (host) + /// too, but right now host parsing looks more like in + /// RFC1034 (in other words domains that are allowed to be + /// registered). + has_sub_delims = true; + continue; case ' ': /// restricted symbols in whole URL case '\t': case '<': case '>': - case '%': case '{': case '}': case '|': case '\\': case '^': - case '~': case '[': case ']': - case ';': - case '=': - case '&': if (colon_pos == nullptr) return std::string_view{}; else @@ -124,6 +135,8 @@ exloop: if ((scheme_end - pos) > 2 && *pos == ':' && *(pos + 1) == '/' && *(pos } done: + if (has_sub_delims) + return std::string_view{}; if (!has_at_symbol) pos = colon_pos ? colon_pos : pos; return checkAndReturnHost(pos, dot_pos, start_of_host); diff --git a/tests/queries/0_stateless/00398_url_functions.reference b/tests/queries/0_stateless/00398_url_functions.reference index c5982afcfaa..6913b3f4fb7 100644 --- a/tests/queries/0_stateless/00398_url_functions.reference +++ b/tests/queries/0_stateless/00398_url_functions.reference @@ -20,7 +20,10 @@ www.example.com 127.0.0.1 www.example.com www.example.com -www.example.com + + + +www.example4.com example.com example.com example.com @@ -38,7 +41,10 @@ www.example.com 127.0.0.1 www.example.com www.example.com -www.example.com + +example2.com +example3.com +www.example4.com example.com example.com example.com diff --git a/tests/queries/0_stateless/00398_url_functions.sql.j2 b/tests/queries/0_stateless/00398_url_functions.sql.j2 index b96b71ca59e..95ac536f18b 100644 --- a/tests/queries/0_stateless/00398_url_functions.sql.j2 +++ b/tests/queries/0_stateless/00398_url_functions.sql.j2 @@ -23,7 +23,10 @@ SELECT domain{{ suffix }}('http://www.example.com?q=4') AS Host; SELECT domain{{ suffix }}('http://127.0.0.1:443/') AS Host; SELECT domain{{ suffix }}('//www.example.com') AS Host; SELECT domain{{ suffix }}('//paul@www.example.com') AS Host; -SELECT domain{{ suffix }}('www.example.com') as Host; +SELECT domain{{ suffix }}('//foo:bar%41%40@e-%41-example1.com') AS Host; +SELECT domain{{ suffix }}('//foo:bar%41%40@example2.com') AS Host; +SELECT domain{{ suffix }}('//foo%41%40:bar@example3.com') AS Host; +SELECT domain{{ suffix }}('www.example4.com') as Host; SELECT domain{{ suffix }}('example.com') as Host; SELECT domainWithoutWWW{{ suffix }}('//paul@www.example.com') AS Host; SELECT domainWithoutWWW{{ suffix }}('http://paul@www.example.com:80/') AS Host; From 3d29e62587ef6d699c28cd14fb0b6b2ac3024645 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 24 Feb 2023 22:33:39 +0100 Subject: [PATCH 655/717] Fix strange code --- src/DataTypes/registerDataTypeDateTime.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DataTypes/registerDataTypeDateTime.cpp b/src/DataTypes/registerDataTypeDateTime.cpp index 5a3e10656b4..8080179ad47 100644 --- a/src/DataTypes/registerDataTypeDateTime.cpp +++ b/src/DataTypes/registerDataTypeDateTime.cpp @@ -64,7 +64,7 @@ static DataTypePtr create(const ASTPtr & arguments) return std::make_shared(); const auto scale = getArgument(arguments, 0, "scale", "DateTime"); - const auto timezone = getArgument(arguments, !!scale, "timezone", "DateTime"); + const auto timezone = getArgument(arguments, scale ? 1 : 0, "timezone", "DateTime"); if (!scale && !timezone) throw Exception::createDeprecated(getExceptionMessage(" has wrong type: ", 0, "scale", "DateTime", Field::Types::Which::UInt64), From d6b618565e3a9e2af16b3fdbba86604139853b57 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 24 Feb 2023 22:35:51 +0100 Subject: [PATCH 656/717] Add a trap --- src/Functions/trap.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Functions/trap.cpp b/src/Functions/trap.cpp index 44deb901b0d..e359a5513ce 100644 --- a/src/Functions/trap.cpp +++ b/src/Functions/trap.cpp @@ -174,6 +174,10 @@ public: if (!handle) throw Exception(ErrorCodes::CANNOT_DLOPEN, "Cannot dlopen: ({})", dlerror()); // NOLINT(concurrency-mt-unsafe) // MT-Safe on Linux, see man dlerror } + else if (mode == "logical error") + { + throw Exception(ErrorCodes::LOGICAL_ERROR, "Logical error: trap"); + } else throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown trap mode"); } From f28ab14759372f787c79e12d54efe30b503fcb48 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 24 Feb 2023 23:29:06 +0100 Subject: [PATCH 657/717] Better handling of fatal errors --- src/Common/ThreadStatus.cpp | 1 + src/Daemon/BaseDaemon.cpp | 17 ++++++++++++++++- src/Server/TCPHandler.cpp | 2 ++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/Common/ThreadStatus.cpp b/src/Common/ThreadStatus.cpp index 9f9a78c4036..46c171b5cb6 100644 --- a/src/Common/ThreadStatus.cpp +++ b/src/Common/ThreadStatus.cpp @@ -237,6 +237,7 @@ void ThreadStatus::setFatalErrorCallback(std::function callback) void ThreadStatus::onFatalError() { + std::lock_guard lock(thread_group->mutex); if (fatal_error_callback) fatal_error_callback(); } diff --git a/src/Daemon/BaseDaemon.cpp b/src/Daemon/BaseDaemon.cpp index 4056e27ad52..60179fd5317 100644 --- a/src/Daemon/BaseDaemon.cpp +++ b/src/Daemon/BaseDaemon.cpp @@ -134,6 +134,8 @@ static void terminateRequestedSignalHandler(int sig, siginfo_t *, void *) } +static std::atomic fatal_error_printed{false}; + /** Handler for "fault" or diagnostic signals. Send data about fault to separate thread to write into log. */ static void signalHandler(int sig, siginfo_t * info, void * context) @@ -159,7 +161,16 @@ static void signalHandler(int sig, siginfo_t * info, void * context) if (sig != SIGTSTP) /// This signal is used for debugging. { /// The time that is usually enough for separate thread to print info into log. - sleepForSeconds(20); /// FIXME: use some feedback from threads that process stacktrace + /// Under MSan full stack unwinding with DWARF info about inline functions takes 101 seconds in one case. + for (size_t i = 0; i < 300; ++i) + { + /// We will synchronize with the thread printing the messages with an atomic variable to finish earlier. + if (fatal_error_printed) + break; + + /// This coarse method of synchronization is perfectly ok for fatal signals. + sleepForSeconds(1); + } call_default_signal_handler(sig); } @@ -309,7 +320,9 @@ private: } if (auto logs_queue = thread_ptr->getInternalTextLogsQueue()) + { DB::CurrentThread::attachInternalTextLogsQueue(logs_queue, DB::LogsLevel::trace); + } } std::string signal_description = "Unknown signal"; @@ -407,6 +420,8 @@ private: /// When everything is done, we will try to send these error messages to client. if (thread_ptr) thread_ptr->onFatalError(); + + fatal_error_printed = true; } }; diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 870a34fa665..a307b472a64 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -611,6 +611,8 @@ void TCPHandler::runImpl() /// It is important to destroy query context here. We do not want it to live arbitrarily longer than the query. query_context.reset(); + CurrentThread::setFatalErrorCallback({}); + if (is_interserver_mode) { /// We don't really have session in interserver mode, new one is created for each query. It's better to reset it now. From 7c3a7ea27a35d013c3d7f21b67f582ebc0bf4dac Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 25 Feb 2023 00:13:15 +0100 Subject: [PATCH 658/717] Fix MSan report in "maxIntersections" function --- tests/queries/0_stateless/02670_max_intersections.reference | 1 + tests/queries/0_stateless/02670_max_intersections.sql | 1 + 2 files changed, 2 insertions(+) create mode 100644 tests/queries/0_stateless/02670_max_intersections.reference create mode 100644 tests/queries/0_stateless/02670_max_intersections.sql diff --git a/tests/queries/0_stateless/02670_max_intersections.reference b/tests/queries/0_stateless/02670_max_intersections.reference new file mode 100644 index 00000000000..30e9e0b1342 --- /dev/null +++ b/tests/queries/0_stateless/02670_max_intersections.reference @@ -0,0 +1 @@ +04010000000000000001000000000000000300000000000000FFFFFFFFFFFFFFFF030000000000000001000000000000000500000000000000FFFFFFFFFFFFFFFF diff --git a/tests/queries/0_stateless/02670_max_intersections.sql b/tests/queries/0_stateless/02670_max_intersections.sql new file mode 100644 index 00000000000..5ac865222fe --- /dev/null +++ b/tests/queries/0_stateless/02670_max_intersections.sql @@ -0,0 +1 @@ +SELECT hex(maxIntersectionsState(*)) FROM VALUES((1, 3), (3, 5)); From e23d68bcba020e58d8ae43a6efc297361f960b6c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 25 Feb 2023 00:14:32 +0100 Subject: [PATCH 659/717] Fix MSan report in "maxIntersections" function --- .../AggregateFunctionMaxIntersections.h | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/AggregateFunctions/AggregateFunctionMaxIntersections.h b/src/AggregateFunctions/AggregateFunctionMaxIntersections.h index 9737e2e8257..d132c5706ab 100644 --- a/src/AggregateFunctions/AggregateFunctionMaxIntersections.h +++ b/src/AggregateFunctions/AggregateFunctionMaxIntersections.h @@ -117,7 +117,16 @@ public: const auto & value = this->data(place).value; size_t size = value.size(); writeVarUInt(size, buf); - buf.write(reinterpret_cast(value.data()), size * sizeof(value[0])); + + for (size_t i = 0; i < size; ++i) + { + /// In this version, pairs were serialized with padding. + /// We must ensure that padding bytes are zero-filled. + char bytes[sizeof(value[0])]{}; + unalignedStore(&bytes[offsetof(typename MaxIntersectionsData::Value, first)], value[i].first); + unalignedStore(&bytes[offsetof(typename MaxIntersectionsData::Value, second)], value[i].second); + buf.write(bytes, sizeof(value[0])); + } } void deserialize(AggregateDataPtr __restrict place, ReadBuffer & buf, std::optional /* version */, Arena * arena) const override From 150f8ec7bff42252f1efde50379b1c1aa09e3551 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 25 Feb 2023 00:22:57 +0100 Subject: [PATCH 660/717] Add a test for #43184 --- tests/queries/0_stateless/02671_quantile_fuse_msan.reference | 2 ++ tests/queries/0_stateless/02671_quantile_fuse_msan.sql | 5 +++++ 2 files changed, 7 insertions(+) create mode 100644 tests/queries/0_stateless/02671_quantile_fuse_msan.reference create mode 100644 tests/queries/0_stateless/02671_quantile_fuse_msan.sql diff --git a/tests/queries/0_stateless/02671_quantile_fuse_msan.reference b/tests/queries/0_stateless/02671_quantile_fuse_msan.reference new file mode 100644 index 00000000000..f6eca3b9698 --- /dev/null +++ b/tests/queries/0_stateless/02671_quantile_fuse_msan.reference @@ -0,0 +1,2 @@ +1970-01-01 00:00:00 1970-01-01 00:00:00 +1 diff --git a/tests/queries/0_stateless/02671_quantile_fuse_msan.sql b/tests/queries/0_stateless/02671_quantile_fuse_msan.sql new file mode 100644 index 00000000000..efeef0b0ebf --- /dev/null +++ b/tests/queries/0_stateless/02671_quantile_fuse_msan.sql @@ -0,0 +1,5 @@ +SET optimize_syntax_fuse_functions=1; +CREATE TEMPORARY TABLE datetime (`d` DateTime('UTC')); +SELECT quantile(0.1)(d), quantile(0.5)(d) FROM datetime; +INSERT INTO datetime SELECT * FROM generateRandom() LIMIT 10; +SELECT max(cityHash64(*)) > 0 FROM (SELECT quantile(0.1)(d), quantile(0.5)(d) FROM datetime); From ae230e3a823417d03dfec6c1b723ae9634dbe0f5 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 25 Feb 2023 02:24:29 +0300 Subject: [PATCH 661/717] Update trap.cpp --- src/Functions/trap.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Functions/trap.cpp b/src/Functions/trap.cpp index e359a5513ce..6260056ef31 100644 --- a/src/Functions/trap.cpp +++ b/src/Functions/trap.cpp @@ -27,6 +27,7 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; extern const int CANNOT_ALLOCATE_MEMORY; extern const int CANNOT_DLOPEN; + extern const int LOGICAL_ERROR; } From a712fc6435c85d8c2c0ab8abd314c06ec4a439a7 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 25 Feb 2023 00:27:54 +0100 Subject: [PATCH 662/717] Fix wrong function name --- src/Columns/ColumnVector.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 96f76b70f31..3ed3ed73328 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -747,7 +747,7 @@ namespace */ template requires (std::is_same_v || std::is_same_v) - void replicateSSE42Int32(const IntType * __restrict data, IntType * __restrict result_data, const IColumn::Offsets & offsets) + void replicateSSE2Int32(const IntType * __restrict data, IntType * __restrict result_data, const IColumn::Offsets & offsets) { const IntType * data_copy_begin_ptr = nullptr; size_t offsets_size = offsets.size(); @@ -842,7 +842,7 @@ ColumnPtr ColumnVector::replicate(const IColumn::Offsets & offsets) const #ifdef __SSE2__ if constexpr (std::is_same_v) { - replicateSSE42Int32(getData().data(), res->getData().data(), offsets); + replicateSSE2Int32(getData().data(), res->getData().data(), offsets); return res; } #endif From 3b433783d5a50f6685e58026c27875dce9a9e89f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 25 Feb 2023 01:03:38 +0100 Subject: [PATCH 663/717] Add a test for #45214 --- ..._suspicious_low_cardinality_msan.reference | 20 +++++++++++++++++++ .../02672_suspicious_low_cardinality_msan.sql | 10 ++++++++++ 2 files changed, 30 insertions(+) create mode 100644 tests/queries/0_stateless/02672_suspicious_low_cardinality_msan.reference create mode 100644 tests/queries/0_stateless/02672_suspicious_low_cardinality_msan.sql diff --git a/tests/queries/0_stateless/02672_suspicious_low_cardinality_msan.reference b/tests/queries/0_stateless/02672_suspicious_low_cardinality_msan.reference new file mode 100644 index 00000000000..7dd7c8eece6 --- /dev/null +++ b/tests/queries/0_stateless/02672_suspicious_low_cardinality_msan.reference @@ -0,0 +1,20 @@ +1023 \N \N 0 2147483646 +1023 \N \N 0 2147483646 +1023 \N \N 0 2147483646 +1023 \N \N 0 2147483646 +1023 \N \N 0 2147483646 +1023 \N \N 0 2147483646 +1023 \N \N 0 2147483646 +1023 \N \N 0 2147483646 +1023 \N \N 0 2147483646 +1023 \N \N 0 2147483646 +1023 \N \N 0 2147483646 +1023 \N \N 0 2147483646 +1023 \N \N 0 2147483646 +1023 \N \N 0 2147483646 +1023 \N \N 0 2147483646 +1023 \N \N 0 2147483646 +1023 \N \N 0 2147483646 +1023 \N \N 0 2147483646 +1023 \N \N 0 2147483646 +1023 \N \N 0 2147483646 diff --git a/tests/queries/0_stateless/02672_suspicious_low_cardinality_msan.sql b/tests/queries/0_stateless/02672_suspicious_low_cardinality_msan.sql new file mode 100644 index 00000000000..f2841ac8897 --- /dev/null +++ b/tests/queries/0_stateless/02672_suspicious_low_cardinality_msan.sql @@ -0,0 +1,10 @@ +DROP TABLE IF EXISTS table1__fuzz_19; + +SET allow_suspicious_low_cardinality_types = 1; +CREATE TABLE table1__fuzz_19 (`id` LowCardinality(UInt16), `v` DateTime64(3, 'UTC')) ENGINE = ReplacingMergeTree(v) PARTITION BY id % 200 ORDER BY id; +INSERT INTO table1__fuzz_19 SELECT number - 205, number FROM numbers(10); +INSERT INTO table1__fuzz_19 SELECT number - 205, number FROM numbers(400, 10); + +SELECT 1023, (((id % -9223372036854775807) = NULL) OR ((id % NULL) = 100) OR ((id % NULL) = 65537)) = ((id % inf) = 9223372036854775806), (id % NULL) = NULL, (id % 3.4028234663852886e38) = 1023, 2147483646 FROM table1__fuzz_19 ORDER BY (((id % 1048577) = 1024) % id) = 1023 DESC NULLS FIRST, id % 2147483646 ASC NULLS FIRST, ((id % 1) = 9223372036854775807) OR ((id % NULL) = 257) DESC NULLS FIRST; + +DROP TABLE table1__fuzz_19; From 4134e56ef074d30b1643a48adab905455923cef5 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 25 Feb 2023 03:20:47 +0100 Subject: [PATCH 664/717] Increase the limit for opened files in clickhouse-local --- programs/local/LocalServer.cpp | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index f62520d3007..8e092bdf8e4 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -1,5 +1,8 @@ #include "LocalServer.h" +#include +#include +#include #include #include #include @@ -179,9 +182,9 @@ void LocalServer::tryInitPath() parent_folder = std::filesystem::temp_directory_path(); } - catch (const fs::filesystem_error& e) + catch (const fs::filesystem_error & e) { - // tmp folder don't exists? misconfiguration? chroot? + // The tmp folder doesn't exist? Is it a misconfiguration? Or chroot? LOG_DEBUG(log, "Can not get temporary folder: {}", e.what()); parent_folder = std::filesystem::current_path(); @@ -390,6 +393,21 @@ try std::cout << std::fixed << std::setprecision(3); std::cerr << std::fixed << std::setprecision(3); + /// Try to increase limit on number of open files. + { + rlimit rlim; + if (getrlimit(RLIMIT_NOFILE, &rlim)) + throw Poco::Exception("Cannot getrlimit"); + + if (rlim.rlim_cur < rlim.rlim_max) + { + rlim.rlim_cur = config().getUInt("max_open_files", static_cast(rlim.rlim_max)); + int rc = setrlimit(RLIMIT_NOFILE, &rlim); + if (rc != 0) + std::cerr << fmt::format("Cannot set max number of file descriptors to {}. Try to specify max_open_files according to your system limits. error: {}", rlim.rlim_cur, errnoToString()) << '\n'; + } + } + #if defined(FUZZING_MODE) static bool first_time = true; if (first_time) From 83c41755ed1b2c5bc59e10cb7f5fa5aafd9c5346 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 25 Feb 2023 03:21:14 +0100 Subject: [PATCH 665/717] Remove unused header --- programs/server/Server.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 3964aa5e565..8c60f840b89 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -6,7 +6,6 @@ #include #include #include -#include #include #include #include From f9faec7ba05af4a5bfca37b523f14919352d15a3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 25 Feb 2023 03:21:34 +0100 Subject: [PATCH 666/717] Do not backoff reading from URL in case of too many opened files --- src/IO/ReadWriteBufferFromHTTP.h | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/IO/ReadWriteBufferFromHTTP.h b/src/IO/ReadWriteBufferFromHTTP.h index 5bc72c5ff62..d00ba6aa8cf 100644 --- a/src/IO/ReadWriteBufferFromHTTP.h +++ b/src/IO/ReadWriteBufferFromHTTP.h @@ -582,10 +582,13 @@ namespace detail } catch (const Poco::Exception & e) { - /** - * Retry request unconditionally if nothing has been read yet. - * Otherwise if it is GET method retry with range header. - */ + /// Too many open files - non-retryable. + if (e.code() == POCO_EMFILE) + throw; + + /** Retry request unconditionally if nothing has been read yet. + * Otherwise if it is GET method retry with range header. + */ bool can_retry_request = !offset_from_begin_pos || method == Poco::Net::HTTPRequest::HTTP_GET; if (!can_retry_request) throw; From cde1bdad10558e7412c61e195405aa6c4f4684d0 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 25 Feb 2023 08:45:53 +0100 Subject: [PATCH 667/717] Spelling --- src/IO/ReadWriteBufferFromHTTP.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/IO/ReadWriteBufferFromHTTP.h b/src/IO/ReadWriteBufferFromHTTP.h index d00ba6aa8cf..a3a3c318f5a 100644 --- a/src/IO/ReadWriteBufferFromHTTP.h +++ b/src/IO/ReadWriteBufferFromHTTP.h @@ -224,9 +224,9 @@ namespace detail enum class InitializeError { - RETRIABLE_ERROR, + RETRYABLE_ERROR, /// If error is not retriable, `exception` variable must be set. - NON_RETRIABLE_ERROR, + NON_RETRYABLE_ERROR, /// Allows to skip not found urls for globs SKIP_NOT_FOUND_URL, NONE, @@ -398,7 +398,7 @@ namespace detail } else if (!isRetriableError(http_status)) { - initialization_error = InitializeError::NON_RETRIABLE_ERROR; + initialization_error = InitializeError::NON_RETRYABLE_ERROR; exception = std::current_exception(); } else @@ -409,7 +409,7 @@ namespace detail } /** - * Throws if error is retriable, otherwise sets initialization_error = NON_RETRIABLE_ERROR and + * Throws if error is retryable, otherwise sets initialization_error = NON_RETRYABLE_ERROR and * saves exception into `exception` variable. In case url is not found and skip_not_found_url == true, * sets initialization_error = SKIP_NOT_FOUND_URL, otherwise throws. */ @@ -453,9 +453,9 @@ namespace detail /// Retry 200OK if (response.getStatus() == Poco::Net::HTTPResponse::HTTPStatus::HTTP_OK) - initialization_error = InitializeError::RETRIABLE_ERROR; + initialization_error = InitializeError::RETRYABLE_ERROR; else - initialization_error = InitializeError::NON_RETRIABLE_ERROR; + initialization_error = InitializeError::NON_RETRYABLE_ERROR; return; } @@ -544,7 +544,7 @@ namespace detail { initialize(); - if (initialization_error == InitializeError::NON_RETRIABLE_ERROR) + if (initialization_error == InitializeError::NON_RETRYABLE_ERROR) { assert(exception); break; @@ -553,7 +553,7 @@ namespace detail { return false; } - else if (initialization_error == InitializeError::RETRIABLE_ERROR) + else if (initialization_error == InitializeError::RETRYABLE_ERROR) { LOG_ERROR( log, From c659d84b4716ad9dcb71453f3476d332cdeb1b70 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 25 Feb 2023 08:46:21 +0100 Subject: [PATCH 668/717] Fix MSan report in Map data type --- src/Functions/FunctionsHashing.h | 16 +++++++++------- src/Functions/IFunction.cpp | 6 ++++-- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/Functions/FunctionsHashing.h b/src/Functions/FunctionsHashing.h index 75505a173cd..37e0eb9f237 100644 --- a/src/Functions/FunctionsHashing.h +++ b/src/Functions/FunctionsHashing.h @@ -1112,7 +1112,9 @@ private: { ToType h; if constexpr (std::endian::native == std::endian::little) + { h = apply(key, reinterpret_cast(&vec_from[i]), sizeof(vec_from[i])); + } else { char tmp_buffer[sizeof(vec_from[i])]; @@ -1131,7 +1133,9 @@ private: ToType h; if constexpr (std::endian::native == std::endian::little) + { h = apply(key, reinterpret_cast(&value), sizeof(value)); + } else { char tmp_buffer[sizeof(value)]; @@ -1271,7 +1275,7 @@ private: { /// NOTE: here, of course, you can do without the materialization of the column. ColumnPtr full_column = col_from_const->convertToFullColumn(); - executeArray(key, type, &*full_column, vec_to); + executeArray(key, type, full_column.get(), vec_to); } else throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of first argument of function {}", @@ -1343,10 +1347,9 @@ private: const auto & type_map = assert_cast(*type); executeForArgument(key, type_map.getNestedType().get(), map->getNestedColumnPtr().get(), vec_to, is_first); } - else if (const auto * const_map = checkAndGetColumnConstData(column)) + else if (const auto * const_map = checkAndGetColumnConst(column)) { - const auto & type_map = assert_cast(*type); - executeForArgument(key, type_map.getNestedType().get(), const_map->getNestedColumnPtr().get(), vec_to, is_first); + executeForArgument(key, type, const_map->convertToFullColumnIfConst().get(), vec_to, is_first); } else { @@ -1382,8 +1385,7 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - size_t rows = input_rows_count; - auto col_to = ColumnVector::create(rows); + auto col_to = ColumnVector::create(input_rows_count); typename ColumnVector::Container & vec_to = col_to->getData(); @@ -1395,7 +1397,7 @@ public: if (arguments.size() <= first_data_argument) { /// Return a fixed random-looking magic number when input is empty - vec_to.assign(rows, static_cast(0xe28dbde7fe22e41c)); + vec_to.assign(input_rows_count, static_cast(0xe28dbde7fe22e41c)); } KeyType key{}; diff --git a/src/Functions/IFunction.cpp b/src/Functions/IFunction.cpp index 1c30dee0482..1f25794536b 100644 --- a/src/Functions/IFunction.cpp +++ b/src/Functions/IFunction.cpp @@ -26,6 +26,7 @@ # pragma GCC diagnostic pop #endif + namespace DB { @@ -66,12 +67,12 @@ ColumnPtr replaceLowCardinalityColumnsByNestedAndGetDictionaryIndexes( if (!low_cardinality_type) throw Exception(ErrorCodes::LOGICAL_ERROR, - "Incompatible type for low cardinality column: {}", + "Incompatible type for LowCardinality column: {}", column.type->getName()); if (can_be_executed_on_default_arguments) { - /// Normal case, when function can be executed on values's default. + /// Normal case, when function can be executed on values' default. column.column = low_cardinality_column->getDictionary().getNestedColumn(); indexes = low_cardinality_column->getIndexesPtr(); } @@ -280,6 +281,7 @@ ColumnPtr IExecutableFunction::executeWithoutSparseColumns(const ColumnsWithType auto res = executeWithoutLowCardinalityColumns(columns_without_low_cardinality, dictionary_type, new_input_rows_count, dry_run); bool res_is_constant = isColumnConst(*res); + auto keys = res_is_constant ? res->cloneResized(1)->convertToFullColumnIfConst() : res; From 21ba6b70e64f9d999f5f713887ee68980ef4dd11 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 25 Feb 2023 08:47:48 +0100 Subject: [PATCH 669/717] Fix MSan report in Map data type --- src/Functions/map.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/map.cpp b/src/Functions/map.cpp index 3160c5ddb43..7fd4d3aa6d5 100644 --- a/src/Functions/map.cpp +++ b/src/Functions/map.cpp @@ -332,7 +332,7 @@ public: } size_t col_key_size = sub_map_column->size(); - auto column = is_const? ColumnConst::create(std::move(sub_map_column), std::move(col_key_size)) : std::move(sub_map_column); + auto column = is_const ? ColumnConst::create(std::move(sub_map_column), std::move(col_key_size)) : std::move(sub_map_column); ColumnsWithTypeAndName new_arguments = { From fbff23403fcec66f0f8ca6fe4c6bf847f98ea611 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 25 Feb 2023 08:53:53 +0100 Subject: [PATCH 670/717] Add a test --- tests/queries/0_stateless/02673_map_hashing_msan.sql | 6 ++++++ tests/queries/0_stateless/02673_map_hashing_reference.sql | 3 +++ 2 files changed, 9 insertions(+) create mode 100644 tests/queries/0_stateless/02673_map_hashing_msan.sql create mode 100644 tests/queries/0_stateless/02673_map_hashing_reference.sql diff --git a/tests/queries/0_stateless/02673_map_hashing_msan.sql b/tests/queries/0_stateless/02673_map_hashing_msan.sql new file mode 100644 index 00000000000..8c7d028e137 --- /dev/null +++ b/tests/queries/0_stateless/02673_map_hashing_msan.sql @@ -0,0 +1,6 @@ +SELECT cityHash64(map(1, 'Hello'), CAST(materialize('World') AS LowCardinality(String))); +SELECT cityHash64(map(), CAST(materialize('') AS LowCardinality(Nullable(String)))); + +SET allow_suspicious_low_cardinality_types = 1; +CREATE TEMPORARY TABLE datetime__fuzz_14 (`d` LowCardinality(Nullable(UInt128))); +SELECT max(mapPopulateSeries(mapPopulateSeries(map(toInt64(1048), toInt64(9223), 3, -2147))), toInt64(1048), map('11', 257, '', NULL), cityHash64(*)) > NULL FROM (SELECT max(cityHash64(mapPopulateSeries(mapPopulateSeries(map(toInt64(1048), toInt64(2147), 655, -2147))), *)) > NULL, map(toInt64(-2147), toInt64(100.0001), -2147, NULL), mapPopulateSeries(map(toInt64(1024), toInt64(1048), 1048, -1)), map(toInt64(256), toInt64(NULL), -1, NULL), quantile(0.0001)(d) FROM datetime__fuzz_14 WITH TOTALS); diff --git a/tests/queries/0_stateless/02673_map_hashing_reference.sql b/tests/queries/0_stateless/02673_map_hashing_reference.sql new file mode 100644 index 00000000000..e37271a63d5 --- /dev/null +++ b/tests/queries/0_stateless/02673_map_hashing_reference.sql @@ -0,0 +1,3 @@ +4786021384179797717 +5368498105280294197 +\N From 5321a8183e3659054f7d8b70d48513991840cefe Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 25 Feb 2023 09:04:54 +0100 Subject: [PATCH 671/717] Fix 02116_interactive_hello test Signed-off-by: Azat Khuzhin --- tests/queries/0_stateless/02116_interactive_hello.expect | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/02116_interactive_hello.expect b/tests/queries/0_stateless/02116_interactive_hello.expect index 92722e203ec..d05d9adc826 100755 --- a/tests/queries/0_stateless/02116_interactive_hello.expect +++ b/tests/queries/0_stateless/02116_interactive_hello.expect @@ -19,9 +19,9 @@ expect_after { spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --history_file=$history_file" -expect -re "ClickHouse client version \[\\d\]{2}.\[\\d\]{1,2}.\[\\d\]{1,2}.\[\\d\]{1,2}.\r" -expect -re "Connecting to database .* at localhost:9000 as user default.\r" -expect -re "Connected to ClickHouse server version \[\\d\]{2}.\[\\d\]{1,2}.\[\\d\]{1,2} revision .*\r" +expect -re "ClickHouse client version \[\\d\]{2}\.\[\\d\]{1,2}\.\[\\d\]{1,2}\.\[\\d\]{1,}\.\r" +expect -re "Connecting to database .* at localhost:9000 as user default\.\r" +expect -re "Connected to ClickHouse server version \[\\d\]{2}\.\[\\d\]{1,2}\.\[\\d\]{1,2} revision \[\\d\]{1,}\.\r" expect ":) " send -- "" From f56eb86f7b629d093697c372f3b7e1e5bdb709dd Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 25 Feb 2023 09:07:08 +0100 Subject: [PATCH 672/717] Fix 01520_client_print_query_id test Signed-off-by: Azat Khuzhin --- .../queries/0_stateless/01520_client_print_query_id.expect | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01520_client_print_query_id.expect b/tests/queries/0_stateless/01520_client_print_query_id.expect index cec1695c433..2034483a73d 100755 --- a/tests/queries/0_stateless/01520_client_print_query_id.expect +++ b/tests/queries/0_stateless/01520_client_print_query_id.expect @@ -21,10 +21,12 @@ expect ":) " # Make a query send -- "SELECT 'print query id'\r" expect { - "Query id: *" { } + # (?n) - Do not match new lines [1] + # [1]: https://www.tcl.tk/man/tcl8.6/TclCmd/re_syntax.html + -re "(?n)Query id: .*" { } timeout { exit 1 } } -expect "print query id" +expect "'print query id" expect ":) " send -- "\4" From 9ec6f94638a580a721d8b8ae6340c5377e20dbd2 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 25 Feb 2023 09:07:55 +0100 Subject: [PATCH 673/717] Add a test --- ...2673_map_hashing_reference.sql => 02673_map_hashing.reference} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/queries/0_stateless/{02673_map_hashing_reference.sql => 02673_map_hashing.reference} (100%) diff --git a/tests/queries/0_stateless/02673_map_hashing_reference.sql b/tests/queries/0_stateless/02673_map_hashing.reference similarity index 100% rename from tests/queries/0_stateless/02673_map_hashing_reference.sql rename to tests/queries/0_stateless/02673_map_hashing.reference From 10c2bfe2ee307cca7b3593745c5624bad92ab062 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 25 Feb 2023 09:08:24 +0100 Subject: [PATCH 674/717] Add a test --- ...673_map_hashing.reference => 02673_map_hashing_msan.reference} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/queries/0_stateless/{02673_map_hashing.reference => 02673_map_hashing_msan.reference} (100%) diff --git a/tests/queries/0_stateless/02673_map_hashing.reference b/tests/queries/0_stateless/02673_map_hashing_msan.reference similarity index 100% rename from tests/queries/0_stateless/02673_map_hashing.reference rename to tests/queries/0_stateless/02673_map_hashing_msan.reference From d8c5da1a50b812a7790b7b592100107a9fcb78c4 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Sat, 25 Feb 2023 14:46:23 +0100 Subject: [PATCH 675/717] A test to check that computed value is equal in all rows --- tests/queries/0_stateless/02673_map_hashing_msan.reference | 3 +++ tests/queries/0_stateless/02673_map_hashing_msan.sql | 1 + 2 files changed, 4 insertions(+) diff --git a/tests/queries/0_stateless/02673_map_hashing_msan.reference b/tests/queries/0_stateless/02673_map_hashing_msan.reference index e37271a63d5..d3fbc1377d4 100644 --- a/tests/queries/0_stateless/02673_map_hashing_msan.reference +++ b/tests/queries/0_stateless/02673_map_hashing_msan.reference @@ -1,3 +1,6 @@ 4786021384179797717 5368498105280294197 +42 15687122600100720591 +42 15687122600100720591 +42 15687122600100720591 \N diff --git a/tests/queries/0_stateless/02673_map_hashing_msan.sql b/tests/queries/0_stateless/02673_map_hashing_msan.sql index 8c7d028e137..2fe3620e68c 100644 --- a/tests/queries/0_stateless/02673_map_hashing_msan.sql +++ b/tests/queries/0_stateless/02673_map_hashing_msan.sql @@ -1,5 +1,6 @@ SELECT cityHash64(map(1, 'Hello'), CAST(materialize('World') AS LowCardinality(String))); SELECT cityHash64(map(), CAST(materialize('') AS LowCardinality(Nullable(String)))); +SELECT materialize(42) as last_element, cityHash64(map(), CAST(materialize('') AS LowCardinality(Nullable(String))), last_element) from numbers(3); SET allow_suspicious_low_cardinality_types = 1; CREATE TEMPORARY TABLE datetime__fuzz_14 (`d` LowCardinality(Nullable(UInt128))); From 7606d2406bdd5dc1ae5a3026d624966148c487fc Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Sat, 25 Feb 2023 14:47:45 +0100 Subject: [PATCH 676/717] Check that argument sizes and result size match --- src/Functions/FunctionsHashing.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Functions/FunctionsHashing.h b/src/Functions/FunctionsHashing.h index 37e0eb9f237..e824546f220 100644 --- a/src/Functions/FunctionsHashing.h +++ b/src/Functions/FunctionsHashing.h @@ -1287,6 +1287,10 @@ private: { WhichDataType which(from_type); + if (icolumn->size() != vec_to.size()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Argument column '{}' size {} doesn't match result column size {} of function {}", + icolumn->getName(), icolumn->size(), vec_to.size(), getName()); + if (which.isUInt8()) executeIntType(key, icolumn, vec_to); else if (which.isUInt16()) executeIntType(key, icolumn, vec_to); else if (which.isUInt32()) executeIntType(key, icolumn, vec_to); From d74632cdb79a2d375666f579fdee2ad37ea89f07 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sat, 25 Feb 2023 15:21:48 +0000 Subject: [PATCH 677/717] Minor fixup of string escaping syntax docs --- docs/en/sql-reference/syntax.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/sql-reference/syntax.md b/docs/en/sql-reference/syntax.md index bde37241ed0..63c5042f9e8 100644 --- a/docs/en/sql-reference/syntax.md +++ b/docs/en/sql-reference/syntax.md @@ -86,10 +86,10 @@ Examples: `1`, `10_000_000`, `0xffff_ffff`, `18446744073709551615`, `0xDEADBEEF` String literals must be enclosed in single quotes, double quotes are not supported. Escaping works either -- in SQL-style based on a preceding single quote where the single-quote character `'` (and only this character) can be escaped as `''`, or -- in C-style based on a preceding backslash with the following supported escape sequences: `\\`, `\'`, `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\a`, `\v`, `\xHH`. The backslash loses its special meaning and will be interpreted literally if it precedes characters different than the listed ones. +- using a preceding single quote where the single-quote character `'` (and only this character) can be escaped as `''`, or +- using a preceding backslash with the following supported escape sequences: `\\`, `\'`, `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\a`, `\v`, `\xHH`. The backslash loses its special meaning, i.e. will be interpreted literally, if it precedes characters different than the listed ones. -In string literals, you need to escape at least `'` and `\` using escape codes `''` (or: `\'`) and `\\`. +In string literals, you need to escape at least `'` and `\` using escape codes `\'` (or: `''`) and `\\`. ### Compound From 3618e37dd2aee9b22a6819dbfa61e3fc7ad03d13 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sat, 25 Feb 2023 15:36:59 +0000 Subject: [PATCH 678/717] Reduce indentation by early out --- src/Storages/Kafka/StorageKafka.cpp | 52 ++++++++++++++--------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index 78a11c56409..7b97273d8af 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -188,14 +188,14 @@ namespace for (const auto & tag : tags) { - if (tag.starts_with(CONFIG_KAFKA_TOPIC_TAG)) /// Multiple occurrences given as "kafka_topic", "kafka_topic[1]", etc. - continue; // used by new per-topic configuration, ignore + if (tag.starts_with(CONFIG_KAFKA_TOPIC_TAG)) /// multiple occurrences given as "kafka_topic", "kafka_topic[1]", etc. + continue; /// used by new per-topic configuration, ignore const String setting_path = config_prefix + "." + tag; const String setting_value = config.getString(setting_path); - // "log_level" has valid underscore, the remaining librdkafka setting use dot.separated.format which isn't acceptable for XML. - // See https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md + /// "log_level" has valid underscore, the remaining librdkafka setting use dot.separated.format which isn't acceptable for XML. + /// See https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md const String setting_name_in_kafka_config = (tag == "log_level") ? tag : boost::replace_all_copy(tag, "_", "."); kafka_config.set(setting_name_in_kafka_config, setting_value); } @@ -210,32 +210,32 @@ namespace for (const auto & tag : tags) { - /// Consider tag . Multiple occurrences given as "kafka_topic", "kafka_topic[1]", etc. - if (tag.starts_with(CONFIG_KAFKA_TOPIC_TAG)) + /// Only consider tag . Multiple occurrences given as "kafka_topic", "kafka_topic[1]", etc. + if (!tag.starts_with(CONFIG_KAFKA_TOPIC_TAG)) + continue; + + /// Read topic name between ... + const String kafka_topic_path = config_prefix + "." + tag; + const String kafpa_topic_name_path = kafka_topic_path + "." + CONFIG_NAME_TAG; + + const String topic_name = config.getString(kafpa_topic_name_path); + if (topic_name == topic) { - /// Read topic name between ... - const String kafka_topic_path = config_prefix + "." + tag; - const String kafpa_topic_name_path = kafka_topic_path + "." + CONFIG_NAME_TAG; - - const String topic_name = config.getString(kafpa_topic_name_path); - if (topic_name == topic) + /// Found it! Now read the per-topic configuration into cppkafka. + Poco::Util::AbstractConfiguration::Keys inner_tags; + config.keys(kafka_topic_path, inner_tags); + for (const auto & inner_tag : inner_tags) { - /// Found it! Now read the per-topic configuration into cppkafka. - Poco::Util::AbstractConfiguration::Keys inner_tags; - config.keys(kafka_topic_path, inner_tags); - for (const auto & inner_tag : inner_tags) - { - if (inner_tag == CONFIG_NAME_TAG) - continue; // ignore + if (inner_tag == CONFIG_NAME_TAG) + continue; // ignore - // "log_level" has valid underscore, the remaining librdkafka setting use dot.separated.format which isn't acceptable for XML. - // See https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md - const String setting_path = kafka_topic_path + "." + inner_tag; - const String setting_value = config.getString(setting_path); + /// "log_level" has valid underscore, the remaining librdkafka setting use dot.separated.format which isn't acceptable for XML. + /// See https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md + const String setting_path = kafka_topic_path + "." + inner_tag; + const String setting_value = config.getString(setting_path); - const String setting_name_in_kafka_config = (inner_tag == "log_level") ? inner_tag : boost::replace_all_copy(inner_tag, "_", "."); - kafka_config.set(setting_name_in_kafka_config, setting_value); - } + const String setting_name_in_kafka_config = (inner_tag == "log_level") ? inner_tag : boost::replace_all_copy(inner_tag, "_", "."); + kafka_config.set(setting_name_in_kafka_config, setting_value); } } } From 6e2cec9ab6b732ffde798fdf28e6b11da4f1ddb4 Mon Sep 17 00:00:00 2001 From: Rich Raposa Date: Sat, 25 Feb 2023 10:03:32 -0700 Subject: [PATCH 679/717] Update tuple-map-functions.md --- docs/en/sql-reference/functions/tuple-map-functions.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/functions/tuple-map-functions.md b/docs/en/sql-reference/functions/tuple-map-functions.md index 1905e53af3e..2dad95745e5 100644 --- a/docs/en/sql-reference/functions/tuple-map-functions.md +++ b/docs/en/sql-reference/functions/tuple-map-functions.md @@ -440,7 +440,7 @@ mapApply(func, map) **Parameters** -- `func` - [Lamda function](../../sql-reference/functions/index.md#higher-order-functions---operator-and-lambdaparams-expr-function). +- `func` - [Lambda function](../../sql-reference/functions/index.md#higher-order-functions---operator-and-lambdaparams-expr-function). - `map` — [Map](../../sql-reference/data-types/map.md). **Returned value** @@ -480,7 +480,7 @@ mapFilter(func, map) **Parameters** -- `func` - [Lamda function](../../sql-reference/functions/index.md#higher-order-functions---operator-and-lambdaparams-expr-function). +- `func` - [Lambda function](../../sql-reference/functions/index.md#higher-order-functions---operator-and-lambdaparams-expr-function). - `map` — [Map](../../sql-reference/data-types/map.md). **Returned value** From 01e7f41c7929ce9c200878d2bf00b397043a4897 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sat, 25 Feb 2023 17:13:57 +0000 Subject: [PATCH 680/717] Fix escaping descriptionn in LIKE function docs --- .../functions/string-search-functions.md | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md index e52eb00ab44..dec50caef86 100644 --- a/docs/en/sql-reference/functions/string-search-functions.md +++ b/docs/en/sql-reference/functions/string-search-functions.md @@ -382,12 +382,12 @@ Checks whether string `haystack` matches the regular expression `pattern`. The p Returns 1 in case of a match, and 0 otherwise. Matching is based on UTF-8, e.g. `.` matches the Unicode code point `¥` which is represented in UTF-8 using two bytes. The regular expression must not contain null bytes. -If the haystack or pattern contain a sequence of bytes that are not valid UTF-8, the behavior is undefined. -No automatic Unicode normalization is performed, if you need it you can use the [normalizeUTF8*()](https://clickhouse.com/docs/en/sql-reference/functions/string-functions/) functions for that. +If the haystack or the pattern are not valid UTF-8, then the behavior is undefined. +No automatic Unicode normalization is performed, you can use the [normalizeUTF8*()](https://clickhouse.com/docs/en/sql-reference/functions/string-functions/) functions for that. Unlike re2's default behavior, `.` matches line breaks. To disable this, prepend the pattern with `(?-s)`. -For patterns to search for substrings in a string, it is better to use LIKE or ‘position’, since they work much faster. +For patterns to search for substrings in a string, it is better to use functions [like](#like) or [position](#position) since they work much faster. ## multiMatchAny(haystack, \[pattern1, pattern2, …, patternn\]) @@ -529,21 +529,24 @@ Result: ## like(haystack, pattern), haystack LIKE pattern operator -Checks whether a string matches a simple regular expression. -The regular expression can contain the metasymbols `%` and `_`. +Checks whether a string matches a LIKE expression. +LIKE expression contains a mix of normal characters and the following metasymbols: -`%` indicates any quantity of any bytes (including zero characters). +- `%` indicates an arbitrary number of arbitrary characters (including zero characters). -`_` indicates any one byte. +- `_` indicates a single arbitrary character. -Use the backslash (`\`) for escaping metasymbols. See the note on escaping in the description of the ‘match’ function. +- `\` is for escaping literals `%`, `_` and `\`. Matching is based on UTF-8, e.g. `_` matches the Unicode code point `¥` which is represented in UTF-8 using two bytes. -If the haystack or pattern contain a sequence of bytes that are not valid UTF-8, then the behavior is undefined. -No automatic Unicode normalization is performed, if you need it you can use the [normalizeUTF8*()](https://clickhouse.com/docs/en/sql-reference/functions/string-functions/) functions for that. +If the haystack or the pattern are not valid UTF-8, then the behavior is undefined. +No automatic Unicode normalization is performed, you can use the [normalizeUTF8*()](https://clickhouse.com/docs/en/sql-reference/functions/string-functions/) functions for that. -For regular expressions like `%needle%`, the code is more optimal and works as fast as the `position` function. -For other regular expressions, the code is the same as for the ‘match’ function. +To match against literals `%`, `_` and `/` (which are LIKE metacharacters), prepend them with a backslash, i.e. `\%`, `\_` and +`\\`. Note that ClickHouse requires backslashes in strings [to be quoted as well](../syntax.md#String), so you would actually need to write `\\%`, `\\_` and `\\\\`. + +For patterns of the form `%needle%`, the function is as fast as the `position` function. +Other LIKE expressions are internally converted to a regular expression and executed with a performance similar to function `match`. ## notLike(haystack, pattern), haystack NOT LIKE pattern operator From ca06ad48e41eba452d9492a04ec418021683d51d Mon Sep 17 00:00:00 2001 From: DanRoscigno Date: Sat, 25 Feb 2023 12:17:33 -0500 Subject: [PATCH 681/717] add note about prod install --- docs/en/getting-started/install.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/en/getting-started/install.md b/docs/en/getting-started/install.md index 6a15c32e9dd..9e0dec83f7e 100644 --- a/docs/en/getting-started/install.md +++ b/docs/en/getting-started/install.md @@ -23,6 +23,10 @@ The quickest and easiest way to get up and running with ClickHouse is to create ## Self-Managed Install +:::tip +For production installs of a specific release version see the [available versions](#available-installation-options) down below. +::: + From 138a050e13d0ef10ced3422aea0903d2c0f0c7e9 Mon Sep 17 00:00:00 2001 From: DanRoscigno Date: Sat, 25 Feb 2023 12:22:02 -0500 Subject: [PATCH 682/717] fix section name --- docs/en/getting-started/install.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/getting-started/install.md b/docs/en/getting-started/install.md index 9e0dec83f7e..0867f3a0795 100644 --- a/docs/en/getting-started/install.md +++ b/docs/en/getting-started/install.md @@ -24,7 +24,7 @@ The quickest and easiest way to get up and running with ClickHouse is to create ## Self-Managed Install :::tip -For production installs of a specific release version see the [available versions](#available-installation-options) down below. +For production installs of a specific release version see the [installation options](#available-installation-options) down below. ::: From cf062f0c5b107f54c9dd20213f6fab81eff816ae Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sat, 25 Feb 2023 17:34:23 +0000 Subject: [PATCH 683/717] Small optimization of LIKE patterns with > 1 trailing % Previously, LIKE pattern '%%%ab' became '.*.*ab'. Now, it becomes 'ab' which is a bit faster (perhaps). + minor code cleanup + a minor doc update --- .../functions/string-search-functions.md | 7 +-- src/Functions/likePatternToRegexp.h | 51 ++++++++++++------- .../00187_like_regexp_prefix.reference | 1 + .../0_stateless/00187_like_regexp_prefix.sql | 2 + 4 files changed, 41 insertions(+), 20 deletions(-) diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md index dec50caef86..1320b571eba 100644 --- a/docs/en/sql-reference/functions/string-search-functions.md +++ b/docs/en/sql-reference/functions/string-search-functions.md @@ -530,7 +530,7 @@ Result: ## like(haystack, pattern), haystack LIKE pattern operator Checks whether a string matches a LIKE expression. -LIKE expression contains a mix of normal characters and the following metasymbols: +A LIKE expression contains a mix of normal characters and the following metasymbols: - `%` indicates an arbitrary number of arbitrary characters (including zero characters). @@ -542,8 +542,9 @@ Matching is based on UTF-8, e.g. `_` matches the Unicode code point `¥` which i If the haystack or the pattern are not valid UTF-8, then the behavior is undefined. No automatic Unicode normalization is performed, you can use the [normalizeUTF8*()](https://clickhouse.com/docs/en/sql-reference/functions/string-functions/) functions for that. -To match against literals `%`, `_` and `/` (which are LIKE metacharacters), prepend them with a backslash, i.e. `\%`, `\_` and -`\\`. Note that ClickHouse requires backslashes in strings [to be quoted as well](../syntax.md#String), so you would actually need to write `\\%`, `\\_` and `\\\\`. +To match against literals `%`, `_` and `/` (which are LIKE metacharacters), prepend them with a backslash, i.e. `\%`, `\_` and `\\`. +The backslash loses its special meaning, i.e. is interpreted literally, if it prepends a character different than `%`, `_` or `\`. +Note that ClickHouse requires backslashes in strings [to be quoted as well](../syntax.md#string), so you would actually need to write `\\%`, `\\_` and `\\\\`. For patterns of the form `%needle%`, the function is as fast as the `position` function. Other LIKE expressions are internally converted to a regular expression and executed with a performance similar to function `match`. diff --git a/src/Functions/likePatternToRegexp.h b/src/Functions/likePatternToRegexp.h index 298137d7bc1..6c0b6241ae2 100644 --- a/src/Functions/likePatternToRegexp.h +++ b/src/Functions/likePatternToRegexp.h @@ -21,7 +21,12 @@ inline String likePatternToRegexp(std::string_view pattern) const char * const end = pattern.begin() + pattern.size(); if (pos < end && *pos == '%') - ++pos; + /// Eat leading % + while (++pos < end) + { + if (*pos != '%') + break; + } else res = "^"; @@ -29,7 +34,18 @@ inline String likePatternToRegexp(std::string_view pattern) { switch (*pos) { - case '^': case '$': case '.': case '[': case '|': case '(': case ')': case '?': case '*': case '+': case '{': + /// Quote characters which have a special meaning in re2 + case '^': + case '$': + case '.': + case '[': + case '|': + case '(': + case ')': + case '?': + case '*': + case '+': + case '{': res += '\\'; res += *pos; break; @@ -44,22 +60,23 @@ inline String likePatternToRegexp(std::string_view pattern) break; case '\\': if (pos + 1 == end) - throw Exception(ErrorCodes::CANNOT_PARSE_ESCAPE_SEQUENCE, "Invalid escape sequence at the end of LIKE pattern"); - /// Known escape sequences. - if (pos[1] == '%' || pos[1] == '_') + throw Exception(ErrorCodes::CANNOT_PARSE_ESCAPE_SEQUENCE, "Invalid escape sequence at the end of LIKE pattern '{}'", pattern); + switch (pos[1]) { - res += pos[1]; - ++pos; - } - else if (pos[1] == '\\') - { - res += "\\\\"; - ++pos; - } - else - { - /// Unknown escape sequence treated literally: as backslash and the following character. - res += "\\\\"; + /// Interpret quoted LIKE metacharacters %, _ and \ as literals: + case '%': + case '_': + res += pos[1]; + ++pos; + break; + case '\\': + res += "\\\\"; /// backslash has a special meaning in re2 --> quote it + ++pos; + break; + /// Unknown escape sequence treated literally: as backslash (which must be quoted in re2) + the following character + default: + res += "\\\\"; + break; } break; default: diff --git a/tests/queries/0_stateless/00187_like_regexp_prefix.reference b/tests/queries/0_stateless/00187_like_regexp_prefix.reference index d00491fd7e5..6ed281c757a 100644 --- a/tests/queries/0_stateless/00187_like_regexp_prefix.reference +++ b/tests/queries/0_stateless/00187_like_regexp_prefix.reference @@ -1 +1,2 @@ 1 +1 diff --git a/tests/queries/0_stateless/00187_like_regexp_prefix.sql b/tests/queries/0_stateless/00187_like_regexp_prefix.sql index 7fec21b68ad..430b86e8b39 100644 --- a/tests/queries/0_stateless/00187_like_regexp_prefix.sql +++ b/tests/queries/0_stateless/00187_like_regexp_prefix.sql @@ -1 +1,3 @@ SELECT materialize('prepre_f') LIKE '%pre_f%'; + +SELECT materialize('prepre_f') LIKE '%%%pre_f%'; From 15ece2ab16b69ee04a71ce0de55b2ca14001b5c1 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sat, 25 Feb 2023 19:26:16 +0000 Subject: [PATCH 684/717] Fix incorrect LIKE-to-substring translation To be backported --- src/Functions/MatchImpl.h | 21 +++++++++++++++++-- .../25339_like_substring_search_bug.reference | 1 + .../25339_like_substring_search_bug.sql | 1 + 3 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/25339_like_substring_search_bug.reference create mode 100644 tests/queries/0_stateless/25339_like_substring_search_bug.sql diff --git a/src/Functions/MatchImpl.h b/src/Functions/MatchImpl.h index db8dd55474e..fe38ac49d62 100644 --- a/src/Functions/MatchImpl.h +++ b/src/Functions/MatchImpl.h @@ -23,9 +23,10 @@ namespace ErrorCodes namespace impl { -/// Is the [I]LIKE expression reduced to finding a substring in a string? +/// Is the [I]LIKE expression equivalent to a substring search? inline bool likePatternIsSubstring(std::string_view pattern, String & res) { + /// TODO: ignore multiple leading or trailing % if (pattern.size() < 2 || !pattern.starts_with('%') || !pattern.ends_with('%')) return false; @@ -45,9 +46,25 @@ inline bool likePatternIsSubstring(std::string_view pattern, String & res) case '\\': ++pos; if (pos == end) + /// pattern ends with \% --> trailing % is to be taken literally and pattern doesn't qualify for substring search return false; else - res += *pos; + { + switch (*pos) + { + /// Known LIKE escape sequences: + case '%': + case '_': + case '\\': + res += *pos; + break; + /// For all other escape sequences, the backslash loses its special meaning + default: + res += '\\'; + res += *pos; + break; + } + } break; default: res += *pos; diff --git a/tests/queries/0_stateless/25339_like_substring_search_bug.reference b/tests/queries/0_stateless/25339_like_substring_search_bug.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/25339_like_substring_search_bug.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/25339_like_substring_search_bug.sql b/tests/queries/0_stateless/25339_like_substring_search_bug.sql new file mode 100644 index 00000000000..10ce3cdde36 --- /dev/null +++ b/tests/queries/0_stateless/25339_like_substring_search_bug.sql @@ -0,0 +1 @@ +SELECT 'Win\Sys' LIKE '%Win\Sys%'; From 03e90cd7cf4830e2b61a8902e52574b3b7935950 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 25 Feb 2023 20:53:17 +0100 Subject: [PATCH 685/717] Add new metrics to system.asynchronous_metrics The following metrics can be useful to calculate various rates (i.e. disk/network IO rates): - AsynchronousHeavyMetricsUpdateInterval - AsynchronousMetricsUpdateInterval The following had been added by analogy with the AsynchronousMetricsCalculationTimeSpent: - AsynchronousHeavyMetricsCalculationTimeSpent Signed-off-by: Azat Khuzhin --- src/Common/AsynchronousMetrics.cpp | 9 ++++++++- src/Interpreters/ServerAsynchronousMetrics.cpp | 9 +++++++-- src/Interpreters/ServerAsynchronousMetrics.h | 1 + 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/Common/AsynchronousMetrics.cpp b/src/Common/AsynchronousMetrics.cpp index 1278f9459b5..99073d79bcd 100644 --- a/src/Common/AsynchronousMetrics.cpp +++ b/src/Common/AsynchronousMetrics.cpp @@ -533,9 +533,16 @@ void AsynchronousMetrics::update(TimePoint update_time) AsynchronousMetricValues new_values; auto current_time = std::chrono::system_clock::now(); - auto time_after_previous_update [[maybe_unused]] = current_time - previous_update_time; + auto time_after_previous_update = current_time - previous_update_time; previous_update_time = update_time; + double update_interval = 0.; + if (first_run) + update_interval = update_period.count(); + else + update_interval = std::chrono::duration_cast(time_after_previous_update).count() / 1e6; + new_values["AsynchronousMetricsUpdateInterval"] = { update_interval, "Metrics update interval" }; + /// This is also a good indicator of system responsiveness. new_values["Jitter"] = { std::chrono::duration_cast(current_time - update_time).count() / 1e9, "The difference in time the thread for calculation of the asynchronous metrics was scheduled to wake up and the time it was in fact, woken up." diff --git a/src/Interpreters/ServerAsynchronousMetrics.cpp b/src/Interpreters/ServerAsynchronousMetrics.cpp index 0d2032bed78..e6e1a03f11c 100644 --- a/src/Interpreters/ServerAsynchronousMetrics.cpp +++ b/src/Interpreters/ServerAsynchronousMetrics.cpp @@ -362,11 +362,14 @@ void ServerAsynchronousMetrics::updateHeavyMetricsIfNeeded(TimePoint current_tim const auto time_after_previous_update = current_time - heavy_metric_previous_update_time; const bool update_heavy_metric = time_after_previous_update >= heavy_metric_update_period || first_run; + Stopwatch watch; if (update_heavy_metric) { heavy_metric_previous_update_time = update_time; - - Stopwatch watch; + if (first_run) + heavy_update_interval = heavy_metric_update_period.count(); + else + heavy_update_interval = std::chrono::duration_cast(time_after_previous_update).count() / 1e6; /// Test shows that listing 100000 entries consuming around 0.15 sec. updateDetachedPartsStats(); @@ -390,7 +393,9 @@ void ServerAsynchronousMetrics::updateHeavyMetricsIfNeeded(TimePoint current_tim watch.elapsedSeconds()); } + new_values["AsynchronousHeavyMetricsCalculationTimeSpent"] = { watch.elapsedSeconds(), "Time in seconds spent for calculation of asynchronous heavy (tables related) metrics (this is the overhead of asynchronous metrics)." }; + new_values["AsynchronousHeavyMetricsUpdateInterval"] = { heavy_update_interval, "Heavy (tables related) metrics update interval" }; new_values["NumberOfDetachedParts"] = { detached_parts_stats.count, "The total number of parts detached from MergeTree tables. A part can be detached by a user with the `ALTER TABLE DETACH` query or by the server itself it the part is broken, unexpected or unneeded. The server does not care about detached parts and they can be removed." }; new_values["NumberOfDetachedByUserParts"] = { detached_parts_stats.detached_by_user, "The total number of parts detached from MergeTree tables by users with the `ALTER TABLE DETACH` query (as opposed to unexpected, broken or ignored parts). The server does not care about detached parts and they can be removed." }; diff --git a/src/Interpreters/ServerAsynchronousMetrics.h b/src/Interpreters/ServerAsynchronousMetrics.h index 81047e2fdf9..8243699a111 100644 --- a/src/Interpreters/ServerAsynchronousMetrics.h +++ b/src/Interpreters/ServerAsynchronousMetrics.h @@ -21,6 +21,7 @@ private: const Duration heavy_metric_update_period; TimePoint heavy_metric_previous_update_time; + double heavy_update_interval = 0.; struct DetachedPartsStats { From 88704c4d9fdb1f4b383feac01d60261ad7df7a8e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 25 Feb 2023 22:23:29 +0100 Subject: [PATCH 686/717] Fix flaky `test_concurrent_queries_restriction_by_query_kind` --- .../test.py | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_concurrent_queries_restriction_by_query_kind/test.py b/tests/integration/test_concurrent_queries_restriction_by_query_kind/test.py index 777c35f8b50..830090a1c0d 100644 --- a/tests/integration/test_concurrent_queries_restriction_by_query_kind/test.py +++ b/tests/integration/test_concurrent_queries_restriction_by_query_kind/test.py @@ -9,8 +9,16 @@ cluster = ClickHouseCluster(__file__) node_insert = cluster.add_instance( "node_insert", main_configs=["configs/concurrent_insert_restriction.xml"] ) -node_select = cluster.add_instance( - "node_select", main_configs=["configs/concurrent_select_restriction.xml"] +node_select1 = cluster.add_instance( + "node_select1", main_configs=["configs/concurrent_select_restriction.xml"] +) + +node_select2 = cluster.add_instance( + "node_select2", main_configs=["configs/concurrent_select_restriction.xml"] +) + +node_select3 = cluster.add_instance( + "node_select3", main_configs=["configs/concurrent_select_restriction.xml"] ) @@ -18,7 +26,13 @@ node_select = cluster.add_instance( def started_cluster(): try: cluster.start() - node_select.query( + node_select1.query( + "create table test_concurrent_insert (x UInt64) ENGINE = MergeTree() order by tuple()" + ) + node_select2.query( + "create table test_concurrent_insert (x UInt64) ENGINE = MergeTree() order by tuple()" + ) + node_select3.query( "create table test_concurrent_insert (x UInt64) ENGINE = MergeTree() order by tuple()" ) node_insert.query( @@ -79,7 +93,7 @@ def common_pattern(node, query_kind, restricted_sql, normal_sql, limit, wait_tim def test_select(started_cluster): common_pattern( - node_select, + node_select1, "select", "select sleep(3)", "insert into test_concurrent_insert values (0)", @@ -89,7 +103,7 @@ def test_select(started_cluster): # subquery is not counted execute_with_background( - node_select, + node_select2, "select sleep(3)", "insert into test_concurrent_insert select sleep(3)", 2, @@ -98,7 +112,7 @@ def test_select(started_cluster): # intersect and except are counted common_pattern( - node_select, + node_select3, "select", "select sleep(1) INTERSECT select sleep(1) EXCEPT select sleep(1)", "insert into test_concurrent_insert values (0)", From c668ddd50678bb065fe9fc40386b9d8bc8e37883 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sat, 25 Feb 2023 22:28:00 +0000 Subject: [PATCH 687/717] Make ASTSelectQuery::formatImpl() more robust Fixes #45204 The problem is that ASTSelectQuery::group_by_with_grouping_sets == true implies ASTSelectQuery::groupBy() but sometimes this wasn't the case. I added a sanity check a few months ago but had no idea how the AST became corrupt. All crashes/exceptions were during AST fuzzing. Looking at Client/QueryFuzzer.cpp, there is a very small chance to run into the issue. In detail: 1. In QueryFuzzer::fuzz(), we find that the AST is a ASTSelectQuery and groupBy() returns true. 2. With small probability, we do select->group_by_with_grouping_sets = !select->group_by_with_grouping_sets; where the (default false) group_by_with_grouping_sets flips true. 3. With small probability, we change the expression type in the following WHERE or PREWHERE if-branches. This situation is illegal. One possibility is changing the fuzzing code to not generate it. The fuzzing code is however generic, and doesn't really care about such details. Therefore, instead add an (theoretically unnecessary) extra check to ASTSelectQuery::formatImpl() for robustness. --- src/Parsers/ASTSelectQuery.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Parsers/ASTSelectQuery.cpp b/src/Parsers/ASTSelectQuery.cpp index fe8ebacec15..2936c2e9ef1 100644 --- a/src/Parsers/ASTSelectQuery.cpp +++ b/src/Parsers/ASTSelectQuery.cpp @@ -113,10 +113,8 @@ void ASTSelectQuery::formatImpl(const FormatSettings & s, FormatState & state, F if (group_by_with_cube) s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << (s.one_line ? "" : " ") << "WITH CUBE" << (s.hilite ? hilite_none : ""); - if (group_by_with_grouping_sets) + if (group_by_with_grouping_sets && groupBy()) { - if (!groupBy()) /// sanity check, issue 43049 - throw Exception(ErrorCodes::LOGICAL_ERROR, "Corrupt AST"); auto nested_frame = frame; nested_frame.surround_each_list_element_with_parens = true; nested_frame.expression_list_prepend_whitespace = false; From 2d38c89e2a6a0bdf3ffca3ac50fe9367c50b5b64 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Sat, 25 Feb 2023 23:29:51 +0100 Subject: [PATCH 688/717] Fix test test_async_backups_to_same_destination. --- .../test_backup_restore_new/test.py | 21 ++++++------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/tests/integration/test_backup_restore_new/test.py b/tests/integration/test_backup_restore_new/test.py index a903821071b..3f67fe8e5f7 100644 --- a/tests/integration/test_backup_restore_new/test.py +++ b/tests/integration/test_backup_restore_new/test.py @@ -648,24 +648,15 @@ def test_async_backups_to_same_destination(interface): "", ) - ids_succeeded = ( - instance.query( - f"SELECT id FROM system.backups WHERE id IN {ids_for_query} AND status == 'BACKUP_CREATED'" - ) - .rstrip("\n") - .split("\n") - ) + ids_succeeded = instance.query( + f"SELECT id FROM system.backups WHERE id IN {ids_for_query} AND status == 'BACKUP_CREATED'" + ).splitlines() - ids_failed = ( - instance.query( - f"SELECT id FROM system.backups WHERE id IN {ids_for_query} AND status == 'BACKUP_FAILED'" - ) - .rstrip("\n") - .split("\n") - ) + ids_failed = instance.query( + f"SELECT id FROM system.backups WHERE id IN {ids_for_query} AND status == 'BACKUP_FAILED'" + ).splitlines() assert len(ids_succeeded) == 1 - assert len(ids_failed) <= 1 assert set(ids_succeeded + ids_failed) == set(ids) # Check that the first backup is all right. From 7b8d2d716e6647def2639329deb9724ef0c46760 Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy Date: Sat, 25 Feb 2023 23:26:38 +0000 Subject: [PATCH 689/717] bypass empty blocks --- src/Processors/Transforms/FillingTransform.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Processors/Transforms/FillingTransform.cpp b/src/Processors/Transforms/FillingTransform.cpp index bfee5ed8bf9..e0c79d50141 100644 --- a/src/Processors/Transforms/FillingTransform.cpp +++ b/src/Processors/Transforms/FillingTransform.cpp @@ -269,6 +269,9 @@ void FillingTransform::transform(Chunk & chunk) if (on_totals) return; + if (!chunk.hasRows() && !generate_suffix) + return; + Columns old_fill_columns; Columns old_interpolate_columns; Columns old_other_columns; From 2472eeaefd36bb45c332352f1510d72903cf5cf9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 26 Feb 2023 02:58:32 +0300 Subject: [PATCH 690/717] Update 02575_merge_prewhere_materialized.sql --- .../0_stateless/02575_merge_prewhere_materialized.sql | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/queries/0_stateless/02575_merge_prewhere_materialized.sql b/tests/queries/0_stateless/02575_merge_prewhere_materialized.sql index 46794b181a0..eae72274c31 100644 --- a/tests/queries/0_stateless/02575_merge_prewhere_materialized.sql +++ b/tests/queries/0_stateless/02575_merge_prewhere_materialized.sql @@ -36,3 +36,8 @@ SELECT * FROM m PREWHERE a = 'OK' ORDER BY a, f; SELECT * FROM m PREWHERE f = 1 ORDER BY a, f; SELECT * FROM m WHERE f = 0 SETTINGS optimize_move_to_prewhere=0; SELECT * FROM m WHERE f = 0 SETTINGS optimize_move_to_prewhere=1; +-- { echoOff } + +DROP TABLE m; +DROP TABLE t1; +DROP TABLE t2; From dd25d8955e0bb11e06d3f54b78cff379b25b7f03 Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy Date: Sun, 26 Feb 2023 02:30:54 +0000 Subject: [PATCH 691/717] test is added --- .../0_stateless/25339_fill_empty_chunk.reference | 11 +++++++++++ tests/queries/0_stateless/25339_fill_empty_chunk.sql | 10 ++++++++++ 2 files changed, 21 insertions(+) create mode 100644 tests/queries/0_stateless/25339_fill_empty_chunk.reference create mode 100644 tests/queries/0_stateless/25339_fill_empty_chunk.sql diff --git a/tests/queries/0_stateless/25339_fill_empty_chunk.reference b/tests/queries/0_stateless/25339_fill_empty_chunk.reference new file mode 100644 index 00000000000..4dd51b52f2f --- /dev/null +++ b/tests/queries/0_stateless/25339_fill_empty_chunk.reference @@ -0,0 +1,11 @@ +1 \N +2 \N +2 \N +2 \N +3 \N +4 \N +5 \N +6 \N +7 \N +8 \N +9 \N diff --git a/tests/queries/0_stateless/25339_fill_empty_chunk.sql b/tests/queries/0_stateless/25339_fill_empty_chunk.sql new file mode 100644 index 00000000000..14ae322d8c9 --- /dev/null +++ b/tests/queries/0_stateless/25339_fill_empty_chunk.sql @@ -0,0 +1,10 @@ +-- this SELECT produces empty chunk in FillingTransform + +SELECT + 2 AS x, + arrayJoin([NULL, NULL, NULL]) +GROUP BY + GROUPING SETS ( + (0), + ([NULL, NULL, NULL])) +ORDER BY x ASC WITH FILL FROM 1 TO 10; From 8f40bbd5b492cf5c38c347c24e86e2c2dea2d771 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 26 Feb 2023 05:47:05 +0300 Subject: [PATCH 692/717] Revert "Fix several `RENAME COLUMN` bugs." --- src/Storages/MergeTree/AlterConversions.cpp | 55 ----- src/Storages/MergeTree/AlterConversions.h | 17 +- .../MergeTree/DataPartStorageOnDiskBase.cpp | 1 - src/Storages/MergeTree/DataPartsExchange.cpp | 16 +- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 43 +--- src/Storages/MergeTree/IMergeTreeDataPart.h | 19 +- src/Storages/MergeTree/MergeTask.cpp | 2 +- src/Storages/MergeTree/MergeTreeData.cpp | 29 +-- src/Storages/MergeTree/MergeTreeData.h | 2 +- .../MergeTree/MergeTreeDataPartInMemory.cpp | 4 +- .../MergeTree/MergeTreeDataWriter.cpp | 4 +- .../MergeTree/MergeTreeMarksLoader.cpp | 16 +- .../MergeTree/MergeTreeWriteAheadLog.cpp | 2 +- .../MergeTree/MergedBlockOutputStream.cpp | 10 +- .../MergedColumnOnlyOutputStream.cpp | 2 +- src/Storages/MergeTree/MutateTask.cpp | 216 ++++-------------- .../ReplicatedMergeTreeAttachThread.cpp | 2 +- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 51 +---- .../MergeTree/ReplicatedMergeTreeQueue.h | 6 +- src/Storages/MutationCommands.cpp | 16 -- src/Storages/MutationCommands.h | 8 - src/Storages/StorageInMemoryMetadata.cpp | 14 -- src/Storages/StorageInMemoryMetadata.h | 14 +- src/Storages/StorageMergeTree.cpp | 44 +--- src/Storages/StorageMergeTree.h | 2 +- src/Storages/StorageReplicatedMergeTree.cpp | 37 ++- src/Storages/StorageReplicatedMergeTree.h | 11 +- .../integration/test_merge_tree_hdfs/test.py | 14 +- tests/integration/test_merge_tree_s3/test.py | 15 +- .../test_merge_tree_s3_failover/test.py | 2 +- tests/integration/test_partition/test.py | 2 - .../test_replicated_merge_tree_s3/test.py | 14 +- .../test.py | 14 +- .../integration/test_s3_zero_copy_ttl/test.py | 4 +- .../01278_alter_rename_combination.reference | 4 +- .../01278_alter_rename_combination.sql | 2 +- ...1_alter_rename_and_other_renames.reference | 6 +- .../01281_alter_rename_and_other_renames.sql | 2 +- ...system_cache_on_write_operations.reference | 40 ++-- .../0_stateless/02361_fsync_profile_events.sh | 4 +- .../02538_alter_rename_sequence.reference | 8 - .../02538_alter_rename_sequence.sql | 59 ----- .../02543_alter_rename_modify_stuck.reference | 1 - .../02543_alter_rename_modify_stuck.sh | 58 ----- .../02555_davengers_rename_chain.reference | 26 --- .../02555_davengers_rename_chain.sh | 143 ------------ 46 files changed, 169 insertions(+), 892 deletions(-) delete mode 100644 src/Storages/MergeTree/AlterConversions.cpp delete mode 100644 tests/queries/0_stateless/02538_alter_rename_sequence.reference delete mode 100644 tests/queries/0_stateless/02538_alter_rename_sequence.sql delete mode 100644 tests/queries/0_stateless/02543_alter_rename_modify_stuck.reference delete mode 100755 tests/queries/0_stateless/02543_alter_rename_modify_stuck.sh delete mode 100644 tests/queries/0_stateless/02555_davengers_rename_chain.reference delete mode 100755 tests/queries/0_stateless/02555_davengers_rename_chain.sh diff --git a/src/Storages/MergeTree/AlterConversions.cpp b/src/Storages/MergeTree/AlterConversions.cpp deleted file mode 100644 index 7a298b0f6ca..00000000000 --- a/src/Storages/MergeTree/AlterConversions.cpp +++ /dev/null @@ -1,55 +0,0 @@ -#include -#include - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; -} - -bool AlterConversions::columnHasNewName(const std::string & old_name) const -{ - for (const auto & [new_name, prev_name] : rename_map) - { - if (old_name == prev_name) - return true; - } - - return false; -} - -std::string AlterConversions::getColumnNewName(const std::string & old_name) const -{ - for (const auto & [new_name, prev_name] : rename_map) - { - if (old_name == prev_name) - return new_name; - } - - throw Exception(ErrorCodes::LOGICAL_ERROR, "Column {} was not renamed", old_name); -} - - -bool AlterConversions::isColumnRenamed(const std::string & new_name) const -{ - for (const auto & [name_to, name_from] : rename_map) - { - if (name_to == new_name) - return true; - } - return false; -} -/// Get column old name before rename (lookup by key in rename_map) -std::string AlterConversions::getColumnOldName(const std::string & new_name) const -{ - for (const auto & [name_to, name_from] : rename_map) - { - if (name_to == new_name) - return name_from; - } - throw Exception(ErrorCodes::LOGICAL_ERROR, "Column {} was not renamed", new_name); -} - -} diff --git a/src/Storages/MergeTree/AlterConversions.h b/src/Storages/MergeTree/AlterConversions.h index ada385d6100..0d58499d424 100644 --- a/src/Storages/MergeTree/AlterConversions.h +++ b/src/Storages/MergeTree/AlterConversions.h @@ -14,22 +14,11 @@ namespace DB /// part->getColumns() and storage->getColumns(). struct AlterConversions { - struct RenamePair - { - std::string rename_to; - std::string rename_from; - }; /// Rename map new_name -> old_name - std::vector rename_map; + std::unordered_map rename_map; - /// Column was renamed (lookup by value in rename_map) - bool columnHasNewName(const std::string & old_name) const; - /// Get new name for column (lookup by value in rename_map) - std::string getColumnNewName(const std::string & old_name) const; - /// Is this name is new name of column (lookup by key in rename_map) - bool isColumnRenamed(const std::string & new_name) const; - /// Get column old name before rename (lookup by key in rename_map) - std::string getColumnOldName(const std::string & new_name) const; + bool isColumnRenamed(const std::string & new_name) const { return rename_map.count(new_name) > 0; } + std::string getColumnOldName(const std::string & new_name) const { return rename_map.at(new_name); } }; } diff --git a/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp b/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp index 260bf524ed2..d8769a94347 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp +++ b/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp @@ -683,7 +683,6 @@ void DataPartStorageOnDiskBase::clearDirectory( request.emplace_back(fs::path(dir) / "default_compression_codec.txt", true); request.emplace_back(fs::path(dir) / "delete-on-destroy.txt", true); request.emplace_back(fs::path(dir) / "txn_version.txt", true); - request.emplace_back(fs::path(dir) / "metadata_version.txt", true); /// Inverted index request.emplace_back(fs::path(dir) / "skp_idx_af.gin_dict", true); diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index 987ad2f2fe5..b671106f46a 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -63,9 +63,8 @@ constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_DEFAULT_COMPRESSION = 4; constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_UUID = 5; constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_ZERO_COPY = 6; constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_PROJECTION = 7; -constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_METADATA_VERSION = 8; // Reserved for ALTER PRIMARY KEY -// constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_PRIMARY_KEY = 9; +// constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_PRIMARY_KEY = 8; std::string getEndpointId(const std::string & node_id) { @@ -121,7 +120,7 @@ void Service::processQuery(const HTMLForm & params, ReadBuffer & /*body*/, Write MergeTreePartInfo::fromPartName(part_name, data.format_version); /// We pretend to work as older server version, to be sure that client will correctly process our version - response.addCookie({"server_protocol_version", toString(std::min(client_protocol_version, REPLICATION_PROTOCOL_VERSION_WITH_METADATA_VERSION))}); + response.addCookie({"server_protocol_version", toString(std::min(client_protocol_version, REPLICATION_PROTOCOL_VERSION_WITH_PARTS_PROJECTION))}); LOG_TRACE(log, "Sending part {}", part_name); @@ -281,10 +280,6 @@ MergeTreeData::DataPart::Checksums Service::sendPartFromDisk( && name == IMergeTreeDataPart::DEFAULT_COMPRESSION_CODEC_FILE_NAME) continue; - if (client_protocol_version < REPLICATION_PROTOCOL_VERSION_WITH_METADATA_VERSION - && name == IMergeTreeDataPart::METADATA_VERSION_FILE_NAME) - continue; - files_to_replicate.insert(name); } @@ -412,7 +407,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchSelectedPart( { {"endpoint", getEndpointId(replica_path)}, {"part", part_name}, - {"client_protocol_version", toString(REPLICATION_PROTOCOL_VERSION_WITH_METADATA_VERSION)}, + {"client_protocol_version", toString(REPLICATION_PROTOCOL_VERSION_WITH_PARTS_PROJECTION)}, {"compress", "false"} }); @@ -697,7 +692,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToMemory( auto block = block_in.read(); throttler->add(block.bytes()); - new_data_part->setColumns(block.getNamesAndTypesList(), {}, metadata_snapshot->getMetadataVersion()); + new_data_part->setColumns(block.getNamesAndTypesList(), {}); if (!is_projection) { @@ -773,8 +768,7 @@ void Fetcher::downloadBaseOrProjectionPartToDisk( if (file_name != "checksums.txt" && file_name != "columns.txt" && - file_name != IMergeTreeDataPart::DEFAULT_COMPRESSION_CODEC_FILE_NAME && - file_name != IMergeTreeDataPart::METADATA_VERSION_FILE_NAME) + file_name != IMergeTreeDataPart::DEFAULT_COMPRESSION_CODEC_FILE_NAME) checksums.addFile(file_name, file_size, expected_hash); } diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index d3ad9fb1eaa..06b4cb07f1f 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -416,11 +416,10 @@ std::pair IMergeTreeDataPart::getMinMaxTime() const } -void IMergeTreeDataPart::setColumns(const NamesAndTypesList & new_columns, const SerializationInfoByName & new_infos, int32_t metadata_version_) +void IMergeTreeDataPart::setColumns(const NamesAndTypesList & new_columns, const SerializationInfoByName & new_infos) { columns = new_columns; serialization_infos = new_infos; - metadata_version = metadata_version_; column_name_to_position.clear(); column_name_to_position.reserve(new_columns.size()); @@ -661,7 +660,6 @@ void IMergeTreeDataPart::appendFilesOfColumnsChecksumsIndexes(Strings & files, b appendFilesOfPartitionAndMinMaxIndex(files); appendFilesOfTTLInfos(files); appendFilesOfDefaultCompressionCodec(files); - appendFilesOfMetadataVersion(files); } if (!parent_part && include_projection) @@ -800,9 +798,6 @@ NameSet IMergeTreeDataPart::getFileNamesWithoutChecksums() const if (getDataPartStorage().exists(TXN_VERSION_METADATA_FILE_NAME)) result.emplace(TXN_VERSION_METADATA_FILE_NAME); - if (getDataPartStorage().exists(METADATA_VERSION_FILE_NAME)) - result.emplace(METADATA_VERSION_FILE_NAME); - return result; } @@ -976,22 +971,11 @@ void IMergeTreeDataPart::removeVersionMetadata() getDataPartStorage().removeFileIfExists("txn_version.txt"); } - -void IMergeTreeDataPart::removeMetadataVersion() -{ - getDataPartStorage().removeFileIfExists(METADATA_VERSION_FILE_NAME); -} - void IMergeTreeDataPart::appendFilesOfDefaultCompressionCodec(Strings & files) { files.push_back(DEFAULT_COMPRESSION_CODEC_FILE_NAME); } -void IMergeTreeDataPart::appendFilesOfMetadataVersion(Strings & files) -{ - files.push_back(METADATA_VERSION_FILE_NAME); -} - CompressionCodecPtr IMergeTreeDataPart::detectDefaultCompressionCodec() const { /// In memory parts doesn't have any compression @@ -1304,7 +1288,8 @@ void IMergeTreeDataPart::loadColumns(bool require) metadata_snapshot = metadata_snapshot->projections.get(name).metadata; NamesAndTypesList loaded_columns; - if (!metadata_manager->exists("columns.txt")) + bool exists = metadata_manager->exists("columns.txt"); + if (!exists) { /// We can get list of columns only from columns.txt in compact parts. if (require || part_type == Type::Compact) @@ -1337,32 +1322,16 @@ void IMergeTreeDataPart::loadColumns(bool require) }; SerializationInfoByName infos(loaded_columns, settings); - if (metadata_manager->exists(SERIALIZATION_FILE_NAME)) + exists = metadata_manager->exists(SERIALIZATION_FILE_NAME); + if (exists) { auto in = metadata_manager->read(SERIALIZATION_FILE_NAME); infos.readJSON(*in); } - int32_t loaded_metadata_version; - if (metadata_manager->exists(METADATA_VERSION_FILE_NAME)) - { - auto in = metadata_manager->read(METADATA_VERSION_FILE_NAME); - readIntText(loaded_metadata_version, *in); - } - else - { - loaded_metadata_version = metadata_snapshot->getMetadataVersion(); - - writeMetadata(METADATA_VERSION_FILE_NAME, {}, [loaded_metadata_version](auto & buffer) - { - writeIntText(loaded_metadata_version, buffer); - }); - } - - setColumns(loaded_columns, infos, loaded_metadata_version); + setColumns(loaded_columns, infos); } - /// Project part / part with project parts / compact part doesn't support LWD. bool IMergeTreeDataPart::supportLightweightDeleteMutate() const { diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index 4326aa630f9..ea1fd209a20 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -137,11 +137,7 @@ public: String getTypeName() const { return getType().toString(); } - /// We could have separate method like setMetadata, but it's much more convenient to set it up with columns - void setColumns(const NamesAndTypesList & new_columns, const SerializationInfoByName & new_infos, int32_t metadata_version_); - - /// Version of metadata for part (columns, pk and so on) - int32_t getMetadataVersion() const { return metadata_version; } + void setColumns(const NamesAndTypesList & new_columns, const SerializationInfoByName & new_infos); const NamesAndTypesList & getColumns() const { return columns; } const ColumnsDescription & getColumnsDescription() const { return columns_description; } @@ -312,9 +308,6 @@ public: mutable VersionMetadata version; - /// Version of part metadata (columns, pk and so on). Managed properly only for replicated merge tree. - int32_t metadata_version; - /// For data in RAM ('index') UInt64 getIndexSizeInBytes() const; UInt64 getIndexSizeInAllocatedBytes() const; @@ -386,12 +379,8 @@ public: /// (number of rows, number of rows with default values, etc). static inline constexpr auto SERIALIZATION_FILE_NAME = "serialization.json"; - /// Version used for transactions. static inline constexpr auto TXN_VERSION_METADATA_FILE_NAME = "txn_version.txt"; - - static inline constexpr auto METADATA_VERSION_FILE_NAME = "metadata_version.txt"; - /// One of part files which is used to check how many references (I'd like /// to say hardlinks, but it will confuse even more) we have for the part /// for zero copy replication. Sadly it's very complex. @@ -454,11 +443,7 @@ public: void writeDeleteOnDestroyMarker(); void removeDeleteOnDestroyMarker(); - /// It may look like a stupid joke. but these two methods are absolutely unrelated. - /// This one is about removing file with metadata about part version (for transactions) void removeVersionMetadata(); - /// This one is about removing file with version of part's metadata (columns, pk and so on) - void removeMetadataVersion(); mutable std::atomic removal_state = DataPartRemovalState::NOT_ATTEMPTED; @@ -597,8 +582,6 @@ private: static void appendFilesOfDefaultCompressionCodec(Strings & files); - static void appendFilesOfMetadataVersion(Strings & files); - /// Found column without specific compression and return codec /// for this column with default parameters. CompressionCodecPtr detectDefaultCompressionCodec() const; diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index ff72ef187f5..b961b70428e 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -208,7 +208,7 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() infos.add(part->getSerializationInfos()); } - global_ctx->new_data_part->setColumns(global_ctx->storage_columns, infos, global_ctx->metadata_snapshot->getMetadataVersion()); + global_ctx->new_data_part->setColumns(global_ctx->storage_columns, infos); const auto & local_part_min_ttl = global_ctx->new_data_part->ttl_infos.part_min_ttl; if (local_part_min_ttl && local_part_min_ttl <= global_ctx->time_of_merge) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index e19eb57e387..e36ab125f97 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -4427,11 +4427,6 @@ MergeTreeData::DataPartPtr MergeTreeData::getPartIfExistsUnlocked(const MergeTre static void loadPartAndFixMetadataImpl(MergeTreeData::MutableDataPartPtr part) { - /// Remove metadata version file and take it from table. - /// Currently we cannot attach parts with different schema, so - /// we can assume that it's equal to table's current schema. - part->removeMetadataVersion(); - part->loadColumnsChecksumsIndexes(false, true); part->modification_time = part->getDataPartStorage().getLastModified().epochTime(); part->removeDeleteOnDestroyMarker(); @@ -7654,23 +7649,15 @@ bool MergeTreeData::canUsePolymorphicParts(const MergeTreeSettings & settings, S AlterConversions MergeTreeData::getAlterConversionsForPart(const MergeTreeDataPartPtr part) const { - std::map commands_map = getAlterMutationCommandsForPart(part); + MutationCommands commands = getFirstAlterMutationCommandsForPart(part); AlterConversions result{}; - auto & rename_map = result.rename_map; - for (const auto & [version, commands] : commands_map) - { - for (const auto & command : commands) - { - /// Currently we need explicit conversions only for RENAME alter - /// all other conversions can be deduced from diff between part columns - /// and columns in storage. - if (command.type == MutationCommand::Type::RENAME_COLUMN) - { - rename_map.emplace_back(AlterConversions::RenamePair{command.rename_to, command.column_name}); - } - } - } + for (const auto & command : commands) + /// Currently we need explicit conversions only for RENAME alter + /// all other conversions can be deduced from diff between part columns + /// and columns in storage. + if (command.type == MutationCommand::Type::RENAME_COLUMN) + result.rename_map[command.rename_to] = command.column_name; return result; } @@ -8057,7 +8044,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::createEmptyPart( if (settings->assign_part_uuids) new_data_part->uuid = UUIDHelpers::generateV4(); - new_data_part->setColumns(columns, {}, metadata_snapshot->getMetadataVersion()); + new_data_part->setColumns(columns, {}); new_data_part->rows_count = block.rows(); new_data_part->partition = partition; diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index cae791d6ffe..faee69b137f 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -1313,7 +1313,7 @@ protected: /// Used to receive AlterConversions for part and apply them on fly. This /// method has different implementations for replicated and non replicated /// MergeTree because they store mutations in different way. - virtual std::map getAlterMutationCommandsForPart(const DataPartPtr & part) const = 0; + virtual MutationCommands getFirstAlterMutationCommandsForPart(const DataPartPtr & part) const = 0; /// Moves part to specified space, used in ALTER ... MOVE ... queries bool movePartsToSpace(const DataPartsVector & parts, SpacePtr space); diff --git a/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp b/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp index 5b1054d0a0e..20049976acf 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp @@ -73,7 +73,7 @@ MutableDataPartStoragePtr MergeTreeDataPartInMemory::flushToDisk(const String & new_data_part_storage->beginTransaction(); new_data_part->uuid = uuid; - new_data_part->setColumns(columns, {}, metadata_snapshot->getMetadataVersion()); + new_data_part->setColumns(columns, {}); new_data_part->partition.value = partition.value; new_data_part->minmax_idx = minmax_idx; @@ -104,7 +104,7 @@ MutableDataPartStoragePtr MergeTreeDataPartInMemory::flushToDisk(const String & .build(); new_projection_part->is_temp = false; // clean up will be done on parent part - new_projection_part->setColumns(projection->getColumns(), {}, metadata_snapshot->getMetadataVersion()); + new_projection_part->setColumns(projection->getColumns(), {}); auto new_projection_part_storage = new_projection_part->getDataPartStoragePtr(); if (new_projection_part_storage->exists()) diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index adb7505a8ba..37cfe4d065e 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -464,7 +464,7 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeTempPartImpl( SerializationInfoByName infos(columns, settings); infos.add(block); - new_data_part->setColumns(columns, infos, metadata_snapshot->getMetadataVersion()); + new_data_part->setColumns(columns, infos); new_data_part->rows_count = block.rows(); new_data_part->partition = std::move(partition); new_data_part->minmax_idx = std::move(minmax_idx); @@ -586,7 +586,7 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeProjectionPartImpl( SerializationInfoByName infos(columns, settings); infos.add(block); - new_data_part->setColumns(columns, infos, metadata_snapshot->getMetadataVersion()); + new_data_part->setColumns(columns, infos); if (new_data_part->isStoredOnDisk()) { diff --git a/src/Storages/MergeTree/MergeTreeMarksLoader.cpp b/src/Storages/MergeTree/MergeTreeMarksLoader.cpp index 1d85ac1bd34..3fc7ff54c35 100644 --- a/src/Storages/MergeTree/MergeTreeMarksLoader.cpp +++ b/src/Storages/MergeTree/MergeTreeMarksLoader.cpp @@ -102,15 +102,6 @@ MarkCache::MappedPtr MergeTreeMarksLoader::loadMarksImpl() auto res = std::make_shared(marks_count * columns_in_mark); - if (file_size == 0 && marks_count != 0) - { - throw Exception( - ErrorCodes::CORRUPTED_DATA, - "Empty marks file '{}': {}, must be: {}", - std::string(fs::path(data_part_storage->getFullPath()) / mrk_path), - file_size, expected_uncompressed_size); - } - if (!index_granularity_info.mark_type.compressed && expected_uncompressed_size != file_size) throw Exception( ErrorCodes::CORRUPTED_DATA, @@ -147,12 +138,7 @@ MarkCache::MappedPtr MergeTreeMarksLoader::loadMarksImpl() } if (i * mark_size != expected_uncompressed_size) - { - throw Exception( - ErrorCodes::CANNOT_READ_ALL_DATA, - "Cannot read all marks from file {}, marks expected {} (bytes size {}), marks read {} (bytes size {})", - mrk_path, marks_count, expected_uncompressed_size, i, reader->count()); - } + throw Exception(ErrorCodes::CANNOT_READ_ALL_DATA, "Cannot read all marks from file {}", mrk_path); } res->protect(); diff --git a/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp b/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp index 87d9bb0f168..fabf2acdad3 100644 --- a/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp +++ b/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp @@ -229,7 +229,7 @@ MergeTreeData::MutableDataPartsVector MergeTreeWriteAheadLog::restore( part->minmax_idx->update(block, storage.getMinMaxColumnsNames(metadata_snapshot->getPartitionKey())); part->partition.create(metadata_snapshot, block, 0, context); - part->setColumns(block.getNamesAndTypesList(), {}, metadata_snapshot->getMetadataVersion()); + part->setColumns(block.getNamesAndTypesList(), {}); if (metadata_snapshot->hasSortingKey()) metadata_snapshot->getSortingKey().expression->execute(block); diff --git a/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/src/Storages/MergeTree/MergedBlockOutputStream.cpp index 53d4a32fc0e..ced43ae25b0 100644 --- a/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -175,7 +175,7 @@ MergedBlockOutputStream::Finalizer MergedBlockOutputStream::finalizePartAsync( serialization_infos.replaceData(new_serialization_infos); files_to_remove_after_sync = removeEmptyColumnsFromPart(new_part, part_columns, serialization_infos, checksums); - new_part->setColumns(part_columns, serialization_infos, metadata_snapshot->getMetadataVersion()); + new_part->setColumns(part_columns, serialization_infos); } auto finalizer = std::make_unique(*writer, new_part, files_to_remove_after_sync, sync); @@ -289,14 +289,6 @@ MergedBlockOutputStream::WrittenFiles MergedBlockOutputStream::finalizePartOnDis written_files.emplace_back(std::move(out)); } - { - /// Write a file with a description of columns. - auto out = new_part->getDataPartStorage().writeFile(IMergeTreeDataPart::METADATA_VERSION_FILE_NAME, 4096, write_settings); - DB::writeIntText(new_part->getMetadataVersion(), *out); - out->preFinalize(); - written_files.emplace_back(std::move(out)); - } - if (default_codec != nullptr) { auto out = new_part->getDataPartStorage().writeFile(IMergeTreeDataPart::DEFAULT_COMPRESSION_CODEC_FILE_NAME, 4096, write_settings); diff --git a/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp b/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp index c70c5187b8b..03829f1daf9 100644 --- a/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp +++ b/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp @@ -85,7 +85,7 @@ MergedColumnOnlyOutputStream::fillChecksums( all_checksums.files.erase(removed_file); } - new_part->setColumns(columns, serialization_infos, metadata_snapshot->getMetadataVersion()); + new_part->setColumns(columns, serialization_infos); return checksums; } diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index ae34b1b707c..bcb1d5d2c28 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -53,7 +53,7 @@ static bool checkOperationIsNotCanceled(ActionBlocker & merges_blocker, MergeLis * First part should be executed by mutations interpreter. * Other is just simple drop/renames, so they can be executed without interpreter. */ -static void splitAndModifyMutationCommands( +static void splitMutationCommands( MergeTreeData::DataPartPtr part, const MutationCommands & commands, MutationCommands & for_interpreter, @@ -98,48 +98,25 @@ static void splitAndModifyMutationCommands( else mutated_columns.emplace(command.column_name); } + + if (command.type == MutationCommand::Type::RENAME_COLUMN) + { + for_interpreter.push_back( + { + .type = MutationCommand::Type::READ_COLUMN, + .column_name = command.rename_to, + }); + part_columns.rename(command.column_name, command.rename_to); + } } } - - auto alter_conversions = part->storage.getAlterConversionsForPart(part); - - /// We don't add renames from commands, instead we take them from rename_map. - /// It's important because required renames depend not only on part's data version (i.e. mutation version) - /// but also on part's metadata version. Why we have such logic only for renames? Because all other types of alter - /// can be deduced based on difference between part's schema and table schema. - for (const auto & [rename_to, rename_from] : alter_conversions.rename_map) - { - if (part_columns.has(rename_from)) - { - /// Actual rename - for_interpreter.push_back( - { - .type = MutationCommand::Type::READ_COLUMN, - .column_name = rename_to, - }); - - /// Not needed for compact parts (not executed), added here only to produce correct - /// set of columns for new part and their serializations - for_file_renames.push_back( - { - .type = MutationCommand::Type::RENAME_COLUMN, - .column_name = rename_from, - .rename_to = rename_to - }); - - part_columns.rename(rename_from, rename_to); - } - } - /// If it's compact part, then we don't need to actually remove files /// from disk we just don't read dropped columns - for (const auto & column : part_columns) + for (const auto & column : part->getColumns()) { if (!mutated_columns.contains(column.name)) - { for_interpreter.emplace_back( MutationCommand{.type = MutationCommand::Type::READ_COLUMN, .column_name = column.name, .data_type = column.type}); - } } } else @@ -165,59 +142,15 @@ static void splitAndModifyMutationCommands( { if (command.type == MutationCommand::Type::READ_COLUMN) for_interpreter.push_back(command); + else if (command.type == MutationCommand::Type::RENAME_COLUMN) + part_columns.rename(command.column_name, command.rename_to); for_file_renames.push_back(command); } } - - auto alter_conversions = part->storage.getAlterConversionsForPart(part); - /// We don't add renames from commands, instead we take them from rename_map. - /// It's important because required renames depend not only on part's data version (i.e. mutation version) - /// but also on part's metadata version. Why we have such logic only for renames? Because all other types of alter - /// can be deduced based on difference between part's schema and table schema. - - for (const auto & [rename_to, rename_from] : alter_conversions.rename_map) - { - for_file_renames.push_back({.type = MutationCommand::Type::RENAME_COLUMN, .column_name = rename_from, .rename_to = rename_to}); - } } } -/// It's legal to squash renames because commands with rename are always "barrier" -/// and executed separately from other types of commands. -static MutationCommands squashRenamesInCommands(const MutationCommands & commands) -{ - NameToNameMap squashed_renames; - for (const auto & command : commands) - { - std::string result_name = command.rename_to; - - bool squashed = false; - for (const auto & [name_from, name_to] : squashed_renames) - { - if (name_to == command.column_name) - { - squashed = true; - squashed_renames[name_from] = result_name; - break; - } - } - if (!squashed) - squashed_renames[command.column_name] = result_name; - } - - MutationCommands squashed_commands; - for (const auto & command : commands) - { - if (squashed_renames.contains(command.column_name)) - { - squashed_commands.push_back(command); - squashed_commands.back().rename_to = squashed_renames[command.column_name]; - } - } - return squashed_commands; -} - /// Get the columns list of the resulting part in the same order as storage_columns. static std::pair getColumnsForNewDataPart( @@ -225,13 +158,8 @@ getColumnsForNewDataPart( const Block & updated_header, NamesAndTypesList storage_columns, const SerializationInfoByName & serialization_infos, - const MutationCommands & commands_for_interpreter, const MutationCommands & commands_for_removes) { - MutationCommands all_commands; - all_commands.insert(all_commands.end(), commands_for_interpreter.begin(), commands_for_interpreter.end()); - all_commands.insert(all_commands.end(), commands_for_removes.begin(), commands_for_removes.end()); - NameSet removed_columns; NameToNameMap renamed_columns_to_from; NameToNameMap renamed_columns_from_to; @@ -247,9 +175,8 @@ getColumnsForNewDataPart( storage_columns.emplace_back(column); } - MutationCommands squashed_commands = squashRenamesInCommands(all_commands); - - for (const auto & command : squashed_commands) + /// All commands are validated in AlterCommand so we don't care about order + for (const auto & command : commands_for_removes) { if (command.type == MutationCommand::UPDATE) { @@ -342,38 +269,20 @@ getColumnsForNewDataPart( /// should it's previous version should be dropped or removed if (renamed_columns_to_from.contains(it->name) && !was_renamed && !was_removed) throw Exception( - ErrorCodes::LOGICAL_ERROR, - "Incorrect mutation commands, trying to rename column {} to {}, " - "but part {} already has column {}", - renamed_columns_to_from[it->name], it->name, source_part->name, it->name); + ErrorCodes::LOGICAL_ERROR, + "Incorrect mutation commands, trying to rename column {} to {}, " + "but part {} already has column {}", + renamed_columns_to_from[it->name], it->name, source_part->name, it->name); /// Column was renamed and no other column renamed to it's name /// or column is dropped. if (!renamed_columns_to_from.contains(it->name) && (was_renamed || was_removed)) - { it = storage_columns.erase(it); - } else { - - if (was_removed) - { /// DROP COLUMN xxx, RENAME COLUMN yyy TO xxx - auto renamed_from = renamed_columns_to_from.at(it->name); - auto maybe_name_and_type = source_columns.tryGetByName(renamed_from); - if (!maybe_name_and_type) - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "Got incorrect mutation commands, column {} was renamed from {}, but it doesn't exist in source columns {}", - it->name, renamed_from, source_columns.toString()); - - it->type = maybe_name_and_type->type; - } - else - { - /// Take a type from source part column. - /// It may differ from column type in storage. - it->type = source_col->second; - } + /// Take a type from source part column. + /// It may differ from column type in storage. + it->type = source_col->second; ++it; } } @@ -639,36 +548,27 @@ static NameToNameVector collectFilesForRenames( /// Collect counts for shared streams of different columns. As an example, Nested columns have shared stream with array sizes. auto stream_counts = getStreamCounts(source_part, source_part->getColumns().getNames()); NameToNameVector rename_vector; - NameSet collected_names; - - auto add_rename = [&rename_vector, &collected_names] (const std::string & file_rename_from, const std::string & file_rename_to) - { - if (collected_names.emplace(file_rename_from).second) - rename_vector.emplace_back(file_rename_from, file_rename_to); - }; - - MutationCommands squashed_commands = squashRenamesInCommands(commands_for_removes); /// Remove old data - for (const auto & command : squashed_commands) + for (const auto & command : commands_for_removes) { if (command.type == MutationCommand::Type::DROP_INDEX) { if (source_part->checksums.has(INDEX_FILE_PREFIX + command.column_name + ".idx2")) { - add_rename(INDEX_FILE_PREFIX + command.column_name + ".idx2", ""); - add_rename(INDEX_FILE_PREFIX + command.column_name + mrk_extension, ""); + rename_vector.emplace_back(INDEX_FILE_PREFIX + command.column_name + ".idx2", ""); + rename_vector.emplace_back(INDEX_FILE_PREFIX + command.column_name + mrk_extension, ""); } else if (source_part->checksums.has(INDEX_FILE_PREFIX + command.column_name + ".idx")) { - add_rename(INDEX_FILE_PREFIX + command.column_name + ".idx", ""); - add_rename(INDEX_FILE_PREFIX + command.column_name + mrk_extension, ""); + rename_vector.emplace_back(INDEX_FILE_PREFIX + command.column_name + ".idx", ""); + rename_vector.emplace_back(INDEX_FILE_PREFIX + command.column_name + mrk_extension, ""); } } else if (command.type == MutationCommand::Type::DROP_PROJECTION) { if (source_part->checksums.has(command.column_name + ".proj")) - add_rename(command.column_name + ".proj", ""); + rename_vector.emplace_back(command.column_name + ".proj", ""); } else if (command.type == MutationCommand::Type::DROP_COLUMN) { @@ -678,8 +578,8 @@ static NameToNameVector collectFilesForRenames( /// Delete files if they are no longer shared with another column. if (--stream_counts[stream_name] == 0) { - add_rename(stream_name + ".bin", ""); - add_rename(stream_name + mrk_extension, ""); + rename_vector.emplace_back(stream_name + ".bin", ""); + rename_vector.emplace_back(stream_name + mrk_extension, ""); } }; @@ -698,8 +598,8 @@ static NameToNameVector collectFilesForRenames( if (stream_from != stream_to) { - add_rename(stream_from + ".bin", stream_to + ".bin"); - add_rename(stream_from + mrk_extension, stream_to + mrk_extension); + rename_vector.emplace_back(stream_from + ".bin", stream_to + ".bin"); + rename_vector.emplace_back(stream_from + mrk_extension, stream_to + mrk_extension); } }; @@ -719,8 +619,8 @@ static NameToNameVector collectFilesForRenames( { if (!new_streams.contains(old_stream) && --stream_counts[old_stream] == 0) { - add_rename(old_stream + ".bin", ""); - add_rename(old_stream + mrk_extension, ""); + rename_vector.emplace_back(old_stream + ".bin", ""); + rename_vector.emplace_back(old_stream + mrk_extension, ""); } } } @@ -737,7 +637,6 @@ void finalizeMutatedPart( ExecuteTTLType execute_ttl_type, const CompressionCodecPtr & codec, ContextPtr context, - StorageMetadataPtr metadata_snapshot, bool sync) { std::vector> written_files; @@ -786,12 +685,6 @@ void finalizeMutatedPart( written_files.push_back(std::move(out_comp)); } - { - auto out_metadata = new_data_part->getDataPartStorage().writeFile(IMergeTreeDataPart::METADATA_VERSION_FILE_NAME, 4096, context->getWriteSettings()); - DB::writeText(metadata_snapshot->getMetadataVersion(), *out_metadata); - written_files.push_back(std::move(out_metadata)); - } - { /// Write a file with a description of columns. auto out_columns = new_data_part->getDataPartStorage().writeFile("columns.txt", 4096, context->getWriteSettings()); @@ -1431,27 +1324,13 @@ private: ctx->new_data_part->storeVersionMetadata(); NameSet hardlinked_files; - - /// NOTE: Renames must be done in order - for (const auto & [rename_from, rename_to] : ctx->files_to_rename) - { - if (rename_to.empty()) /// It's DROP COLUMN - { - /// pass - } - else - { - ctx->new_data_part->getDataPartStorage().createHardLinkFrom( - ctx->source_part->getDataPartStorage(), rename_from, rename_to); - hardlinked_files.insert(rename_from); - } - } /// Create hardlinks for unchanged files for (auto it = ctx->source_part->getDataPartStorage().iterate(); it->isValid(); it->next()) { if (ctx->files_to_skip.contains(it->name())) continue; + String destination; String file_name = it->name(); auto rename_it = std::find_if(ctx->files_to_rename.begin(), ctx->files_to_rename.end(), [&file_name](const auto & rename_pair) @@ -1461,17 +1340,20 @@ private: if (rename_it != ctx->files_to_rename.end()) { - /// RENAMEs and DROPs already processed - continue; + if (rename_it->second.empty()) + continue; + destination = rename_it->second; + } + else + { + destination = it->name(); } - - String destination = it->name(); if (it->isFile()) { ctx->new_data_part->getDataPartStorage().createHardLinkFrom( - ctx->source_part->getDataPartStorage(), file_name, destination); - hardlinked_files.insert(file_name); + ctx->source_part->getDataPartStorage(), it->name(), destination); + hardlinked_files.insert(it->name()); } else if (!endsWith(it->name(), ".tmp_proj")) // ignore projection tmp merge dir { @@ -1567,7 +1449,7 @@ private: } } - MutationHelpers::finalizeMutatedPart(ctx->source_part, ctx->new_data_part, ctx->execute_ttl_type, ctx->compression_codec, ctx->context, ctx->metadata_snapshot, ctx->need_sync); + MutationHelpers::finalizeMutatedPart(ctx->source_part, ctx->new_data_part, ctx->execute_ttl_type, ctx->compression_codec, ctx->context, ctx->need_sync); } @@ -1729,7 +1611,7 @@ bool MutateTask::prepare() context_for_reading->setSetting("allow_asynchronous_read_from_io_pool_for_merge_tree", false); context_for_reading->setSetting("max_streams_for_merge_tree_reading", Field(0)); - MutationHelpers::splitAndModifyMutationCommands(ctx->source_part, ctx->commands_for_part, ctx->for_interpreter, ctx->for_file_renames); + MutationHelpers::splitMutationCommands(ctx->source_part, ctx->commands_for_part, ctx->for_interpreter, ctx->for_file_renames); ctx->stage_progress = std::make_unique(1.0); @@ -1774,9 +1656,9 @@ bool MutateTask::prepare() auto [new_columns, new_infos] = MutationHelpers::getColumnsForNewDataPart( ctx->source_part, ctx->updated_header, ctx->storage_columns, - ctx->source_part->getSerializationInfos(), ctx->for_interpreter, ctx->for_file_renames); + ctx->source_part->getSerializationInfos(), ctx->commands_for_part); - ctx->new_data_part->setColumns(new_columns, new_infos, ctx->metadata_snapshot->getMetadataVersion()); + ctx->new_data_part->setColumns(new_columns, new_infos); ctx->new_data_part->partition.assign(ctx->source_part->partition); /// Don't change granularity type while mutating subset of columns diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp index f1bdc9f43af..557123ddae2 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp @@ -149,7 +149,7 @@ void ReplicatedMergeTreeAttachThread::runImpl() const bool replica_metadata_version_exists = zookeeper->tryGet(replica_path + "/metadata_version", replica_metadata_version); if (replica_metadata_version_exists) { - storage.setInMemoryMetadata(metadata_snapshot->withMetadataVersion(parse(replica_metadata_version))); + storage.metadata_version = parse(replica_metadata_version); } else { diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index bac5beb6364..125529e7b94 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -11,7 +11,6 @@ #include #include -#include namespace DB { @@ -1755,40 +1754,19 @@ ReplicatedMergeTreeMergePredicate ReplicatedMergeTreeQueue::getMergePredicate(zk } -std::map ReplicatedMergeTreeQueue::getAlterMutationCommandsForPart(const MergeTreeData::DataPartPtr & part) const +MutationCommands ReplicatedMergeTreeQueue::getFirstAlterMutationCommandsForPart(const MergeTreeData::DataPartPtr & part) const { - std::unique_lock lock(state_mutex); + std::lock_guard lock(state_mutex); auto in_partition = mutations_by_partition.find(part->info.partition_id); if (in_partition == mutations_by_partition.end()) - return {}; + return MutationCommands{}; - Int64 part_metadata_version = part->getMetadataVersion(); - std::map result; - /// Here we return mutation commands for part which has bigger alter version than part metadata version. - /// Please note, we don't use getDataVersion(). It's because these alter commands are used for in-fly conversions - /// of part's metadata. - for (const auto & [mutation_version, mutation_status] : in_partition->second | std::views::reverse) - { - int32_t alter_version = mutation_status->entry->alter_version; - if (alter_version != -1) - { - if (!alter_sequence.canExecuteDataAlter(alter_version, lock)) - continue; + Int64 part_version = part->info.getDataVersion(); + for (auto [mutation_version, mutation_status] : in_partition->second) + if (mutation_version > part_version && mutation_status->entry->alter_version != -1) + return mutation_status->entry->commands; - /// we take commands with bigger metadata version - if (alter_version > part_metadata_version) - { - result[mutation_version] = mutation_status->entry->commands; - } - else - { - /// entries are ordered, we processing them in reverse order so we can break - break; - } - } - } - - return result; + return MutationCommands{}; } MutationCommands ReplicatedMergeTreeQueue::getMutationCommands( @@ -1830,18 +1808,7 @@ MutationCommands ReplicatedMergeTreeQueue::getMutationCommands( MutationCommands commands; for (auto it = begin; it != end; ++it) - { - const auto & commands_from_entry = it->second->entry->commands; - - if (commands_from_entry.containBarrierCommand()) - { - if (commands.empty()) - commands.insert(commands.end(), commands_from_entry.begin(), commands_from_entry.end()); - break; - } - else - commands.insert(commands.end(), commands_from_entry.begin(), commands_from_entry.end()); - } + commands.insert(commands.end(), it->second->entry->commands.begin(), it->second->entry->commands.end()); return commands; } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h index df1fd5a3c85..36552129690 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h @@ -393,10 +393,10 @@ public: MutationCommands getMutationCommands(const MergeTreeData::DataPartPtr & part, Int64 desired_mutation_version) const; - /// Return mutation commands for part which could be not applied to - /// it according to part mutation version. Used when we apply alter commands on fly, + /// Return mutation commands for part with smallest mutation version bigger + /// than data part version. Used when we apply alter commands on fly, /// without actual data modification on disk. - std::map getAlterMutationCommandsForPart(const MergeTreeData::DataPartPtr & part) const; + MutationCommands getFirstAlterMutationCommandsForPart(const MergeTreeData::DataPartPtr & part) const; /// Mark finished mutations as done. If the function needs to be called again at some later time /// (because some mutations are probably done but we are not sure yet), returns true. diff --git a/src/Storages/MutationCommands.cpp b/src/Storages/MutationCommands.cpp index aa77988348d..0c9e9223929 100644 --- a/src/Storages/MutationCommands.cpp +++ b/src/Storages/MutationCommands.cpp @@ -23,12 +23,6 @@ namespace ErrorCodes extern const int MULTIPLE_ASSIGNMENTS_TO_COLUMN; } - -bool MutationCommand::isBarrierCommand() const -{ - return type == RENAME_COLUMN; -} - std::optional MutationCommand::parse(ASTAlterCommand * command, bool parse_alter_commands) { if (command->type == ASTAlterCommand::DELETE) @@ -218,14 +212,4 @@ bool MutationCommands::hasNonEmptyMutationCommands() const return false; } -bool MutationCommands::containBarrierCommand() const -{ - for (const auto & command : *this) - { - if (command.isBarrierCommand()) - return true; - } - return false; -} - } diff --git a/src/Storages/MutationCommands.h b/src/Storages/MutationCommands.h index 5ef0cfda1be..aca91c16e85 100644 --- a/src/Storages/MutationCommands.h +++ b/src/Storages/MutationCommands.h @@ -67,9 +67,6 @@ struct MutationCommand /// If parse_alter_commands, than consider more Alter commands as mutation commands static std::optional parse(ASTAlterCommand * command, bool parse_alter_commands = false); - - /// This command shouldn't stick with other commands - bool isBarrierCommand() const; }; /// Multiple mutation commands, possible from different ALTER queries @@ -82,11 +79,6 @@ public: void readText(ReadBuffer & in); std::string toString() const; bool hasNonEmptyMutationCommands() const; - - /// These set of commands contain barrier command and shouldn't - /// stick with other commands. Commands from one set have already been validated - /// to be executed without issues on the creation state. - bool containBarrierCommand() const; }; using MutationCommandsConstPtr = std::shared_ptr; diff --git a/src/Storages/StorageInMemoryMetadata.cpp b/src/Storages/StorageInMemoryMetadata.cpp index 45abd4bebef..f6550c6cd5d 100644 --- a/src/Storages/StorageInMemoryMetadata.cpp +++ b/src/Storages/StorageInMemoryMetadata.cpp @@ -41,7 +41,6 @@ StorageInMemoryMetadata::StorageInMemoryMetadata(const StorageInMemoryMetadata & , settings_changes(other.settings_changes ? other.settings_changes->clone() : nullptr) , select(other.select) , comment(other.comment) - , metadata_version(other.metadata_version) { } @@ -70,7 +69,6 @@ StorageInMemoryMetadata & StorageInMemoryMetadata::operator=(const StorageInMemo settings_changes.reset(); select = other.select; comment = other.comment; - metadata_version = other.metadata_version; return *this; } @@ -124,18 +122,6 @@ void StorageInMemoryMetadata::setSelectQuery(const SelectQueryDescription & sele select = select_; } -void StorageInMemoryMetadata::setMetadataVersion(int32_t metadata_version_) -{ - metadata_version = metadata_version_; -} - -StorageInMemoryMetadata StorageInMemoryMetadata::withMetadataVersion(int32_t metadata_version_) const -{ - StorageInMemoryMetadata copy(*this); - copy.setMetadataVersion(metadata_version_); - return copy; -} - const ColumnsDescription & StorageInMemoryMetadata::getColumns() const { return columns; diff --git a/src/Storages/StorageInMemoryMetadata.h b/src/Storages/StorageInMemoryMetadata.h index 25618c5b03f..eadce581334 100644 --- a/src/Storages/StorageInMemoryMetadata.h +++ b/src/Storages/StorageInMemoryMetadata.h @@ -50,10 +50,6 @@ struct StorageInMemoryMetadata String comment; - /// Version of metadata. Managed properly by ReplicatedMergeTree only - /// (zero-initialization is important) - int32_t metadata_version = 0; - StorageInMemoryMetadata() = default; StorageInMemoryMetadata(const StorageInMemoryMetadata & other); @@ -62,7 +58,7 @@ struct StorageInMemoryMetadata StorageInMemoryMetadata(StorageInMemoryMetadata && other) = default; StorageInMemoryMetadata & operator=(StorageInMemoryMetadata && other) = default; - /// NOTE: Thread unsafe part. You should not modify same StorageInMemoryMetadata + /// NOTE: Thread unsafe part. You should modify same StorageInMemoryMetadata /// structure from different threads. It should be used as MultiVersion /// object. See example in IStorage. @@ -94,11 +90,6 @@ struct StorageInMemoryMetadata /// Set SELECT query for (Materialized)View void setSelectQuery(const SelectQueryDescription & select_); - /// Set version of metadata. - void setMetadataVersion(int32_t metadata_version_); - /// Get copy of current metadata with metadata_version_ - StorageInMemoryMetadata withMetadataVersion(int32_t metadata_version_) const; - /// Returns combined set of columns const ColumnsDescription & getColumns() const; @@ -227,9 +218,6 @@ struct StorageInMemoryMetadata const SelectQueryDescription & getSelectQuery() const; bool hasSelectQuery() const; - /// Get version of metadata - int32_t getMetadataVersion() const { return metadata_version; } - /// Check that all the requested names are in the table and have the correct types. void check(const NamesAndTypesList & columns) const; diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 11922bebe7b..8210ef93172 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -1143,24 +1143,9 @@ MergeMutateSelectedEntryPtr StorageMergeTree::selectPartsToMutate( if (current_ast_elements + commands_size >= max_ast_elements) break; - const auto & single_mutation_commands = it->second.commands; - - if (single_mutation_commands.containBarrierCommand()) - { - if (commands->empty()) - { - commands->insert(commands->end(), single_mutation_commands.begin(), single_mutation_commands.end()); - last_mutation_to_apply = it; - } - break; - } - else - { - current_ast_elements += commands_size; - commands->insert(commands->end(), single_mutation_commands.begin(), single_mutation_commands.end()); - last_mutation_to_apply = it; - } - + current_ast_elements += commands_size; + commands->insert(commands->end(), it->second.commands.begin(), it->second.commands.end()); + last_mutation_to_apply = it; } assert(commands->empty() == (last_mutation_to_apply == mutations_end_it)); @@ -1255,10 +1240,7 @@ bool StorageMergeTree::scheduleDataProcessingJob(BackgroundJobsAssignee & assign } if (mutate_entry) { - /// We take new metadata snapshot here. It's because mutation commands can be executed only with metadata snapshot - /// which is equal or more fresh than commands themselves. In extremely rare case it can happen that we will have alter - /// in between we took snapshot above and selected commands. That is why we take new snapshot here. - auto task = std::make_shared(*this, getInMemoryMetadataPtr(), mutate_entry, shared_lock, common_assignee_trigger); + auto task = std::make_shared(*this, metadata_snapshot, mutate_entry, shared_lock, common_assignee_trigger); assignee.scheduleMergeMutateTask(task); return true; } @@ -2127,22 +2109,14 @@ void StorageMergeTree::attachRestoredParts(MutableDataPartsVector && parts) } -std::map StorageMergeTree::getAlterMutationCommandsForPart(const DataPartPtr & part) const +MutationCommands StorageMergeTree::getFirstAlterMutationCommandsForPart(const DataPartPtr & part) const { std::lock_guard lock(currently_processing_in_background_mutex); - Int64 part_data_version = part->info.getDataVersion(); - - std::map result; - if (!current_mutations_by_version.empty()) - { - const auto & [latest_mutation_id, latest_commands] = *current_mutations_by_version.rbegin(); - if (part_data_version < static_cast(latest_mutation_id)) - { - result[latest_mutation_id] = latest_commands.commands; - } - } - return result; + auto it = current_mutations_by_version.upper_bound(part->info.getDataVersion()); + if (it == current_mutations_by_version.end()) + return {}; + return it->second.commands; } void StorageMergeTree::startBackgroundMovesIfNeeded() diff --git a/src/Storages/StorageMergeTree.h b/src/Storages/StorageMergeTree.h index a0629bb8d3e..052a672a187 100644 --- a/src/Storages/StorageMergeTree.h +++ b/src/Storages/StorageMergeTree.h @@ -267,7 +267,7 @@ private: protected: - std::map getAlterMutationCommandsForPart(const DataPartPtr & part) const override; + MutationCommands getFirstAlterMutationCommandsForPart(const DataPartPtr & part) const override; }; } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 8ae9b1baa96..429339386b4 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -461,7 +461,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( Coordination::Stat metadata_stat; current_zookeeper->get(zookeeper_path + "/metadata", &metadata_stat); - setInMemoryMetadata(metadata_snapshot->withMetadataVersion(metadata_stat.version)); + metadata_version = metadata_stat.version; } catch (Coordination::Exception & e) { @@ -781,7 +781,7 @@ bool StorageReplicatedMergeTree::createTableIfNotExists(const StorageMetadataPtr zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest(replica_path + "/columns", metadata_snapshot->getColumns().toString(), zkutil::CreateMode::Persistent)); - ops.emplace_back(zkutil::makeCreateRequest(replica_path + "/metadata_version", toString(metadata_snapshot->getMetadataVersion()), + ops.emplace_back(zkutil::makeCreateRequest(replica_path + "/metadata_version", std::to_string(metadata_version), zkutil::CreateMode::Persistent)); /// The following 3 nodes were added in version 1.1.xxx, so we create them here, not in createNewZooKeeperNodes() @@ -854,7 +854,7 @@ void StorageReplicatedMergeTree::createReplica(const StorageMetadataPtr & metada zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest(replica_path + "/columns", metadata_snapshot->getColumns().toString(), zkutil::CreateMode::Persistent)); - ops.emplace_back(zkutil::makeCreateRequest(replica_path + "/metadata_version", toString(metadata_snapshot->getMetadataVersion()), + ops.emplace_back(zkutil::makeCreateRequest(replica_path + "/metadata_version", std::to_string(metadata_version), zkutil::CreateMode::Persistent)); /// The following 3 nodes were added in version 1.1.xxx, so we create them here, not in createNewZooKeeperNodes() @@ -1159,19 +1159,16 @@ void StorageReplicatedMergeTree::checkTableStructure(const String & zookeeper_pr } void StorageReplicatedMergeTree::setTableStructure(const StorageID & table_id, const ContextPtr & local_context, - ColumnsDescription new_columns, const ReplicatedMergeTreeTableMetadata::Diff & metadata_diff, int32_t new_metadata_version) + ColumnsDescription new_columns, const ReplicatedMergeTreeTableMetadata::Diff & metadata_diff) { StorageInMemoryMetadata old_metadata = getInMemoryMetadata(); - StorageInMemoryMetadata new_metadata = metadata_diff.getNewMetadata(new_columns, local_context, old_metadata); - new_metadata.setMetadataVersion(new_metadata_version); /// Even if the primary/sorting/partition keys didn't change we must reinitialize it /// because primary/partition key column types might have changed. checkTTLExpressions(new_metadata, old_metadata); setProperties(new_metadata, old_metadata); - DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(local_context, table_id, new_metadata); } @@ -2785,9 +2782,8 @@ void StorageReplicatedMergeTree::cloneMetadataIfNeeded(const String & source_rep return; } - auto metadata_snapshot = getInMemoryMetadataPtr(); Int32 source_metadata_version = parse(source_metadata_version_str); - if (metadata_snapshot->getMetadataVersion() == source_metadata_version) + if (metadata_version == source_metadata_version) return; /// Our metadata it not up to date with source replica metadata. @@ -2805,7 +2801,7 @@ void StorageReplicatedMergeTree::cloneMetadataIfNeeded(const String & source_rep /// if all such entries were cleaned up from the log and source_queue. LOG_WARNING(log, "Metadata version ({}) on replica is not up to date with metadata ({}) on source replica {}", - metadata_snapshot->getMetadataVersion(), source_metadata_version, source_replica); + metadata_version, source_metadata_version, source_replica); String source_metadata; String source_columns; @@ -4984,15 +4980,14 @@ bool StorageReplicatedMergeTree::optimize( bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMergeTree::LogEntry & entry) { - auto current_metadata = getInMemoryMetadataPtr(); - if (entry.alter_version < current_metadata->getMetadataVersion()) + if (entry.alter_version < metadata_version) { /// TODO Can we replace it with LOGICAL_ERROR? /// As for now, it may rarely happen due to reordering of ALTER_METADATA entries in the queue of /// non-initial replica and also may happen after stale replica recovery. LOG_WARNING(log, "Attempt to update metadata of version {} " "to older version {} when processing log entry {}: {}", - current_metadata->getMetadataVersion(), entry.alter_version, entry.znode_name, entry.toString()); + metadata_version, entry.alter_version, entry.znode_name, entry.toString()); return true; } @@ -5040,10 +5035,10 @@ bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMer LOG_INFO(log, "Metadata changed in ZooKeeper. Applying changes locally."); auto metadata_diff = ReplicatedMergeTreeTableMetadata(*this, getInMemoryMetadataPtr()).checkAndFindDiff(metadata_from_entry, getInMemoryMetadataPtr()->getColumns(), getContext()); - setTableStructure(table_id, alter_context, std::move(columns_from_entry), metadata_diff, entry.alter_version); + setTableStructure(table_id, alter_context, std::move(columns_from_entry), metadata_diff); + metadata_version = entry.alter_version; - current_metadata = getInMemoryMetadataPtr(); - LOG_INFO(log, "Applied changes to the metadata of the table. Current metadata version: {}", current_metadata->getMetadataVersion()); + LOG_INFO(log, "Applied changes to the metadata of the table. Current metadata version: {}", metadata_version); } { @@ -5055,7 +5050,7 @@ bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMer /// This transaction may not happen, but it's OK, because on the next retry we will eventually create/update this node /// TODO Maybe do in in one transaction for Replicated database? - zookeeper->createOrUpdate(fs::path(replica_path) / "metadata_version", std::to_string(current_metadata->getMetadataVersion()), zkutil::CreateMode::Persistent); + zookeeper->createOrUpdate(fs::path(replica_path) / "metadata_version", std::to_string(metadata_version), zkutil::CreateMode::Persistent); return true; } @@ -5179,7 +5174,7 @@ void StorageReplicatedMergeTree::alter( size_t mutation_path_idx = std::numeric_limits::max(); String new_metadata_str = future_metadata_in_zk.toString(); - ops.emplace_back(zkutil::makeSetRequest(fs::path(zookeeper_path) / "metadata", new_metadata_str, current_metadata->getMetadataVersion())); + ops.emplace_back(zkutil::makeSetRequest(fs::path(zookeeper_path) / "metadata", new_metadata_str, metadata_version)); String new_columns_str = future_metadata.columns.toString(); ops.emplace_back(zkutil::makeSetRequest(fs::path(zookeeper_path) / "columns", new_columns_str, -1)); @@ -5195,7 +5190,7 @@ void StorageReplicatedMergeTree::alter( /// We can be sure, that in case of successful commit in zookeeper our /// version will increments by 1. Because we update with version check. - int new_metadata_version = current_metadata->getMetadataVersion() + 1; + int new_metadata_version = metadata_version + 1; alter_entry->type = LogEntry::ALTER_METADATA; alter_entry->source_replica = replica_name; @@ -7969,9 +7964,9 @@ bool StorageReplicatedMergeTree::canUseAdaptiveGranularity() const } -std::map StorageReplicatedMergeTree::getAlterMutationCommandsForPart(const DataPartPtr & part) const +MutationCommands StorageReplicatedMergeTree::getFirstAlterMutationCommandsForPart(const DataPartPtr & part) const { - return queue.getAlterMutationCommandsForPart(part); + return queue.getFirstAlterMutationCommandsForPart(part); } diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index ac8229481f9..46c78e9064a 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -215,6 +215,8 @@ public: /// It's used if not set in engine's arguments while creating a replicated table. static String getDefaultReplicaName(const ContextPtr & context_); + int getMetadataVersion() const { return metadata_version; } + /// Modify a CREATE TABLE query to make a variant which must be written to a backup. void adjustCreateQueryForBackup(ASTPtr & create_query) const override; @@ -426,6 +428,7 @@ private: std::atomic shutdown_called {false}; std::atomic flush_called {false}; + int metadata_version = 0; /// Threads. /// A task that keeps track of the updates in the logs of all replicas and loads them into the queue. @@ -502,10 +505,8 @@ private: /// A part of ALTER: apply metadata changes only (data parts are altered separately). /// Must be called under IStorage::lockForAlter() lock. - void setTableStructure( - const StorageID & table_id, const ContextPtr & local_context, - ColumnsDescription new_columns, const ReplicatedMergeTreeTableMetadata::Diff & metadata_diff, - int32_t new_metadata_version); + void setTableStructure(const StorageID & table_id, const ContextPtr & local_context, + ColumnsDescription new_columns, const ReplicatedMergeTreeTableMetadata::Diff & metadata_diff); /** Check that the set of parts corresponds to that in ZK (/replicas/me/parts/). * If any parts described in ZK are not locally, throw an exception. @@ -832,7 +833,7 @@ private: void waitMutationToFinishOnReplicas( const Strings & replicas, const String & mutation_id) const; - std::map getAlterMutationCommandsForPart(const DataPartPtr & part) const override; + MutationCommands getFirstAlterMutationCommandsForPart(const DataPartPtr & part) const override; void startBackgroundMovesIfNeeded() override; diff --git a/tests/integration/test_merge_tree_hdfs/test.py b/tests/integration/test_merge_tree_hdfs/test.py index 782237539fa..3950077e619 100644 --- a/tests/integration/test_merge_tree_hdfs/test.py +++ b/tests/integration/test_merge_tree_hdfs/test.py @@ -43,18 +43,8 @@ def create_table(cluster, table_name, additional_settings=None): FILES_OVERHEAD = 1 FILES_OVERHEAD_PER_COLUMN = 2 # Data and mark files -FILES_OVERHEAD_DEFAULT_COMPRESSION_CODEC = 1 -FILES_OVERHEAD_METADATA_VERSION = 1 -FILES_OVERHEAD_PER_PART_WIDE = ( - FILES_OVERHEAD_PER_COLUMN * 3 - + 2 - + 6 - + FILES_OVERHEAD_DEFAULT_COMPRESSION_CODEC - + FILES_OVERHEAD_METADATA_VERSION -) -FILES_OVERHEAD_PER_PART_COMPACT = ( - 10 + FILES_OVERHEAD_DEFAULT_COMPRESSION_CODEC + FILES_OVERHEAD_METADATA_VERSION -) +FILES_OVERHEAD_PER_PART_WIDE = FILES_OVERHEAD_PER_COLUMN * 3 + 2 + 6 + 1 +FILES_OVERHEAD_PER_PART_COMPACT = 10 + 1 @pytest.fixture(scope="module") diff --git a/tests/integration/test_merge_tree_s3/test.py b/tests/integration/test_merge_tree_s3/test.py index 696c016f760..f0f81100320 100644 --- a/tests/integration/test_merge_tree_s3/test.py +++ b/tests/integration/test_merge_tree_s3/test.py @@ -52,18 +52,8 @@ def cluster(): FILES_OVERHEAD = 1 FILES_OVERHEAD_PER_COLUMN = 2 # Data and mark files -FILES_OVERHEAD_DEFAULT_COMPRESSION_CODEC = 1 -FILES_OVERHEAD_METADATA_VERSION = 1 -FILES_OVERHEAD_PER_PART_WIDE = ( - FILES_OVERHEAD_PER_COLUMN * 3 - + 2 - + 6 - + FILES_OVERHEAD_DEFAULT_COMPRESSION_CODEC - + FILES_OVERHEAD_METADATA_VERSION -) -FILES_OVERHEAD_PER_PART_COMPACT = ( - 10 + FILES_OVERHEAD_DEFAULT_COMPRESSION_CODEC + FILES_OVERHEAD_METADATA_VERSION -) +FILES_OVERHEAD_PER_PART_WIDE = FILES_OVERHEAD_PER_COLUMN * 3 + 2 + 6 + 1 +FILES_OVERHEAD_PER_PART_COMPACT = 10 + 1 def create_table(node, table_name, **additional_settings): @@ -242,6 +232,7 @@ def test_insert_same_partition_and_merge(cluster, merge_vertical, node_name): def test_alter_table_columns(cluster, node_name): node = cluster.instances[node_name] create_table(node, "s3_test") + minio = cluster.minio_client node.query( "INSERT INTO s3_test VALUES {}".format(generate_values("2020-01-03", 4096)) diff --git a/tests/integration/test_merge_tree_s3_failover/test.py b/tests/integration/test_merge_tree_s3_failover/test.py index fef2e26bd2e..c61cacc9d8c 100644 --- a/tests/integration/test_merge_tree_s3_failover/test.py +++ b/tests/integration/test_merge_tree_s3_failover/test.py @@ -89,7 +89,7 @@ def drop_table(cluster): # S3 request will be failed for an appropriate part file write. -FILES_PER_PART_BASE = 6 # partition.dat, metadata_version.txt, default_compression_codec.txt, count.txt, columns.txt, checksums.txt +FILES_PER_PART_BASE = 5 # partition.dat, default_compression_codec.txt, count.txt, columns.txt, checksums.txt FILES_PER_PART_WIDE = ( FILES_PER_PART_BASE + 1 + 1 + 3 * 2 ) # Primary index, MinMax, Mark and data file for column(s) diff --git a/tests/integration/test_partition/test.py b/tests/integration/test_partition/test.py index b59cc21f39f..ae4393fc6f6 100644 --- a/tests/integration/test_partition/test.py +++ b/tests/integration/test_partition/test.py @@ -105,8 +105,6 @@ def partition_complex_assert_checksums(): "c4ca4238a0b923820dcc509a6f75849b\tshadow/1/data/test/partition_complex/19700102_2_2_0/count.txt\n" "c4ca4238a0b923820dcc509a6f75849b\tshadow/1/data/test/partition_complex/19700201_1_1_0/count.txt\n" "cfcb770c3ecd0990dcceb1bde129e6c6\tshadow/1/data/test/partition_complex/19700102_2_2_0/p.bin\n" - "cfcd208495d565ef66e7dff9f98764da\tshadow/1/data/test/partition_complex/19700102_2_2_0/metadata_version.txt\n" - "cfcd208495d565ef66e7dff9f98764da\tshadow/1/data/test/partition_complex/19700201_1_1_0/metadata_version.txt\n" "e2af3bef1fd129aea73a890ede1e7a30\tshadow/1/data/test/partition_complex/19700201_1_1_0/k.bin\n" "f2312862cc01adf34a93151377be2ddf\tshadow/1/data/test/partition_complex/19700201_1_1_0/minmax_p.idx\n" ) diff --git a/tests/integration/test_replicated_merge_tree_s3/test.py b/tests/integration/test_replicated_merge_tree_s3/test.py index b90e28dfdb2..0d978bb6967 100644 --- a/tests/integration/test_replicated_merge_tree_s3/test.py +++ b/tests/integration/test_replicated_merge_tree_s3/test.py @@ -44,18 +44,8 @@ def cluster(): FILES_OVERHEAD = 1 FILES_OVERHEAD_PER_COLUMN = 2 # Data and mark files -FILES_OVERHEAD_DEFAULT_COMPRESSION_CODEC = 1 -FILES_OVERHEAD_METADATA_VERSION = 1 -FILES_OVERHEAD_PER_PART_WIDE = ( - FILES_OVERHEAD_PER_COLUMN * 3 - + 2 - + 6 - + FILES_OVERHEAD_DEFAULT_COMPRESSION_CODEC - + FILES_OVERHEAD_METADATA_VERSION -) -FILES_OVERHEAD_PER_PART_COMPACT = ( - 10 + FILES_OVERHEAD_DEFAULT_COMPRESSION_CODEC + FILES_OVERHEAD_METADATA_VERSION -) +FILES_OVERHEAD_PER_PART_WIDE = FILES_OVERHEAD_PER_COLUMN * 3 + 2 + 6 + 1 +FILES_OVERHEAD_PER_PART_COMPACT = 10 + 1 def random_string(length): diff --git a/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py b/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py index 62ca73c210b..f0bc12e3125 100644 --- a/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py +++ b/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py @@ -47,18 +47,8 @@ def cluster(): FILES_OVERHEAD = 1 FILES_OVERHEAD_PER_COLUMN = 2 # Data and mark files -FILES_OVERHEAD_DEFAULT_COMPRESSION_CODEC = 1 -FILES_OVERHEAD_METADATA_VERSION = 1 -FILES_OVERHEAD_PER_PART_WIDE = ( - FILES_OVERHEAD_PER_COLUMN * 3 - + 2 - + 6 - + FILES_OVERHEAD_DEFAULT_COMPRESSION_CODEC - + FILES_OVERHEAD_METADATA_VERSION -) -FILES_OVERHEAD_PER_PART_COMPACT = ( - 10 + FILES_OVERHEAD_DEFAULT_COMPRESSION_CODEC + FILES_OVERHEAD_METADATA_VERSION -) +FILES_OVERHEAD_PER_PART_WIDE = FILES_OVERHEAD_PER_COLUMN * 3 + 2 + 6 + 1 +FILES_OVERHEAD_PER_PART_COMPACT = 10 + 1 def random_string(length): diff --git a/tests/integration/test_s3_zero_copy_ttl/test.py b/tests/integration/test_s3_zero_copy_ttl/test.py index 7dcf3734653..9a782aacef6 100644 --- a/tests/integration/test_s3_zero_copy_ttl/test.py +++ b/tests/integration/test_s3_zero_copy_ttl/test.py @@ -86,9 +86,9 @@ def test_ttl_move_and_s3(started_cluster): print(f"Total objects: {counter}") - if counter == 330: + if counter == 300: break print(f"Attempts remaining: {attempt}") - assert counter == 330 + assert counter == 300 diff --git a/tests/queries/0_stateless/01278_alter_rename_combination.reference b/tests/queries/0_stateless/01278_alter_rename_combination.reference index e70c2d2e6f8..cc912e9b265 100644 --- a/tests/queries/0_stateless/01278_alter_rename_combination.reference +++ b/tests/queries/0_stateless/01278_alter_rename_combination.reference @@ -1,7 +1,7 @@ -CREATE TABLE default.rename_table\n(\n `key` Int32,\n `old_value1` Int32,\n `value1` Int32\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS min_bytes_for_wide_part = 0, index_granularity = 8192 +CREATE TABLE default.rename_table\n(\n `key` Int32,\n `old_value1` Int32,\n `value1` Int32\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 key old_value1 value1 1 2 3 -CREATE TABLE default.rename_table\n(\n `k` Int32,\n `v1` Int32,\n `v2` Int32\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS min_bytes_for_wide_part = 0, index_granularity = 8192 +CREATE TABLE default.rename_table\n(\n `k` Int32,\n `v1` Int32,\n `v2` Int32\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 k v1 v2 1 2 3 4 5 6 diff --git a/tests/queries/0_stateless/01278_alter_rename_combination.sql b/tests/queries/0_stateless/01278_alter_rename_combination.sql index 51322f5d86f..fa73362622c 100644 --- a/tests/queries/0_stateless/01278_alter_rename_combination.sql +++ b/tests/queries/0_stateless/01278_alter_rename_combination.sql @@ -1,6 +1,6 @@ DROP TABLE IF EXISTS rename_table; -CREATE TABLE rename_table (key Int32, value1 Int32, value2 Int32) ENGINE = MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part=0; +CREATE TABLE rename_table (key Int32, value1 Int32, value2 Int32) ENGINE = MergeTree ORDER BY tuple(); INSERT INTO rename_table VALUES (1, 2, 3); diff --git a/tests/queries/0_stateless/01281_alter_rename_and_other_renames.reference b/tests/queries/0_stateless/01281_alter_rename_and_other_renames.reference index 532b8ce8712..bf3358aea60 100644 --- a/tests/queries/0_stateless/01281_alter_rename_and_other_renames.reference +++ b/tests/queries/0_stateless/01281_alter_rename_and_other_renames.reference @@ -1,11 +1,11 @@ -CREATE TABLE default.rename_table_multiple\n(\n `key` Int32,\n `value1_string` String,\n `value2` Int32\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS min_bytes_for_wide_part = 0, index_granularity = 8192 +CREATE TABLE default.rename_table_multiple\n(\n `key` Int32,\n `value1_string` String,\n `value2` Int32\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 key value1_string value2 1 2 3 -CREATE TABLE default.rename_table_multiple\n(\n `key` Int32,\n `value1_string` String,\n `value2_old` Int32,\n `value2` Int64 DEFAULT 7\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS min_bytes_for_wide_part = 0, index_granularity = 8192 +CREATE TABLE default.rename_table_multiple\n(\n `key` Int32,\n `value1_string` String,\n `value2_old` Int32,\n `value2` Int64 DEFAULT 7\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 key value1_string value2_old value2 1 2 3 7 4 5 6 7 -CREATE TABLE default.rename_table_multiple\n(\n `key` Int32,\n `value1_string` String,\n `value2_old` Int64 DEFAULT 7\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS min_bytes_for_wide_part = 0, index_granularity = 8192 +CREATE TABLE default.rename_table_multiple\n(\n `key` Int32,\n `value1_string` String,\n `value2_old` Int64 DEFAULT 7\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 key value1_string value2_old 1 2 7 4 5 7 diff --git a/tests/queries/0_stateless/01281_alter_rename_and_other_renames.sql b/tests/queries/0_stateless/01281_alter_rename_and_other_renames.sql index b0ccd7751ab..f9462f0478e 100644 --- a/tests/queries/0_stateless/01281_alter_rename_and_other_renames.sql +++ b/tests/queries/0_stateless/01281_alter_rename_and_other_renames.sql @@ -1,6 +1,6 @@ DROP TABLE IF EXISTS rename_table_multiple; -CREATE TABLE rename_table_multiple (key Int32, value1 String, value2 Int32) ENGINE = MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part=0; +CREATE TABLE rename_table_multiple (key Int32, value1 String, value2 Int32) ENGINE = MergeTree ORDER BY tuple(); INSERT INTO rename_table_multiple VALUES (1, 2, 3); diff --git a/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.reference b/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.reference index f3fac9b32d3..bbca9bbbfee 100644 --- a/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.reference +++ b/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.reference @@ -7,25 +7,25 @@ file_segment_range_begin: 0 file_segment_range_end: 745 size: 746 state: DOWNLOADED -8 -8 +7 +7 0 2 2 -8 +7 Row 1: ────── file_segment_range_begin: 0 file_segment_range_end: 1659 size: 1660 state: DOWNLOADED -8 -8 -8 -8 -24 -35 -43 +7 +7 +7 +7 +21 +31 +38 5010500 18816 Using storage policy: local_cache @@ -37,24 +37,24 @@ file_segment_range_begin: 0 file_segment_range_end: 745 size: 746 state: DOWNLOADED -8 -8 +7 +7 0 2 2 -8 +7 Row 1: ────── file_segment_range_begin: 0 file_segment_range_end: 1659 size: 1660 state: DOWNLOADED -8 -8 -8 -8 -24 -35 -43 +7 +7 +7 +7 +21 +31 +38 5010500 18816 diff --git a/tests/queries/0_stateless/02361_fsync_profile_events.sh b/tests/queries/0_stateless/02361_fsync_profile_events.sh index 5b603133f6c..44a1bd58d36 100755 --- a/tests/queries/0_stateless/02361_fsync_profile_events.sh +++ b/tests/queries/0_stateless/02361_fsync_profile_events.sh @@ -44,8 +44,8 @@ for i in {1..100}; do ")" # Non retriable errors - if [[ $FileSync -ne 8 ]]; then - echo "FileSync: $FileSync != 8" >&2 + if [[ $FileSync -ne 7 ]]; then + echo "FileSync: $FileSync != 11" >&2 exit 2 fi # Check that all files was synced diff --git a/tests/queries/0_stateless/02538_alter_rename_sequence.reference b/tests/queries/0_stateless/02538_alter_rename_sequence.reference deleted file mode 100644 index 73aa1b7e8d8..00000000000 --- a/tests/queries/0_stateless/02538_alter_rename_sequence.reference +++ /dev/null @@ -1,8 +0,0 @@ -1 2 3 -4 5 6 -{"column1_renamed":"1","column2_renamed":"2","column3":"3"} -{"column1_renamed":"4","column2_renamed":"5","column3":"6"} -1 2 3 -4 5 6 -{"column1_renamed":"1","column2_renamed":"2","column3":"3"} -{"column1_renamed":"4","column2_renamed":"5","column3":"6"} diff --git a/tests/queries/0_stateless/02538_alter_rename_sequence.sql b/tests/queries/0_stateless/02538_alter_rename_sequence.sql deleted file mode 100644 index d7df27dc702..00000000000 --- a/tests/queries/0_stateless/02538_alter_rename_sequence.sql +++ /dev/null @@ -1,59 +0,0 @@ -DROP TABLE IF EXISTS wrong_metadata; - -CREATE TABLE wrong_metadata( - column1 UInt64, - column2 UInt64, - column3 UInt64 -) -ENGINE ReplicatedMergeTree('/test/{database}/tables/wrong_metadata', '1') -ORDER BY tuple(); - -INSERT INTO wrong_metadata VALUES (1, 2, 3); - -SYSTEM STOP REPLICATION QUEUES wrong_metadata; - -ALTER TABLE wrong_metadata RENAME COLUMN column1 TO column1_renamed SETTINGS replication_alter_partitions_sync = 0; - -INSERT INTO wrong_metadata VALUES (4, 5, 6); - -SELECT * FROM wrong_metadata ORDER BY column1; - -SYSTEM START REPLICATION QUEUES wrong_metadata; - -SYSTEM SYNC REPLICA wrong_metadata; - -ALTER TABLE wrong_metadata RENAME COLUMN column2 to column2_renamed SETTINGS replication_alter_partitions_sync = 2; - -SELECT * FROM wrong_metadata ORDER BY column1_renamed FORMAT JSONEachRow; - -DROP TABLE IF EXISTS wrong_metadata; - - -CREATE TABLE wrong_metadata_wide( - column1 UInt64, - column2 UInt64, - column3 UInt64 -) -ENGINE ReplicatedMergeTree('/test/{database}/tables/wrong_metadata_wide', '1') -ORDER BY tuple() -SETTINGS min_bytes_for_wide_part = 0; - -INSERT INTO wrong_metadata_wide VALUES (1, 2, 3); - -SYSTEM STOP REPLICATION QUEUES wrong_metadata_wide; - -ALTER TABLE wrong_metadata_wide RENAME COLUMN column1 TO column1_renamed SETTINGS replication_alter_partitions_sync = 0; - -INSERT INTO wrong_metadata_wide VALUES (4, 5, 6); - -SELECT * FROM wrong_metadata_wide ORDER by column1; - -SYSTEM START REPLICATION QUEUES wrong_metadata_wide; - -SYSTEM SYNC REPLICA wrong_metadata_wide; - -ALTER TABLE wrong_metadata_wide RENAME COLUMN column2 to column2_renamed SETTINGS replication_alter_partitions_sync = 2; - -SELECT * FROM wrong_metadata_wide ORDER BY column1_renamed FORMAT JSONEachRow; - -DROP TABLE IF EXISTS wrong_metadata_wide; diff --git a/tests/queries/0_stateless/02543_alter_rename_modify_stuck.reference b/tests/queries/0_stateless/02543_alter_rename_modify_stuck.reference deleted file mode 100644 index 156128e3dd2..00000000000 --- a/tests/queries/0_stateless/02543_alter_rename_modify_stuck.reference +++ /dev/null @@ -1 +0,0 @@ -{"v":"1","v2":"77"} diff --git a/tests/queries/0_stateless/02543_alter_rename_modify_stuck.sh b/tests/queries/0_stateless/02543_alter_rename_modify_stuck.sh deleted file mode 100755 index adaf1846552..00000000000 --- a/tests/queries/0_stateless/02543_alter_rename_modify_stuck.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env bash - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# shellcheck source=../shell_config.sh -. "$CURDIR"/../shell_config.sh - - -$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS table_to_rename" - -$CLICKHOUSE_CLIENT --query="CREATE TABLE table_to_rename(v UInt64, v1 UInt64)ENGINE = MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0" - -$CLICKHOUSE_CLIENT --query="INSERT INTO table_to_rename VALUES (1, 1)" - - -# we want to following mutations to stuck -# That is why we stop merges and wait in loops until they actually start -$CLICKHOUSE_CLIENT --query="SYSTEM STOP MERGES table_to_rename" - -$CLICKHOUSE_CLIENT --query="ALTER TABLE table_to_rename RENAME COLUMN v1 to v2" & - -counter=0 retries=60 - -I=0 -while [[ $counter -lt $retries ]]; do - I=$((I + 1)) - result=$($CLICKHOUSE_CLIENT --query "show create table table_to_rename") - if [[ $result == *"v2"* ]]; then - break; - fi - sleep 0.1 - ((++counter)) -done - - -$CLICKHOUSE_CLIENT --query="ALTER TABLE table_to_rename UPDATE v2 = 77 WHERE 1 = 1 SETTINGS mutations_sync = 2" & - -counter=0 retries=60 - -I=0 -while [[ $counter -lt $retries ]]; do - I=$((I + 1)) - result=$($CLICKHOUSE_CLIENT --query "SELECT count() from system.mutations where database='${CLICKHOUSE_DATABASE}' and table='table_to_rename'") - if [[ $result == "2" ]]; then - break; - fi - sleep 0.1 - ((++counter)) -done - - -$CLICKHOUSE_CLIENT --query="SYSTEM START MERGES table_to_rename" - -wait - -$CLICKHOUSE_CLIENT --query="SELECT * FROM table_to_rename FORMAT JSONEachRow" - - - $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS table_to_rename" diff --git a/tests/queries/0_stateless/02555_davengers_rename_chain.reference b/tests/queries/0_stateless/02555_davengers_rename_chain.reference deleted file mode 100644 index a9fc4b395e2..00000000000 --- a/tests/queries/0_stateless/02555_davengers_rename_chain.reference +++ /dev/null @@ -1,26 +0,0 @@ -{"a1":"1","b1":"2","c":"3"} -~~~~~~~ -{"a1":"1","b1":"2","c":"3"} -{"a1":"4","b1":"5","c":"6"} -~~~~~~~ -{"a1":"1","b1":"2","c":"3"} -{"a1":"4","b1":"5","c":"6"} -{"a1":"7","b1":"8","c":"9"} -~~~~~~~ -{"b":"1","a":"2","c":"3"} -{"b":"4","a":"5","c":"6"} -{"b":"7","a":"8","c":"9"} -~~~~~~~ -{"a1":"1","b1":"2","c":"3"} -~~~~~~~ -{"a1":"1","b1":"2","c":"3"} -{"a1":"4","b1":"5","c":"6"} -~~~~~~~ -{"a1":"1","b1":"2","c":"3"} -{"a1":"4","b1":"5","c":"6"} -{"a1":"7","b1":"8","c":"9"} -~~~~~~~ -{"b":"1","a":"2","c":"3"} -{"b":"4","a":"5","c":"6"} -{"b":"7","a":"8","c":"9"} -~~~~~~~ diff --git a/tests/queries/0_stateless/02555_davengers_rename_chain.sh b/tests/queries/0_stateless/02555_davengers_rename_chain.sh deleted file mode 100755 index b23f8085fd7..00000000000 --- a/tests/queries/0_stateless/02555_davengers_rename_chain.sh +++ /dev/null @@ -1,143 +0,0 @@ -#!/usr/bin/env bash -# Tags: replica -CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# shellcheck source=../shell_config.sh -. "$CUR_DIR"/../shell_config.sh - -$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS wrong_metadata" - -$CLICKHOUSE_CLIENT -n --query="CREATE TABLE wrong_metadata( - a UInt64, - b UInt64, - c UInt64 -) -ENGINE ReplicatedMergeTree('/test/{database}/tables/wrong_metadata', '1') -ORDER BY tuple() -SETTINGS min_bytes_for_wide_part = 0" - -$CLICKHOUSE_CLIENT --query="INSERT INTO wrong_metadata VALUES (1, 2, 3)" - - -$CLICKHOUSE_CLIENT --query="SYSTEM STOP MERGES wrong_metadata" - - -$CLICKHOUSE_CLIENT --query="ALTER TABLE wrong_metadata RENAME COLUMN a TO a1, RENAME COLUMN b to b1 SETTINGS replication_alter_partitions_sync = 0" - -counter=0 retries=60 -I=0 -while [[ $counter -lt $retries ]]; do - I=$((I + 1)) - result=$($CLICKHOUSE_CLIENT --query "SHOW CREATE TABLE wrong_metadata") - if [[ $result == *"\`a1\` UInt64"* ]]; then - break; - fi - sleep 0.1 - ((++counter)) -done - - -$CLICKHOUSE_CLIENT --query="SELECT * FROM wrong_metadata ORDER BY a1 FORMAT JSONEachRow" - -$CLICKHOUSE_CLIENT --query="SELECT '~~~~~~~'" - -$CLICKHOUSE_CLIENT --query="INSERT INTO wrong_metadata VALUES (4, 5, 6)" - - -$CLICKHOUSE_CLIENT --query="SELECT * FROM wrong_metadata ORDER BY a1 FORMAT JSONEachRow" -$CLICKHOUSE_CLIENT --query="SELECT '~~~~~~~'" - - -$CLICKHOUSE_CLIENT --query="ALTER TABLE wrong_metadata RENAME COLUMN a1 TO b, RENAME COLUMN b1 to a SETTINGS replication_alter_partitions_sync = 0" - -counter=0 retries=60 -I=0 -while [[ $counter -lt $retries ]]; do - I=$((I + 1)) - result=$($CLICKHOUSE_CLIENT --query "SELECT * FROM system.mutations WHERE table = 'wrong_metadata' AND database='${CLICKHOUSE_DATABASE}'") - if [[ $result == *"b1 TO a"* ]]; then - break; - fi - sleep 0.1 - ((++counter)) -done - -$CLICKHOUSE_CLIENT --query="INSERT INTO wrong_metadata VALUES (7, 8, 9)" - -$CLICKHOUSE_CLIENT --query="SELECT * FROM wrong_metadata ORDER by a1 FORMAT JSONEachRow" -$CLICKHOUSE_CLIENT --query="SELECT '~~~~~~~'" - -$CLICKHOUSE_CLIENT --query="SYSTEM START MERGES wrong_metadata" - -$CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA wrong_metadata" - -$CLICKHOUSE_CLIENT --query="SELECT * FROM wrong_metadata order by a FORMAT JSONEachRow" - -$CLICKHOUSE_CLIENT --query="SELECT '~~~~~~~'" - - -$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS wrong_metadata" - -$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS wrong_metadata_compact" - -$CLICKHOUSE_CLIENT -n --query="CREATE TABLE wrong_metadata_compact( - a UInt64, - b UInt64, - c UInt64 -) -ENGINE ReplicatedMergeTree('/test/{database}/tables/wrong_metadata_compact', '1') -ORDER BY tuple() -SETTINGS min_bytes_for_wide_part = 10000000" - -$CLICKHOUSE_CLIENT --query="INSERT INTO wrong_metadata_compact VALUES (1, 2, 3)" - -$CLICKHOUSE_CLIENT --query="SYSTEM STOP MERGES wrong_metadata_compact" - -$CLICKHOUSE_CLIENT --query="ALTER TABLE wrong_metadata_compact RENAME COLUMN a TO a1, RENAME COLUMN b to b1 SETTINGS replication_alter_partitions_sync = 0" - -counter=0 retries=60 -I=0 -while [[ $counter -lt $retries ]]; do - I=$((I + 1)) - result=$($CLICKHOUSE_CLIENT --query "SHOW CREATE TABLE wrong_metadata_compact") - if [[ $result == *"\`a1\` UInt64"* ]]; then - break; - fi - sleep 0.1 - ((++counter)) -done - -$CLICKHOUSE_CLIENT --query="SELECT * FROM wrong_metadata_compact ORDER BY a1 FORMAT JSONEachRow" -$CLICKHOUSE_CLIENT --query="SELECT '~~~~~~~'" - -$CLICKHOUSE_CLIENT --query="INSERT INTO wrong_metadata_compact VALUES (4, 5, 6)" - -$CLICKHOUSE_CLIENT --query="SELECT * FROM wrong_metadata_compact ORDER BY a1 FORMAT JSONEachRow" -$CLICKHOUSE_CLIENT --query="SELECT '~~~~~~~'" - -$CLICKHOUSE_CLIENT --query="ALTER TABLE wrong_metadata_compact RENAME COLUMN a1 TO b, RENAME COLUMN b1 to a SETTINGS replication_alter_partitions_sync = 0" - -counter=0 retries=60 -I=0 -while [[ $counter -lt $retries ]]; do - I=$((I + 1)) - result=$($CLICKHOUSE_CLIENT --query "SELECT * FROM system.mutations WHERE table = 'wrong_metadata_compact' AND database='${CLICKHOUSE_DATABASE}'") - if [[ $result == *"b1 TO a"* ]]; then - break; - fi - sleep 0.1 - ((++counter)) -done - -$CLICKHOUSE_CLIENT --query="INSERT INTO wrong_metadata_compact VALUES (7, 8, 9)" - -$CLICKHOUSE_CLIENT --query="SELECT * FROM wrong_metadata_compact ORDER by a1 FORMAT JSONEachRow" -$CLICKHOUSE_CLIENT --query="SELECT '~~~~~~~'" - -$CLICKHOUSE_CLIENT --query="SYSTEM START MERGES wrong_metadata_compact" - -$CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA wrong_metadata_compact" - -$CLICKHOUSE_CLIENT --query="SELECT * FROM wrong_metadata_compact order by a FORMAT JSONEachRow" -$CLICKHOUSE_CLIENT --query="SELECT '~~~~~~~'" - -$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS wrong_metadata_compact" From cb4d665c025e558c9880bd5e3dbdf9aeba1a660c Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 26 Feb 2023 07:35:38 +0100 Subject: [PATCH 693/717] tests: fix 02116_interactive_hello for "official build" Example of such greeting: 2023-02-26 15:26:33 expect: does "\u001b[0J\u001b[?25hClickHouse client version 23.3.1.207 (official build).\r\nConnecting to database test_i4r0kozg at localhost:9000 as user default.\r\nConnected to ClickHouse server version 23.3.1 revision 54461.\r\n\r\nWarnings:\r\n * Server logging level is set to 'test' and performance is degraded. This cannot be used in production.\r\n * Linux is not using a fast clock source. Performance can be degraded. Check /sys/devices/system/clocksource/clocksource0/current_clocksource\r\n * The setting 'allow_remote_fs_zero_copy_replication' is enabled for MergeTree tables. But the feature of 'zero-copy replication' is under development and is not ready for production. The usage of this feature can lead to data corruption and loss. The setting should be disabled in production.\r\n * Table system.session_log is enabled. It's unreliable and may contain garbage. Do not use it for any kind of security monitoring.\r\n\r\n\u001b[?2004h\u001b[1Gfunctional-tests :) " (spawn_id exp4) match regular expression "ClickHouse client version [\d]{2}.[\d]{1,2}.[\d]{1,2}.[\d]{1,}.\r"? Gate "ClickHouse client version *\r"? gate=yes re=no CI: https://s3.amazonaws.com/clickhouse-test-reports/0/df1e18ad4cb9b08240273169ca7dd6ca1cac617c/stateless_tests__aarch64_/run.log Signed-off-by: Azat Khuzhin --- tests/queries/0_stateless/02116_interactive_hello.expect | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/02116_interactive_hello.expect b/tests/queries/0_stateless/02116_interactive_hello.expect index d05d9adc826..5d30e646c4c 100755 --- a/tests/queries/0_stateless/02116_interactive_hello.expect +++ b/tests/queries/0_stateless/02116_interactive_hello.expect @@ -19,9 +19,10 @@ expect_after { spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --history_file=$history_file" -expect -re "ClickHouse client version \[\\d\]{2}\.\[\\d\]{1,2}\.\[\\d\]{1,2}\.\[\\d\]{1,}\.\r" -expect -re "Connecting to database .* at localhost:9000 as user default\.\r" -expect -re "Connected to ClickHouse server version \[\\d\]{2}\.\[\\d\]{1,2}\.\[\\d\]{1,2} revision \[\\d\]{1,}\.\r" +# (?n) - Do not match new lines +expect -re "(?n)ClickHouse client version \[\\d\]{2}\.\[\\d\]{1,2}\.\[\\d\]{1,2}\.\[\\d\]{1,}.*\r" +expect -re "(?n)Connecting to database .* at localhost:9000 as user default\.\r" +expect -re "(?n)Connected to ClickHouse server version \[\\d\]{2}\.\[\\d\]{1,2}\.\[\\d\]{1,2} revision \[\\d\]{1,}\.\r" expect ":) " send -- "" From 05b50a1964681ba3860ea30957edd02113b41f9c Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 26 Feb 2023 08:05:01 +0100 Subject: [PATCH 694/717] Remove long/no-fasttest for some expect tests Those tests was simply broken and timed out without failing the test before this PRs: - #46911 - #46857 - #46779, #46636, #46619 So after those fixes they should be fast and fasttest compatible. Signed-off-by: Azat Khuzhin --- tests/queries/0_stateless/01179_insert_values_semicolon.expect | 1 - .../01300_client_save_history_when_terminated_long.expect | 1 - .../0_stateless/01565_query_loop_after_client_error.expect | 1 - .../01910_client_replxx_container_overflow_long.expect | 1 - tests/queries/0_stateless/02003_memory_limit_in_client.expect | 1 - .../02112_delayed_clickhouse_client_with_queries_file.expect | 1 - .../02112_delayed_clickhouse_local_with_queries_file.expect | 1 - tests/queries/0_stateless/02116_interactive_hello.expect | 1 - ...02310_clickhouse_client_INSERT_progress_profile_events.expect | 1 - .../02310_clickhouse_local_INSERT_progress_profile_events.expect | 1 - 10 files changed, 10 deletions(-) diff --git a/tests/queries/0_stateless/01179_insert_values_semicolon.expect b/tests/queries/0_stateless/01179_insert_values_semicolon.expect index 4784fc38a84..cb22fb8265c 100755 --- a/tests/queries/0_stateless/01179_insert_values_semicolon.expect +++ b/tests/queries/0_stateless/01179_insert_values_semicolon.expect @@ -1,5 +1,4 @@ #!/usr/bin/expect -f -# Tags: long set basedir [file dirname $argv0] set basename [file tail $argv0] diff --git a/tests/queries/0_stateless/01300_client_save_history_when_terminated_long.expect b/tests/queries/0_stateless/01300_client_save_history_when_terminated_long.expect index 4091d9fec91..a593075bb9a 100755 --- a/tests/queries/0_stateless/01300_client_save_history_when_terminated_long.expect +++ b/tests/queries/0_stateless/01300_client_save_history_when_terminated_long.expect @@ -1,5 +1,4 @@ #!/usr/bin/expect -f -# Tags: long set basedir [file dirname $argv0] set basename [file tail $argv0] diff --git a/tests/queries/0_stateless/01565_query_loop_after_client_error.expect b/tests/queries/0_stateless/01565_query_loop_after_client_error.expect index bf701225605..0faf8f0192b 100755 --- a/tests/queries/0_stateless/01565_query_loop_after_client_error.expect +++ b/tests/queries/0_stateless/01565_query_loop_after_client_error.expect @@ -1,5 +1,4 @@ #!/usr/bin/expect -f -# Tags: long # This is a separate test, because we want to test the interactive mode. # https://github.com/ClickHouse/ClickHouse/issues/19353 diff --git a/tests/queries/0_stateless/01910_client_replxx_container_overflow_long.expect b/tests/queries/0_stateless/01910_client_replxx_container_overflow_long.expect index e5b6bc65e9b..9105cf30f79 100755 --- a/tests/queries/0_stateless/01910_client_replxx_container_overflow_long.expect +++ b/tests/queries/0_stateless/01910_client_replxx_container_overflow_long.expect @@ -1,5 +1,4 @@ #!/usr/bin/expect -f -# Tags: long set basedir [file dirname $argv0] set basename [file tail $argv0] diff --git a/tests/queries/0_stateless/02003_memory_limit_in_client.expect b/tests/queries/0_stateless/02003_memory_limit_in_client.expect index b74238c7477..377656fa641 100755 --- a/tests/queries/0_stateless/02003_memory_limit_in_client.expect +++ b/tests/queries/0_stateless/02003_memory_limit_in_client.expect @@ -1,5 +1,4 @@ #!/usr/bin/expect -f -# Tags: no-fasttest # This is a test for system.warnings. Testing in interactive mode is necessary, # as we want to see certain warnings from client diff --git a/tests/queries/0_stateless/02112_delayed_clickhouse_client_with_queries_file.expect b/tests/queries/0_stateless/02112_delayed_clickhouse_client_with_queries_file.expect index 5318c7eac32..340aec5bed2 100755 --- a/tests/queries/0_stateless/02112_delayed_clickhouse_client_with_queries_file.expect +++ b/tests/queries/0_stateless/02112_delayed_clickhouse_client_with_queries_file.expect @@ -1,5 +1,4 @@ #!/usr/bin/expect -f -# Tags: no-parallel set basedir [file dirname $argv0] set basename [file tail $argv0] diff --git a/tests/queries/0_stateless/02112_delayed_clickhouse_local_with_queries_file.expect b/tests/queries/0_stateless/02112_delayed_clickhouse_local_with_queries_file.expect index 404daeda88e..0b63b38849a 100755 --- a/tests/queries/0_stateless/02112_delayed_clickhouse_local_with_queries_file.expect +++ b/tests/queries/0_stateless/02112_delayed_clickhouse_local_with_queries_file.expect @@ -1,5 +1,4 @@ #!/usr/bin/expect -f -# Tags: no-parallel set basedir [file dirname $argv0] set basename [file tail $argv0] diff --git a/tests/queries/0_stateless/02116_interactive_hello.expect b/tests/queries/0_stateless/02116_interactive_hello.expect index 5d30e646c4c..3f2140861ea 100755 --- a/tests/queries/0_stateless/02116_interactive_hello.expect +++ b/tests/queries/0_stateless/02116_interactive_hello.expect @@ -1,5 +1,4 @@ #!/usr/bin/expect -f -# Tags: long set basedir [file dirname $argv0] set basename [file tail $argv0] diff --git a/tests/queries/0_stateless/02310_clickhouse_client_INSERT_progress_profile_events.expect b/tests/queries/0_stateless/02310_clickhouse_client_INSERT_progress_profile_events.expect index 6e545cf43cd..9f6937cf80d 100755 --- a/tests/queries/0_stateless/02310_clickhouse_client_INSERT_progress_profile_events.expect +++ b/tests/queries/0_stateless/02310_clickhouse_client_INSERT_progress_profile_events.expect @@ -1,5 +1,4 @@ #!/usr/bin/expect -f -# Tags: long # This is the regression for the concurrent access in ProgressIndication, # so it is important to read enough rows here (10e6). diff --git a/tests/queries/0_stateless/02310_clickhouse_local_INSERT_progress_profile_events.expect b/tests/queries/0_stateless/02310_clickhouse_local_INSERT_progress_profile_events.expect index f144735c320..5e514cd7c12 100755 --- a/tests/queries/0_stateless/02310_clickhouse_local_INSERT_progress_profile_events.expect +++ b/tests/queries/0_stateless/02310_clickhouse_local_INSERT_progress_profile_events.expect @@ -1,5 +1,4 @@ #!/usr/bin/expect -f -# Tags: long # This is the regression for the concurrent access in ProgressIndication, # so it is important to read enough rows here (10e6). From 584f9d5e8ff3b0aefc080486607c7234800cbef2 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 26 Feb 2023 12:02:27 +0100 Subject: [PATCH 695/717] Fix 02160_client_autocomplete_parse_query Signed-off-by: Azat Khuzhin --- .../0_stateless/02160_client_autocomplete_parse_query.expect | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02160_client_autocomplete_parse_query.expect b/tests/queries/0_stateless/02160_client_autocomplete_parse_query.expect index 68544a4f014..01d61a6ad0f 100755 --- a/tests/queries/0_stateless/02160_client_autocomplete_parse_query.expect +++ b/tests/queries/0_stateless/02160_client_autocomplete_parse_query.expect @@ -44,9 +44,8 @@ expect ":) " # Generate UIUD to avoid matching old database/tables/columns from previous test runs. send -- "select 'begin-' || replace(toString(generateUUIDv4()), '-', '') || '-end' format TSV\r" -expect -re TSV.*TSV.*begin-(.*)-end.* +expect -re "TSV.*TSV.*begin-(.*)-end.*:\\) " set uuid $expect_out(1,string) -expect ":) " # CREATE DATABASE send -- "create database new_${uuid}_database\r" From c8c8d4b42f277f223ae2a866389b35a9bc58e876 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 26 Feb 2023 13:33:43 +0100 Subject: [PATCH 696/717] Always bind Ctrl-T to reverse search (not only when skim is available) This will avoid exceptions for fast tests Signed-off-by: Azat Khuzhin --- src/Client/ReplxxLineReader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Client/ReplxxLineReader.cpp b/src/Client/ReplxxLineReader.cpp index 405cb7af03b..fa5ba8b8ba9 100644 --- a/src/Client/ReplxxLineReader.cpp +++ b/src/Client/ReplxxLineReader.cpp @@ -432,6 +432,7 @@ ReplxxLineReader::ReplxxLineReader( }; rx.bind_key(Replxx::KEY::control('R'), interactive_history_search); +#endif /// Rebind regular incremental search to C-T. /// @@ -443,7 +444,6 @@ ReplxxLineReader::ReplxxLineReader( uint32_t reverse_search = Replxx::KEY::control('R'); return rx.invoke(Replxx::ACTION::HISTORY_INCREMENTAL_SEARCH, reverse_search); }); -#endif } ReplxxLineReader::~ReplxxLineReader() From b20ac16d08085708a8b132e170bef2790fa53c11 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sun, 26 Feb 2023 15:17:28 +0000 Subject: [PATCH 697/717] Slightly easier to read SyntaxExceptions E.g. exception text for input '2 ': - before: Syntax error: Not a valid integer: 2 - now: Syntax error: Not a valid integer: '2 ' --- base/poco/Foundation/src/NumberParser.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/base/poco/Foundation/src/NumberParser.cpp b/base/poco/Foundation/src/NumberParser.cpp index 4081f3b2663..8d32e1a722c 100644 --- a/base/poco/Foundation/src/NumberParser.cpp +++ b/base/poco/Foundation/src/NumberParser.cpp @@ -44,7 +44,7 @@ int NumberParser::parse(const std::string& s, char thSep) if (tryParse(s, result, thSep)) return result; else - throw SyntaxException("Not a valid integer", s); + throw SyntaxException("Not a valid integer", "'" + s + "'"); } @@ -60,7 +60,7 @@ unsigned NumberParser::parseUnsigned(const std::string& s, char thSep) if (tryParseUnsigned(s, result, thSep)) return result; else - throw SyntaxException("Not a valid unsigned integer", s); + throw SyntaxException("Not a valid unsigned integer", "'" + s + "'"); } @@ -76,7 +76,7 @@ unsigned NumberParser::parseHex(const std::string& s) if (tryParseHex(s, result)) return result; else - throw SyntaxException("Not a valid hexadecimal integer", s); + throw SyntaxException("Not a valid hexadecimal integer", "'" + s + "'"); } @@ -94,7 +94,7 @@ unsigned NumberParser::parseOct(const std::string& s) if (tryParseOct(s, result)) return result; else - throw SyntaxException("Not a valid hexadecimal integer", s); + throw SyntaxException("Not a valid hexadecimal integer", "'" + s + "'"); } @@ -112,7 +112,7 @@ Int64 NumberParser::parse64(const std::string& s, char thSep) if (tryParse64(s, result, thSep)) return result; else - throw SyntaxException("Not a valid integer", s); + throw SyntaxException("Not a valid integer", "'" + s + "'"); } @@ -128,7 +128,7 @@ UInt64 NumberParser::parseUnsigned64(const std::string& s, char thSep) if (tryParseUnsigned64(s, result, thSep)) return result; else - throw SyntaxException("Not a valid unsigned integer", s); + throw SyntaxException("Not a valid unsigned integer", "'" + s + "'"); } @@ -144,7 +144,7 @@ UInt64 NumberParser::parseHex64(const std::string& s) if (tryParseHex64(s, result)) return result; else - throw SyntaxException("Not a valid hexadecimal integer", s); + throw SyntaxException("Not a valid hexadecimal integer", "'" + s + "'"); } @@ -162,7 +162,7 @@ UInt64 NumberParser::parseOct64(const std::string& s) if (tryParseOct64(s, result)) return result; else - throw SyntaxException("Not a valid hexadecimal integer", s); + throw SyntaxException("Not a valid hexadecimal integer", "'" + s + "'"); } @@ -180,7 +180,7 @@ double NumberParser::parseFloat(const std::string& s, char decSep, char thSep) if (tryParseFloat(s, result, decSep, thSep)) return result; else - throw SyntaxException("Not a valid floating-point number", s); + throw SyntaxException("Not a valid floating-point number", "'" + s + "'"); } @@ -196,7 +196,7 @@ bool NumberParser::parseBool(const std::string& s) if (tryParseBool(s, result)) return result; else - throw SyntaxException("Not a valid bool number", s); + throw SyntaxException("Not a valid bool number", "'" + s + "'"); } From e83efb590b1615926234128cb8e5c316738fa8ca Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 26 Feb 2023 16:46:21 +0100 Subject: [PATCH 698/717] Fix few more expect tests - remove helpers to simplify them - use CLICKHOUSE_TMP for uniq name [1] [1]: https://s3.amazonaws.com/clickhouse-test-reports/46915/c8c8d4b42f277f223ae2a866389b35a9bc58e876/stateless_tests__aarch64_/run.log - use more unique string to make this test really test something Signed-off-by: Azat Khuzhin --- ..._delayed_clickhouse_client_with_queries_file.expect | 10 ++++------ ...2_delayed_clickhouse_local_with_queries_file.expect | 10 ++++------ tests/queries/0_stateless/helpers/02112_clean.sh | 5 ----- tests/queries/0_stateless/helpers/02112_prepare.sh | 5 ----- 4 files changed, 8 insertions(+), 22 deletions(-) delete mode 100755 tests/queries/0_stateless/helpers/02112_clean.sh delete mode 100755 tests/queries/0_stateless/helpers/02112_prepare.sh diff --git a/tests/queries/0_stateless/02112_delayed_clickhouse_client_with_queries_file.expect b/tests/queries/0_stateless/02112_delayed_clickhouse_client_with_queries_file.expect index 340aec5bed2..5f882ae9824 100755 --- a/tests/queries/0_stateless/02112_delayed_clickhouse_client_with_queries_file.expect +++ b/tests/queries/0_stateless/02112_delayed_clickhouse_client_with_queries_file.expect @@ -15,15 +15,13 @@ expect_after { -i $any_spawn_id timeout { exit 1 } } -system "$basedir/helpers/02112_prepare.sh" -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT --disable_suggestion --interactive --queries-file $basedir/file_02112" +system "echo \"drop table if exists t; create table t(i String) engine=Memory; insert into t select 'test string'\" > $env(CLICKHOUSE_TMP)/file_02112" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT --disable_suggestion --interactive --queries-file $env(CLICKHOUSE_TMP)/file_02112" expect ":) " -send -- "select * from t format TSV\r" -expect "1" +send -- "select i from t format TSV\r" +expect "test string" expect ":) " send "" expect eof - -system "$basedir/helpers/02112_clean.sh" diff --git a/tests/queries/0_stateless/02112_delayed_clickhouse_local_with_queries_file.expect b/tests/queries/0_stateless/02112_delayed_clickhouse_local_with_queries_file.expect index 0b63b38849a..0a136ff2e74 100755 --- a/tests/queries/0_stateless/02112_delayed_clickhouse_local_with_queries_file.expect +++ b/tests/queries/0_stateless/02112_delayed_clickhouse_local_with_queries_file.expect @@ -15,15 +15,13 @@ expect_after { -i $any_spawn_id timeout { exit 1 } } -system "$basedir/helpers/02112_prepare.sh" -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_LOCAL --disable_suggestion --interactive --queries-file $basedir/file_02112" +system "echo \"drop table if exists t; create table t(i String) engine=Memory; insert into t select 'test string'\" > $env(CLICKHOUSE_TMP)/file_02112" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_LOCAL --disable_suggestion --interactive --queries-file $env(CLICKHOUSE_TMP)/file_02112" expect ":) " -send -- "select * from t format TSV\r" -expect "1" +send -- "select \* from t format TSV\r" +expect "test string" expect ":) " send "" expect eof - -system "$basedir/helpers/02112_clean.sh" diff --git a/tests/queries/0_stateless/helpers/02112_clean.sh b/tests/queries/0_stateless/helpers/02112_clean.sh deleted file mode 100755 index 95af0cede9c..00000000000 --- a/tests/queries/0_stateless/helpers/02112_clean.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -FILE=${CURDIR}/../file_02112 -rm "$FILE" diff --git a/tests/queries/0_stateless/helpers/02112_prepare.sh b/tests/queries/0_stateless/helpers/02112_prepare.sh deleted file mode 100755 index c2791b01140..00000000000 --- a/tests/queries/0_stateless/helpers/02112_prepare.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -FILE=${CURDIR}/../file_02112 -echo "drop table if exists t;create table t(i Int32) engine=Memory; insert into t select 1" > "$FILE" From 92dde41e2ab6aa3a21dc9bf5bdb36e76c57c1d67 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 26 Feb 2023 18:27:51 +0100 Subject: [PATCH 699/717] Fix 01180_client_syntax_errors flakiness Signed-off-by: Azat Khuzhin --- tests/queries/0_stateless/01180_client_syntax_errors.expect | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/01180_client_syntax_errors.expect b/tests/queries/0_stateless/01180_client_syntax_errors.expect index d5c2ab220d5..508ebc3433b 100755 --- a/tests/queries/0_stateless/01180_client_syntax_errors.expect +++ b/tests/queries/0_stateless/01180_client_syntax_errors.expect @@ -31,6 +31,7 @@ expect "Syntax error: failed at position 93 ('UInt64'):*" send -- "select (1, 2\r" expect "Syntax error: failed at position 8 ('('):" expect "Unmatched parentheses: (" +expect ":) " send -- "\4" expect eof From 10c92b6cb0f5f98ff130f224dbcae9e26042ea53 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Sun, 26 Feb 2023 19:06:31 +0100 Subject: [PATCH 700/717] Increased ddl timeout during DROP of table and updated node for concurrent backups on different node --- .../test_disallow_concurrency.py | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py b/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py index ad16725e266..707ea6848bb 100644 --- a/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py +++ b/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py @@ -82,8 +82,10 @@ def drop_after_test(): try: yield finally: - node0.query("DROP TABLE IF EXISTS tbl ON CLUSTER 'cluster' NO DELAY") - node0.query("DROP DATABASE IF EXISTS mydb ON CLUSTER 'cluster' NO DELAY") + node0.query("DROP TABLE IF EXISTS tbl ON CLUSTER 'cluster' NO DELAY", + settings={ + "distributed_ddl_task_timeout": 360, + }) backup_id_counter = 0 @@ -138,7 +140,10 @@ def test_concurrent_backups_on_same_node(): # This restore part is added to confirm creating an internal backup & restore work # even when a concurrent backup is stopped - nodes[0].query(f"DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY") + nodes[0].query(f"DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY", + settings={ + "distributed_ddl_task_timeout": 360, + }) nodes[0].query(f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_name}") nodes[0].query("SYSTEM SYNC REPLICA ON CLUSTER 'cluster' tbl") @@ -158,9 +163,14 @@ def test_concurrent_backups_on_different_nodes(): f"SELECT status FROM system.backups WHERE status == 'CREATING_BACKUP' AND id = '{id}'", "CREATING_BACKUP", ) - assert "Concurrent backups not supported" in nodes[2].query_and_get_error( + assert "Concurrent backups not supported" in nodes[0].query_and_get_error( f"BACKUP TABLE tbl ON CLUSTER 'cluster' TO {backup_name}" ) + assert_eq_with_retry( + nodes[0], + f"SELECT status FROM system.backups WHERE status == 'BACKUP_CREATED' AND id = '{id}'", + "BACKUP_CREATED", + ) def test_concurrent_restores_on_same_node(): @@ -185,7 +195,10 @@ def test_concurrent_restores_on_same_node(): "BACKUP_CREATED", ) - nodes[0].query(f"DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY") + nodes[0].query(f"DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY", + settings={ + "distributed_ddl_task_timeout": 360, + }) restore_id = ( nodes[0] .query(f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_name} ASYNC") @@ -223,7 +236,10 @@ def test_concurrent_restores_on_different_node(): "BACKUP_CREATED", ) - nodes[0].query(f"DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY") + nodes[0].query(f"DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY", + settings={ + "distributed_ddl_task_timeout": 360, + }) restore_id = ( nodes[0] .query(f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_name} ASYNC") From 4c5d39725c5e6a37c45c4cff6980407d6c8f1ff2 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Sun, 26 Feb 2023 19:08:26 +0100 Subject: [PATCH 701/717] Updated node to check BACKUP_CREATED to node1 in test_concurrent_backups_on_different_nodes --- .../test_backup_restore_on_cluster/test_disallow_concurrency.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py b/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py index 707ea6848bb..3ad9e6e47eb 100644 --- a/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py +++ b/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py @@ -167,7 +167,7 @@ def test_concurrent_backups_on_different_nodes(): f"BACKUP TABLE tbl ON CLUSTER 'cluster' TO {backup_name}" ) assert_eq_with_retry( - nodes[0], + nodes[1], f"SELECT status FROM system.backups WHERE status == 'BACKUP_CREATED' AND id = '{id}'", "BACKUP_CREATED", ) From 5f7220e498ddf9720d0f40d023d88e54cb683a36 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sun, 26 Feb 2023 18:15:55 +0000 Subject: [PATCH 702/717] Automatic style fix --- .../test_disallow_concurrency.py | 40 +++++++++++-------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py b/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py index 3ad9e6e47eb..43c8adda65a 100644 --- a/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py +++ b/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py @@ -82,10 +82,12 @@ def drop_after_test(): try: yield finally: - node0.query("DROP TABLE IF EXISTS tbl ON CLUSTER 'cluster' NO DELAY", - settings={ - "distributed_ddl_task_timeout": 360, - }) + node0.query( + "DROP TABLE IF EXISTS tbl ON CLUSTER 'cluster' NO DELAY", + settings={ + "distributed_ddl_task_timeout": 360, + }, + ) backup_id_counter = 0 @@ -140,10 +142,12 @@ def test_concurrent_backups_on_same_node(): # This restore part is added to confirm creating an internal backup & restore work # even when a concurrent backup is stopped - nodes[0].query(f"DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY", - settings={ - "distributed_ddl_task_timeout": 360, - }) + nodes[0].query( + f"DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY", + settings={ + "distributed_ddl_task_timeout": 360, + }, + ) nodes[0].query(f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_name}") nodes[0].query("SYSTEM SYNC REPLICA ON CLUSTER 'cluster' tbl") @@ -195,10 +199,12 @@ def test_concurrent_restores_on_same_node(): "BACKUP_CREATED", ) - nodes[0].query(f"DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY", - settings={ - "distributed_ddl_task_timeout": 360, - }) + nodes[0].query( + f"DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY", + settings={ + "distributed_ddl_task_timeout": 360, + }, + ) restore_id = ( nodes[0] .query(f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_name} ASYNC") @@ -236,10 +242,12 @@ def test_concurrent_restores_on_different_node(): "BACKUP_CREATED", ) - nodes[0].query(f"DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY", - settings={ - "distributed_ddl_task_timeout": 360, - }) + nodes[0].query( + f"DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY", + settings={ + "distributed_ddl_task_timeout": 360, + }, + ) restore_id = ( nodes[0] .query(f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_name} ASYNC") From 73aec86f5a78ff8cc67c8a2dbda781644faaabab Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 26 Feb 2023 03:45:54 +0100 Subject: [PATCH 703/717] Change exception message --- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 06b4cb07f1f..85420cabb8d 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -880,7 +880,7 @@ void IMergeTreeDataPart::writeMetadata(const String & filename, const WriteSetti } catch (...) { - tryLogCurrentException("DataPartStorageOnDiskFull"); + tryLogCurrentException("IMergeTreeDataPart"); } throw; From cc0c0c6133ee62e63a3c42ceb6c285a2d8f5611f Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sun, 26 Feb 2023 20:01:35 +0000 Subject: [PATCH 704/717] A better alternative to #46344 The (experimental) inverted index writes/reads files different from the standard files written by the other skip indexes. The original problem was that with database engine "ordinary", DROP TABLE of a table with inverted index finds unknown files in persistence and complains. The same will happen with engine "atomic" but deferred. As a hotfix, the error was silenced by explicitly adding the four files created in a specific test to the deletion code. This PR tries a cleaner solution where all needed files are provided via the normal checksum structure. One drawback remains which is that the affected files were written earlier and we don't have their checksums available. Therefore, the inverted index is currently excluded from CHECK TABLE. Minimal repro: SET allow_experimental_inverted_index = 1; DROP TABLE IF EXISTS tab; CREATE TABLE tab(s String, INDEX af(s) TYPE inverted(2)) ENGINE = MergeTree() ORDER BY s; INSERT INTO tab VALUES ('Alick a01'); CHECK TABLE tab; DROP TABLE IF EXISTS tab; run ./clickhouse-test with --db-engine Ordinary --- .../MergeTree/DataPartStorageOnDiskBase.cpp | 6 ----- .../MergeTree/MergeTreeDataPartChecksum.cpp | 4 ++++ .../MergeTreeDataPartWriterOnDisk.cpp | 22 ++++++++++++++----- src/Storages/MergeTree/checkDataPart.cpp | 6 ++++- .../02346_full_text_search.reference | 1 + .../0_stateless/02346_full_text_search.sql | 3 +++ 6 files changed, 30 insertions(+), 12 deletions(-) diff --git a/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp b/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp index d8769a94347..175df9b6e28 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp +++ b/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp @@ -684,12 +684,6 @@ void DataPartStorageOnDiskBase::clearDirectory( request.emplace_back(fs::path(dir) / "delete-on-destroy.txt", true); request.emplace_back(fs::path(dir) / "txn_version.txt", true); - /// Inverted index - request.emplace_back(fs::path(dir) / "skp_idx_af.gin_dict", true); - request.emplace_back(fs::path(dir) / "skp_idx_af.gin_post", true); - request.emplace_back(fs::path(dir) / "skp_idx_af.gin_seg", true); - request.emplace_back(fs::path(dir) / "skp_idx_af.gin_sid", true); - disk->removeSharedFiles(request, !can_remove_shared_data, names_not_to_remove); disk->removeDirectory(dir); } diff --git a/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp b/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp index 719a60b2f31..8f4d066baa3 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp @@ -75,6 +75,10 @@ void MergeTreeDataPartChecksums::checkEqual(const MergeTreeDataPartChecksums & r { const String & name = it.first; + /// Exclude files written by inverted index from check. No correct checksums are available for them currently. + if (name.ends_with(".gin_dict") || name.ends_with(".gin_post") || name.ends_with(".gin_seg") || name.ends_with(".gin_sid")) + continue; + auto jt = rhs.files.find(name); if (jt == rhs.files.end()) throw Exception(ErrorCodes::NO_FILE_IN_DATA_PART, "No file {} in data part", name); diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp index 1dec7c2cd7c..fe5dbcefabf 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp @@ -208,26 +208,26 @@ void MergeTreeDataPartWriterOnDisk::initSkipIndices() auto ast = parseQuery(codec_parser, "(" + Poco::toUpper(settings.marks_compression_codec) + ")", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); CompressionCodecPtr marks_compression_codec = CompressionCodecFactory::instance().get(ast, nullptr); - for (const auto & index_helper : skip_indices) + for (const auto & skip_index : skip_indices) { - String stream_name = index_helper->getFileName(); + String stream_name = skip_index->getFileName(); skip_indices_streams.emplace_back( std::make_unique( stream_name, data_part->getDataPartStoragePtr(), - stream_name, index_helper->getSerializedFileExtension(), + stream_name, skip_index->getSerializedFileExtension(), stream_name, marks_file_extension, default_codec, settings.max_compress_block_size, marks_compression_codec, settings.marks_compress_block_size, settings.query_write_settings)); GinIndexStorePtr store = nullptr; - if (dynamic_cast(&*index_helper) != nullptr) + if (dynamic_cast(&*skip_index) != nullptr) { store = std::make_shared(stream_name, data_part->getDataPartStoragePtr(), data_part->getDataPartStoragePtr(), storage.getSettings()->max_digestion_size_per_segment); gin_index_stores[stream_name] = store; } - skip_indices_aggregators.push_back(index_helper->createIndexAggregatorForPart(store)); + skip_indices_aggregators.push_back(skip_index->createIndexAggregatorForPart(store)); skip_index_accumulated_marks.push_back(0); } } @@ -388,6 +388,18 @@ void MergeTreeDataPartWriterOnDisk::fillSkipIndicesChecksums(MergeTreeData::Data auto & stream = *skip_indices_streams[i]; if (!skip_indices_aggregators[i]->empty()) skip_indices_aggregators[i]->getGranuleAndReset()->serializeBinary(stream.compressed_hashing); + + /// Register additional files written only by the inverted index. Required because otherwise DROP TABLE complains about unknown + /// files. Note that the provided actual checksums are bogus. The problem is that at this point the file writes happened already and + /// we'd need to re-open + hash the files (fixing this is TODO). For now, CHECK TABLE skips these four files. + if (dynamic_cast(&*skip_indices[i]) != nullptr) + { + String filename_without_extension = skip_indices[i]->getFileName(); + checksums.files[filename_without_extension + ".gin_dict"] = MergeTreeDataPartChecksums::Checksum(); + checksums.files[filename_without_extension + ".gin_post"] = MergeTreeDataPartChecksums::Checksum(); + checksums.files[filename_without_extension + ".gin_seg"] = MergeTreeDataPartChecksums::Checksum(); + checksums.files[filename_without_extension + ".gin_sid"] = MergeTreeDataPartChecksums::Checksum(); + } } for (auto & stream : skip_indices_streams) diff --git a/src/Storages/MergeTree/checkDataPart.cpp b/src/Storages/MergeTree/checkDataPart.cpp index 2ec83d99eeb..3a33572d047 100644 --- a/src/Storages/MergeTree/checkDataPart.cpp +++ b/src/Storages/MergeTree/checkDataPart.cpp @@ -163,12 +163,16 @@ IMergeTreeDataPart::Checksums checkDataPart( auto file_name = it->name(); /// We will check projections later. - if (data_part_storage.isDirectory(file_name) && endsWith(file_name, ".proj")) + if (data_part_storage.isDirectory(file_name) && file_name.ends_with(".proj")) { projections_on_disk.insert(file_name); continue; } + /// Exclude files written by inverted index from check. No correct checksums are available for them currently. + if (file_name.ends_with(".gin_dict") || file_name.ends_with(".gin_post") || file_name.ends_with(".gin_seg") || file_name.ends_with(".gin_sid")) + continue; + auto checksum_it = checksums_data.files.find(file_name); /// Skip files that we already calculated. Also skip metadata files that are not checksummed. diff --git a/tests/queries/0_stateless/02346_full_text_search.reference b/tests/queries/0_stateless/02346_full_text_search.reference index e035e93867b..f1e21e511d0 100644 --- a/tests/queries/0_stateless/02346_full_text_search.reference +++ b/tests/queries/0_stateless/02346_full_text_search.reference @@ -1,4 +1,5 @@ af inverted +1 101 Alick a01 1 101 Alick a01 diff --git a/tests/queries/0_stateless/02346_full_text_search.sql b/tests/queries/0_stateless/02346_full_text_search.sql index af49c5d52c2..2b10800e78f 100644 --- a/tests/queries/0_stateless/02346_full_text_search.sql +++ b/tests/queries/0_stateless/02346_full_text_search.sql @@ -15,6 +15,9 @@ INSERT INTO tab VALUES (101, 'Alick a01'), (102, 'Blick a02'), (103, 'Click a03' -- check inverted index was created SELECT name, type FROM system.data_skipping_indices WHERE table =='tab' AND database = currentDatabase() LIMIT 1; +-- throw in a random consistency check +CHECK TABLE tab; + -- search inverted index with == SELECT * FROM tab WHERE s == 'Alick a01'; From c32f579d6729d3ec51cf0f42bb635e595b4b0fc8 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 26 Feb 2023 22:58:40 +0100 Subject: [PATCH 705/717] Review from @tavplubix --- .../AggregateFunctionGroupArray.h | 6 ++++++ .../AggregateFunctionGroupArrayMoving.h | 3 +++ .../AggregateFunctionMaxIntersections.h | 17 +++++++++++------ 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionGroupArray.h b/src/AggregateFunctions/AggregateFunctionGroupArray.h index a2b5ed9c493..eaffb04e2a9 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupArray.h +++ b/src/AggregateFunctions/AggregateFunctionGroupArray.h @@ -63,6 +63,9 @@ static constexpr const char * getNameByTrait() template struct GroupArraySamplerData { + /// For easy serialization. + static_assert(std::has_unique_object_representations_v || std::is_floating_point_v); + // Switch to ordinary Allocator after 4096 bytes to avoid fragmentation and trash in Arena using Allocator = MixedAlignedArenaAllocator; using Array = PODArray; @@ -97,6 +100,9 @@ struct GroupArrayNumericData; template struct GroupArrayNumericData { + /// For easy serialization. + static_assert(std::has_unique_object_representations_v || std::is_floating_point_v); + // Switch to ordinary Allocator after 4096 bytes to avoid fragmentation and trash in Arena using Allocator = MixedAlignedArenaAllocator; using Array = PODArray; diff --git a/src/AggregateFunctions/AggregateFunctionGroupArrayMoving.h b/src/AggregateFunctions/AggregateFunctionGroupArrayMoving.h index 5a2da8aa459..8d7010c10db 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupArrayMoving.h +++ b/src/AggregateFunctions/AggregateFunctionGroupArrayMoving.h @@ -32,6 +32,9 @@ namespace ErrorCodes template struct MovingData { + /// For easy serialization. + static_assert(std::has_unique_object_representations_v || std::is_floating_point_v); + using Accumulator = T; /// Switch to ordinary Allocator after 4096 bytes to avoid fragmentation and trash in Arena diff --git a/src/AggregateFunctions/AggregateFunctionMaxIntersections.h b/src/AggregateFunctions/AggregateFunctionMaxIntersections.h index d132c5706ab..563d56c6f40 100644 --- a/src/AggregateFunctions/AggregateFunctionMaxIntersections.h +++ b/src/AggregateFunctions/AggregateFunctionMaxIntersections.h @@ -118,14 +118,19 @@ public: size_t size = value.size(); writeVarUInt(size, buf); + /// In this version, pairs were serialized with padding. + /// We must ensure that padding bytes are zero-filled. + + static_assert(offsetof(typename MaxIntersectionsData::Value, first) == 0); + static_assert(offsetof(typename MaxIntersectionsData::Value, second) > 0); + + char zero_padding[offsetof(typename MaxIntersectionsData::Value, second) - sizeof(value[0].first)]{}; + for (size_t i = 0; i < size; ++i) { - /// In this version, pairs were serialized with padding. - /// We must ensure that padding bytes are zero-filled. - char bytes[sizeof(value[0])]{}; - unalignedStore(&bytes[offsetof(typename MaxIntersectionsData::Value, first)], value[i].first); - unalignedStore(&bytes[offsetof(typename MaxIntersectionsData::Value, second)], value[i].second); - buf.write(bytes, sizeof(value[0])); + writePODBinary(value[i].first, buf); + writePODBinary(zero_padding, buf); + writePODBinary(value[i].second, buf); } } From fd97f7ad0a1af1bf63118e13bb185feef6a17478 Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Sun, 26 Feb 2023 19:25:13 -0400 Subject: [PATCH 706/717] Update settings.md --- docs/en/operations/settings/settings.md | 29 +++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 11d0e6c28de..f960d2df98e 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -3954,6 +3954,13 @@ Use this setting only for backward compatibility if your use cases depend on old Automatically applies [FINAL](../../sql-reference/statements/select/from/#final-modifier) modifier to all tables in a query, to tables where [FINAL](../../sql-reference/statements/select/from/#final-modifier) is applicable, including joined tables and tables in sub-queries, and distributed tables. +Possible values: + +- 0 - disabled +- 1 - enabled + +Default value: `0`. + Example: ```sql @@ -3987,3 +3994,25 @@ SELECT * FROM test; │ 1 │ second │ └─────┴────────┘ ``` + +## asterisk_include_materialized_columns {#asterisk_include_materialized_columns} + +Include [MATERIALIZED](../../sql-reference/statements/create/table/#materialized) columns for wildcard query (`SELECT *`). + +Possible values: + +- 0 - disabled +- 1 - enabled + +Default value: `0`. + +## asterisk_include_alias_columns {#asterisk_include_alias_columns} + +Include [ALIAS](../../sql-reference/statements/create/table/#alias) columns for wildcard query (`SELECT *`). + +Possible values: + +- 0 - disabled +- 1 - enabled + +Default value: `0`. From 458c71179965ca14097455448a6bbba12bc70e90 Mon Sep 17 00:00:00 2001 From: jun won <35126809+juunone@users.noreply.github.com> Date: Mon, 27 Feb 2023 14:04:19 +0900 Subject: [PATCH 707/717] docs: fixed Moving Sliding Average (per 10 days) query typo fixed Moving Sliding Average (per 10 days) query typo --- docs/en/sql-reference/window-functions/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/window-functions/index.md b/docs/en/sql-reference/window-functions/index.md index a8186c20026..59d49830852 100644 --- a/docs/en/sql-reference/window-functions/index.md +++ b/docs/en/sql-reference/window-functions/index.md @@ -567,7 +567,7 @@ SELECT ts, value, round(avg(value) OVER (PARTITION BY metric ORDER BY toDate(ts) - Range BETWEEN 10 PRECEDING AND CURRENT ROW),2) moving_avg_10_days_temp + Range BETWEEN 10 PRECEDING AND CURRENT ROW),2) AS moving_avg_10_days_temp FROM sensors ORDER BY metric ASC, From d32a3035fea94200242d5acd2c8455117482a295 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Mon, 27 Feb 2023 08:10:19 +0000 Subject: [PATCH 708/717] Docs: Sidebar entry "Working with maps" --> Maps --- docs/en/sql-reference/functions/tuple-map-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/tuple-map-functions.md b/docs/en/sql-reference/functions/tuple-map-functions.md index 2dad95745e5..d6cdb187612 100644 --- a/docs/en/sql-reference/functions/tuple-map-functions.md +++ b/docs/en/sql-reference/functions/tuple-map-functions.md @@ -1,7 +1,7 @@ --- slug: /en/sql-reference/functions/tuple-map-functions sidebar_position: 46 -sidebar_label: Working with maps +sidebar_label: Maps title: "Functions for maps" --- From 86dd9b9855d150761e22ac1af25002fdfaafd844 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Mon, 27 Feb 2023 08:13:09 +0000 Subject: [PATCH 709/717] Docs: Sidebar: Remove leading "For" from "Searching/Replacing in Strings" --- docs/en/sql-reference/functions/string-replace-functions.md | 2 +- docs/en/sql-reference/functions/string-search-functions.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/functions/string-replace-functions.md b/docs/en/sql-reference/functions/string-replace-functions.md index d1f0e44f6b4..50e15f70f5d 100644 --- a/docs/en/sql-reference/functions/string-replace-functions.md +++ b/docs/en/sql-reference/functions/string-replace-functions.md @@ -1,7 +1,7 @@ --- slug: /en/sql-reference/functions/string-replace-functions sidebar_position: 42 -sidebar_label: For Replacing in Strings +sidebar_label: Replacing in Strings --- # Functions for Searching and Replacing in Strings diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md index dec50caef86..43ce04d3103 100644 --- a/docs/en/sql-reference/functions/string-search-functions.md +++ b/docs/en/sql-reference/functions/string-search-functions.md @@ -1,7 +1,7 @@ --- slug: /en/sql-reference/functions/string-search-functions sidebar_position: 41 -sidebar_label: For Searching in Strings +sidebar_label: Searching in Strings --- # Functions for Searching in Strings From e153aae54ed965d914992218d1f9b5e771ac8822 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Mon, 27 Feb 2023 08:15:04 +0000 Subject: [PATCH 710/717] Capitalize "Map" in heading --- docs/en/sql-reference/functions/tuple-map-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/tuple-map-functions.md b/docs/en/sql-reference/functions/tuple-map-functions.md index d6cdb187612..4a6780df292 100644 --- a/docs/en/sql-reference/functions/tuple-map-functions.md +++ b/docs/en/sql-reference/functions/tuple-map-functions.md @@ -2,7 +2,7 @@ slug: /en/sql-reference/functions/tuple-map-functions sidebar_position: 46 sidebar_label: Maps -title: "Functions for maps" +title: "Functions for Maps" --- ## map From 22d09caea2fadce928094aac6ce902dc7f1c6933 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Mon, 27 Feb 2023 08:31:40 +0000 Subject: [PATCH 711/717] SIZES_OF_ARRAYS_DOESN_MATCH --> SIZES_OF_ARRAYS_DONT_MATCH --- src/AggregateFunctions/AggregateFunctionArray.h | 4 ++-- src/AggregateFunctions/AggregateFunctionForEach.h | 4 ++-- src/Common/ErrorCodes.cpp | 2 +- src/DataTypes/NestedUtils.cpp | 4 ++-- src/Functions/FunctionHelpers.cpp | 4 ++-- src/Functions/array/FunctionArrayMapped.h | 4 ++-- src/Functions/array/arrayDistance.cpp | 6 +++--- src/Functions/array/arrayEnumerateExtended.h | 4 ++-- src/Functions/array/arrayEnumerateRanked.h | 8 ++++---- src/Functions/array/arrayReduce.cpp | 4 ++-- src/Functions/array/arrayReduceInRanges.cpp | 4 ++-- src/Functions/array/arrayUniq.cpp | 4 ++-- src/Functions/array/arrayZip.cpp | 4 ++-- src/Functions/nested.cpp | 4 ++-- src/Functions/tupleElement.cpp | 6 +++--- src/Functions/validateNestedArraySizes.cpp | 4 ++-- src/Interpreters/ArrayJoinAction.cpp | 4 ++-- 17 files changed, 37 insertions(+), 37 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionArray.h b/src/AggregateFunctions/AggregateFunctionArray.h index d1494f46f4b..21394e3ce05 100644 --- a/src/AggregateFunctions/AggregateFunctionArray.h +++ b/src/AggregateFunctions/AggregateFunctionArray.h @@ -13,7 +13,7 @@ struct Settings; namespace ErrorCodes { - extern const int SIZES_OF_ARRAYS_DOESNT_MATCH; + extern const int SIZES_OF_ARRAYS_DONT_MATCH; extern const int ILLEGAL_TYPE_OF_ARGUMENT; } @@ -129,7 +129,7 @@ public: const IColumn::Offsets & ith_offsets = ith_column.getOffsets(); if (ith_offsets[row_num] != end || (row_num != 0 && ith_offsets[row_num - 1] != begin)) - throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH, "Arrays passed to {} aggregate function have different sizes", getName()); + throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DONT_MATCH, "Arrays passed to {} aggregate function have different sizes", getName()); } for (size_t i = begin; i < end; ++i) diff --git a/src/AggregateFunctions/AggregateFunctionForEach.h b/src/AggregateFunctions/AggregateFunctionForEach.h index e035d645222..f041dd11209 100644 --- a/src/AggregateFunctions/AggregateFunctionForEach.h +++ b/src/AggregateFunctions/AggregateFunctionForEach.h @@ -19,7 +19,7 @@ namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; - extern const int SIZES_OF_ARRAYS_DOESNT_MATCH; + extern const int SIZES_OF_ARRAYS_DONT_MATCH; } @@ -197,7 +197,7 @@ public: const IColumn::Offsets & ith_offsets = ith_column.getOffsets(); if (ith_offsets[row_num] != end || (row_num != 0 && ith_offsets[row_num - 1] != begin)) - throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH, "Arrays passed to {} aggregate function have different sizes", getName()); + throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DONT_MATCH, "Arrays passed to {} aggregate function have different sizes", getName()); } AggregateFunctionForEachData & state = ensureAggregateData(place, end - begin, *arena); diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 028663a2176..c00129249e2 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -197,7 +197,7 @@ M(187, COLLATION_COMPARISON_FAILED) \ M(188, UNKNOWN_ACTION) \ M(189, TABLE_MUST_NOT_BE_CREATED_MANUALLY) \ - M(190, SIZES_OF_ARRAYS_DOESNT_MATCH) \ + M(190, SIZES_OF_ARRAYS_DONT_MATCH) \ M(191, SET_SIZE_LIMIT_EXCEEDED) \ M(192, UNKNOWN_USER) \ M(193, WRONG_PASSWORD) \ diff --git a/src/DataTypes/NestedUtils.cpp b/src/DataTypes/NestedUtils.cpp index e5ba23b9df8..f029ac6ba27 100644 --- a/src/DataTypes/NestedUtils.cpp +++ b/src/DataTypes/NestedUtils.cpp @@ -25,7 +25,7 @@ namespace DB namespace ErrorCodes { extern const int ILLEGAL_COLUMN; - extern const int SIZES_OF_ARRAYS_DOESNT_MATCH; + extern const int SIZES_OF_ARRAYS_DONT_MATCH; } namespace Nested @@ -242,7 +242,7 @@ void validateArraySizes(const Block & block) const ColumnArray & another_array_column = assert_cast(*elem.column); if (!first_array_column.hasEqualOffsets(another_array_column)) - throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH, + throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DONT_MATCH, "Elements '{}' and '{}' " "of Nested data structure '{}' (Array columns) have different array sizes.", block.getByPosition(it->second).name, elem.name, split.first); diff --git a/src/Functions/FunctionHelpers.cpp b/src/Functions/FunctionHelpers.cpp index c981f666219..ff09274d907 100644 --- a/src/Functions/FunctionHelpers.cpp +++ b/src/Functions/FunctionHelpers.cpp @@ -16,7 +16,7 @@ namespace ErrorCodes { extern const int ILLEGAL_COLUMN; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; - extern const int SIZES_OF_ARRAYS_DOESNT_MATCH; + extern const int SIZES_OF_ARRAYS_DONT_MATCH; extern const int ILLEGAL_TYPE_OF_ARGUMENT; } @@ -213,7 +213,7 @@ checkAndGetNestedArrayOffset(const IColumn ** columns, size_t num_arguments) if (i == 0) offsets = offsets_i; else if (*offsets_i != *offsets) - throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH, "Lengths of all arrays passed to aggregate function must be equal."); + throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DONT_MATCH, "Lengths of all arrays passed to aggregate function must be equal."); } return {nested_columns, offsets->data()}; } diff --git a/src/Functions/array/FunctionArrayMapped.h b/src/Functions/array/FunctionArrayMapped.h index 989ec500cfb..61abc607349 100644 --- a/src/Functions/array/FunctionArrayMapped.h +++ b/src/Functions/array/FunctionArrayMapped.h @@ -37,7 +37,7 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int LOGICAL_ERROR; - extern const int SIZES_OF_ARRAYS_DOESNT_MATCH; + extern const int SIZES_OF_ARRAYS_DONT_MATCH; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } @@ -361,7 +361,7 @@ public: if (getOffsetsPtr(*column_array) != offsets_column && getOffsets(*column_array) != typeid_cast(*offsets_column).getData()) throw Exception( - ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH, + ErrorCodes::SIZES_OF_ARRAYS_DONT_MATCH, "{}s passed to {} must have equal size", argument_type_name, getName()); diff --git a/src/Functions/array/arrayDistance.cpp b/src/Functions/array/arrayDistance.cpp index c1137848cc5..c68c89ee0d5 100644 --- a/src/Functions/array/arrayDistance.cpp +++ b/src/Functions/array/arrayDistance.cpp @@ -16,7 +16,7 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int LOGICAL_ERROR; - extern const int SIZES_OF_ARRAYS_DOESNT_MATCH; + extern const int SIZES_OF_ARRAYS_DONT_MATCH; extern const int ARGUMENT_OUT_OF_BOUND; } @@ -356,7 +356,7 @@ private: { ColumnArray::Offset prev_offset = row > 0 ? offsets_x[row] : 0; throw Exception( - ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH, + ErrorCodes::SIZES_OF_ARRAYS_DONT_MATCH, "Arguments of function {} have different array sizes: {} and {}", getName(), offsets_x[row] - prev_offset, @@ -423,7 +423,7 @@ private: if (unlikely(offsets_x[0] != offsets_y[row] - prev_offset)) { throw Exception( - ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH, + ErrorCodes::SIZES_OF_ARRAYS_DONT_MATCH, "Arguments of function {} have different array sizes: {} and {}", getName(), offsets_x[0], diff --git a/src/Functions/array/arrayEnumerateExtended.h b/src/Functions/array/arrayEnumerateExtended.h index c3d69bb6972..3f145c05b54 100644 --- a/src/Functions/array/arrayEnumerateExtended.h +++ b/src/Functions/array/arrayEnumerateExtended.h @@ -20,7 +20,7 @@ namespace ErrorCodes extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int ILLEGAL_COLUMN; extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int SIZES_OF_ARRAYS_DOESNT_MATCH; + extern const int SIZES_OF_ARRAYS_DONT_MATCH; } class FunctionArrayEnumerateUniq; @@ -153,7 +153,7 @@ ColumnPtr FunctionArrayEnumerateExtended::executeImpl(const ColumnsWith offsets_column = array->getOffsetsPtr(); } else if (offsets_i != *offsets) - throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH, "Lengths of all arrays passed to {} must be equal.", + throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DONT_MATCH, "Lengths of all arrays passed to {} must be equal.", getName()); const auto * array_data = &array->getData(); diff --git a/src/Functions/array/arrayEnumerateRanked.h b/src/Functions/array/arrayEnumerateRanked.h index 73feb3e46ea..8a348c07421 100644 --- a/src/Functions/array/arrayEnumerateRanked.h +++ b/src/Functions/array/arrayEnumerateRanked.h @@ -60,7 +60,7 @@ namespace DB namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; - extern const int SIZES_OF_ARRAYS_DOESNT_MATCH; + extern const int SIZES_OF_ARRAYS_DONT_MATCH; } class FunctionArrayEnumerateUniqRanked; @@ -194,7 +194,7 @@ ColumnPtr FunctionArrayEnumerateRankedExtended::executeImpl( { if (*offsets_by_depth[0] != array->getOffsets()) { - throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH, + throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DONT_MATCH, "Lengths and effective depths of all arrays passed to {} must be equal.", getName()); } } @@ -217,7 +217,7 @@ ColumnPtr FunctionArrayEnumerateRankedExtended::executeImpl( { if (*offsets_by_depth[col_depth] != array->getOffsets()) { - throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH, + throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DONT_MATCH, "Lengths and effective depths of all arrays passed to {} must be equal.", getName()); } } @@ -225,7 +225,7 @@ ColumnPtr FunctionArrayEnumerateRankedExtended::executeImpl( if (col_depth < arrays_depths.depths[array_num]) { - throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH, + throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DONT_MATCH, "{}: Passed array number {} depth ({}) is more than the actual array depth ({}).", getName(), array_num, std::to_string(arrays_depths.depths[array_num]), col_depth); } diff --git a/src/Functions/array/arrayReduce.cpp b/src/Functions/array/arrayReduce.cpp index d4896595941..a4b2cc037ab 100644 --- a/src/Functions/array/arrayReduce.cpp +++ b/src/Functions/array/arrayReduce.cpp @@ -19,7 +19,7 @@ namespace DB namespace ErrorCodes { - extern const int SIZES_OF_ARRAYS_DOESNT_MATCH; + extern const int SIZES_OF_ARRAYS_DONT_MATCH; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int ILLEGAL_COLUMN; extern const int ILLEGAL_TYPE_OF_ARGUMENT; @@ -144,7 +144,7 @@ ColumnPtr FunctionArrayReduce::executeImpl(const ColumnsWithTypeAndName & argume if (i == 0) offsets = offsets_i; else if (*offsets_i != *offsets) - throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH, "Lengths of all arrays passed to {} must be equal.", + throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DONT_MATCH, "Lengths of all arrays passed to {} must be equal.", getName()); } const IColumn ** aggregate_arguments = aggregate_arguments_vec.data(); diff --git a/src/Functions/array/arrayReduceInRanges.cpp b/src/Functions/array/arrayReduceInRanges.cpp index 07391c963a6..790bc3ef879 100644 --- a/src/Functions/array/arrayReduceInRanges.cpp +++ b/src/Functions/array/arrayReduceInRanges.cpp @@ -21,7 +21,7 @@ namespace DB namespace ErrorCodes { - extern const int SIZES_OF_ARRAYS_DOESNT_MATCH; + extern const int SIZES_OF_ARRAYS_DONT_MATCH; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int ILLEGAL_COLUMN; extern const int ILLEGAL_TYPE_OF_ARGUMENT; @@ -190,7 +190,7 @@ ColumnPtr FunctionArrayReduceInRanges::executeImpl( if (i == 0) offsets = offsets_i; else if (*offsets_i != *offsets) - throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH, "Lengths of all arrays passed to {} must be equal.", + throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DONT_MATCH, "Lengths of all arrays passed to {} must be equal.", getName()); } const IColumn ** aggregate_arguments = aggregate_arguments_vec.data(); diff --git a/src/Functions/array/arrayUniq.cpp b/src/Functions/array/arrayUniq.cpp index 1d1cf4e6392..81ba5b62094 100644 --- a/src/Functions/array/arrayUniq.cpp +++ b/src/Functions/array/arrayUniq.cpp @@ -18,7 +18,7 @@ namespace DB namespace ErrorCodes { - extern const int SIZES_OF_ARRAYS_DOESNT_MATCH; + extern const int SIZES_OF_ARRAYS_DONT_MATCH; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int ILLEGAL_COLUMN; extern const int ILLEGAL_TYPE_OF_ARGUMENT; @@ -151,7 +151,7 @@ ColumnPtr FunctionArrayUniq::executeImpl(const ColumnsWithTypeAndName & argument if (i == 0) offsets = &offsets_i; else if (offsets_i != *offsets) - throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH, "Lengths of all arrays passed to {} must be equal.", + throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DONT_MATCH, "Lengths of all arrays passed to {} must be equal.", getName()); const auto * array_data = &array->getData(); diff --git a/src/Functions/array/arrayZip.cpp b/src/Functions/array/arrayZip.cpp index 3a50491fd4b..44c323e3fe3 100644 --- a/src/Functions/array/arrayZip.cpp +++ b/src/Functions/array/arrayZip.cpp @@ -13,7 +13,7 @@ namespace DB namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int SIZES_OF_ARRAYS_DOESNT_MATCH; + extern const int SIZES_OF_ARRAYS_DONT_MATCH; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int ILLEGAL_COLUMN; } @@ -81,7 +81,7 @@ public: } else if (!column_array->hasEqualOffsets(static_cast(*first_array_column))) { - throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH, + throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DONT_MATCH, "The argument 1 and argument {} of function {} have different array sizes", i + 1, getName()); } diff --git a/src/Functions/nested.cpp b/src/Functions/nested.cpp index 7617951784f..b22330cd881 100644 --- a/src/Functions/nested.cpp +++ b/src/Functions/nested.cpp @@ -20,7 +20,7 @@ namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; - extern const int SIZES_OF_ARRAYS_DOESNT_MATCH; + extern const int SIZES_OF_ARRAYS_DONT_MATCH; } namespace @@ -118,7 +118,7 @@ public: const auto * rhs_array = assert_cast(arguments[i].column.get()); if (!lhs_array->hasEqualOffsets(*rhs_array)) - throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH, + throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DONT_MATCH, "The argument 1 and argument {} of function {} have different array offsets", i + 1, getName()); diff --git a/src/Functions/tupleElement.cpp b/src/Functions/tupleElement.cpp index 879b6feed40..b1fd200f5cd 100644 --- a/src/Functions/tupleElement.cpp +++ b/src/Functions/tupleElement.cpp @@ -21,7 +21,7 @@ namespace ErrorCodes extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int NOT_FOUND_COLUMN_IN_BLOCK; extern const int NUMBER_OF_DIMENSIONS_MISMATCHED; - extern const int SIZES_OF_ARRAYS_DOESNT_MATCH; + extern const int SIZES_OF_ARRAYS_DONT_MATCH; } namespace @@ -200,7 +200,7 @@ private: const auto & array_y = *assert_cast(col_y.get()); if (!array_x.hasEqualOffsets(array_y)) { - throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH, + throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DONT_MATCH, "The argument 1 and argument 3 of function {} have different array sizes", getName()); } } @@ -222,7 +222,7 @@ private: { if (unlikely(offsets_x[0] != offsets_y[row] - prev_offset)) { - throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH, + throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DONT_MATCH, "The argument 1 and argument 3 of function {} have different array sizes", getName()); } prev_offset = offsets_y[row]; diff --git a/src/Functions/validateNestedArraySizes.cpp b/src/Functions/validateNestedArraySizes.cpp index 7e1dbc798d8..c422637ba7f 100644 --- a/src/Functions/validateNestedArraySizes.cpp +++ b/src/Functions/validateNestedArraySizes.cpp @@ -12,7 +12,7 @@ namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; - extern const int SIZES_OF_ARRAYS_DOESNT_MATCH; + extern const int SIZES_OF_ARRAYS_DONT_MATCH; } /** Function validateNestedArraySizes is used to check the consistency of Nested DataType subcolumns's offsets when Update @@ -106,7 +106,7 @@ ColumnPtr FunctionValidateNestedArraySizes::executeImpl( else if (first_length != length) { throw Exception( - ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH, + ErrorCodes::SIZES_OF_ARRAYS_DONT_MATCH, "Elements '{}' and '{}' of Nested data structure (Array columns) " "have different array sizes ({} and {} respectively) on row {}", arguments[1].name, arguments[args_idx].name, first_length, length, i); diff --git a/src/Interpreters/ArrayJoinAction.cpp b/src/Interpreters/ArrayJoinAction.cpp index 3650b888f9e..4f42122e98f 100644 --- a/src/Interpreters/ArrayJoinAction.cpp +++ b/src/Interpreters/ArrayJoinAction.cpp @@ -14,7 +14,7 @@ namespace DB namespace ErrorCodes { extern const int LOGICAL_ERROR; - extern const int SIZES_OF_ARRAYS_DOESNT_MATCH; + extern const int SIZES_OF_ARRAYS_DONT_MATCH; extern const int TYPE_MISMATCH; } @@ -186,7 +186,7 @@ void ArrayJoinAction::execute(Block & block) const ColumnArray & array = typeid_cast(*array_ptr); if (!is_unaligned && !array.hasEqualOffsets(*any_array)) - throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH, "Sizes of ARRAY-JOIN-ed arrays do not match"); + throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DONT_MATCH, "Sizes of ARRAY-JOIN-ed arrays do not match"); current.column = typeid_cast(*array_ptr).getDataPtr(); current.type = type->getNestedType(); From 4c9b9b362c16ddbbaeda2446284b861c30cef01e Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Mon, 27 Feb 2023 09:30:50 +0000 Subject: [PATCH 712/717] Replace dynamic_cast by typeid_cast --- src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp | 6 +++--- src/Storages/MergeTree/checkDataPart.cpp | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp index fe5dbcefabf..b0101bb962c 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp @@ -222,7 +222,7 @@ void MergeTreeDataPartWriterOnDisk::initSkipIndices() settings.query_write_settings)); GinIndexStorePtr store = nullptr; - if (dynamic_cast(&*skip_index) != nullptr) + if (typeid_cast(&*skip_index) != nullptr) { store = std::make_shared(stream_name, data_part->getDataPartStoragePtr(), data_part->getDataPartStoragePtr(), storage.getSettings()->max_digestion_size_per_segment); gin_index_stores[stream_name] = store; @@ -284,7 +284,7 @@ void MergeTreeDataPartWriterOnDisk::calculateAndSerializeSkipIndices(const Block WriteBuffer & marks_out = stream.compress_marks ? stream.marks_compressed_hashing : stream.marks_hashing; GinIndexStorePtr store; - if (dynamic_cast(&*index_helper) != nullptr) + if (typeid_cast(&*index_helper) != nullptr) { String stream_name = index_helper->getFileName(); auto it = gin_index_stores.find(stream_name); @@ -392,7 +392,7 @@ void MergeTreeDataPartWriterOnDisk::fillSkipIndicesChecksums(MergeTreeData::Data /// Register additional files written only by the inverted index. Required because otherwise DROP TABLE complains about unknown /// files. Note that the provided actual checksums are bogus. The problem is that at this point the file writes happened already and /// we'd need to re-open + hash the files (fixing this is TODO). For now, CHECK TABLE skips these four files. - if (dynamic_cast(&*skip_indices[i]) != nullptr) + if (typeid_cast(&*skip_indices[i]) != nullptr) { String filename_without_extension = skip_indices[i]->getFileName(); checksums.files[filename_without_extension + ".gin_dict"] = MergeTreeDataPartChecksums::Checksum(); diff --git a/src/Storages/MergeTree/checkDataPart.cpp b/src/Storages/MergeTree/checkDataPart.cpp index 3a33572d047..de31258b2f9 100644 --- a/src/Storages/MergeTree/checkDataPart.cpp +++ b/src/Storages/MergeTree/checkDataPart.cpp @@ -157,7 +157,7 @@ IMergeTreeDataPart::Checksums checkDataPart( } NameSet projections_on_disk; - const auto & checksum_files_txt = checksums_txt.files; + const auto & checksums_txt_files = checksums_txt.files; for (auto it = data_part_storage.iterate(); it->isValid(); it->next()) { auto file_name = it->name(); @@ -178,8 +178,8 @@ IMergeTreeDataPart::Checksums checkDataPart( /// Skip files that we already calculated. Also skip metadata files that are not checksummed. if (checksum_it == checksums_data.files.end() && !files_without_checksums.contains(file_name)) { - auto txt_checksum_it = checksum_files_txt.find(file_name); - if (txt_checksum_it == checksum_files_txt.end() || txt_checksum_it->second.uncompressed_size == 0) + auto txt_checksum_it = checksums_txt_files.find(file_name); + if (txt_checksum_it == checksums_txt_files.end() || txt_checksum_it->second.uncompressed_size == 0) { /// The file is not compressed. checksum_file(file_name); From cb460c6903a4f2c39f801708f16f4d3e810f181f Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Mon, 27 Feb 2023 13:11:17 +0100 Subject: [PATCH 713/717] Fix missing format_description --- tests/ci/docker_manifests_merge.py | 2 +- tests/ci/docker_server.py | 2 +- tests/ci/install_check.py | 2 +- tests/ci/run_check.py | 1 + tests/ci/sqlancer_check.py | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/ci/docker_manifests_merge.py b/tests/ci/docker_manifests_merge.py index e0917581089..0484ea8f641 100644 --- a/tests/ci/docker_manifests_merge.py +++ b/tests/ci/docker_manifests_merge.py @@ -218,7 +218,7 @@ def main(): else: description = "Nothing to update" - format_description(description) + description = format_description(description) gh = Github(get_best_robot_token(), per_page=100) post_commit_status(gh, pr_info.sha, NAME, description, status, url) diff --git a/tests/ci/docker_server.py b/tests/ci/docker_server.py index fa4969a98d5..c6854c5aa78 100644 --- a/tests/ci/docker_server.py +++ b/tests/ci/docker_server.py @@ -369,7 +369,7 @@ def main(): description = f"Processed tags: {', '.join(tags)}" - format_description(description) + description = format_description(description) gh = Github(get_best_robot_token(), per_page=100) post_commit_status(gh, pr_info.sha, NAME, description, status, url) diff --git a/tests/ci/install_check.py b/tests/ci/install_check.py index b0d0af380bd..54245670b26 100644 --- a/tests/ci/install_check.py +++ b/tests/ci/install_check.py @@ -345,7 +345,7 @@ def main(): ch_helper = ClickHouseHelper() mark_flaky_tests(ch_helper, args.check_name, test_results) - format_description(description) + description = format_description(description) post_commit_status(gh, pr_info.sha, args.check_name, description, state, report_url) diff --git a/tests/ci/run_check.py b/tests/ci/run_check.py index e80987e8bc5..c6810173f7a 100644 --- a/tests/ci/run_check.py +++ b/tests/ci/run_check.py @@ -200,6 +200,7 @@ if __name__ == "__main__": pr_info = PRInfo(need_orgs=True, pr_event_from_api=True, need_changed_files=True) can_run, description, labels_state = should_run_checks_for_pr(pr_info) + description = format_description(description) gh = Github(get_best_robot_token(), per_page=100) commit = get_commit(gh, pr_info.sha) diff --git a/tests/ci/sqlancer_check.py b/tests/ci/sqlancer_check.py index d71b9ec3e9c..1a6c4d14616 100644 --- a/tests/ci/sqlancer_check.py +++ b/tests/ci/sqlancer_check.py @@ -176,7 +176,7 @@ def main(): # status = "failure" description = "Task failed: $?=" + str(retcode) - format_description(description) + description = format_description(description) report_url = upload_results( s3_helper, From e680cd7b1264442eb3b174b005e45120274dde06 Mon Sep 17 00:00:00 2001 From: Peignon Melvyn Date: Mon, 27 Feb 2023 13:48:54 +0100 Subject: [PATCH 714/717] Update schema-inference.md Fixed the example --- docs/en/interfaces/schema-inference.md | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/docs/en/interfaces/schema-inference.md b/docs/en/interfaces/schema-inference.md index 728afa73a17..a07438a8748 100644 --- a/docs/en/interfaces/schema-inference.md +++ b/docs/en/interfaces/schema-inference.md @@ -1177,8 +1177,22 @@ This setting can be used to specify the types of columns that could not be deter **Example** ```sql -DESC format(JSONEachRow, '{"id" : 1, "age" : 25, "name" : "Josh", "status" : null, "hobbies" : ["football", "cooking"]}' -SETTINGS schema_inference_hints = 'age LowCardinality(UInt8), status Nullable(String)' +DESC format(JSONEachRow, '{"id" : 1, "age" : 25, "name" : "Josh", "status" : null, "hobbies" : ["football", "cooking"]}') SETTINGS schema_inference_hints = 'age LowCardinality(UInt8), status Nullable(String)', allow_suspicious_low_cardinality_types=1 + +DESCRIBE TABLE format(JSONEachRow, '{"id" : 1, "age" : 25, "name" : "Josh", "status" : null, "hobbies" : ["football", "cooking"]}') +SETTINGS schema_inference_hints = 'age LowCardinality(UInt8), status Nullable(String)', allow_suspicious_low_cardinality_types = 1 + +Query id: 1752429f-1cd3-4d61-97af-8324ff97ee25 + +┌─name────┬─type────────────────────┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┬─ttl_expression─┐ +│ id │ Nullable(Int64) │ │ │ │ │ │ +│ age │ LowCardinality(UInt8) │ │ │ │ │ │ +│ name │ Nullable(String) │ │ │ │ │ │ +│ status │ Nullable(String) │ │ │ │ │ │ +│ hobbies │ Array(Nullable(String)) │ │ │ │ │ │ +└─────────┴─────────────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘ + + ``` ```response ┌─name────┬─type────────────────────┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┬─ttl_expression─┐ From 7c5bd4a5f10c4ef483ea0eb6cc36443663c6dc9c Mon Sep 17 00:00:00 2001 From: Peignon Melvyn Date: Mon, 27 Feb 2023 13:53:44 +0100 Subject: [PATCH 715/717] Update schema-inference.md --- docs/en/interfaces/schema-inference.md | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/docs/en/interfaces/schema-inference.md b/docs/en/interfaces/schema-inference.md index a07438a8748..2b50794c458 100644 --- a/docs/en/interfaces/schema-inference.md +++ b/docs/en/interfaces/schema-inference.md @@ -1178,21 +1178,6 @@ This setting can be used to specify the types of columns that could not be deter ```sql DESC format(JSONEachRow, '{"id" : 1, "age" : 25, "name" : "Josh", "status" : null, "hobbies" : ["football", "cooking"]}') SETTINGS schema_inference_hints = 'age LowCardinality(UInt8), status Nullable(String)', allow_suspicious_low_cardinality_types=1 - -DESCRIBE TABLE format(JSONEachRow, '{"id" : 1, "age" : 25, "name" : "Josh", "status" : null, "hobbies" : ["football", "cooking"]}') -SETTINGS schema_inference_hints = 'age LowCardinality(UInt8), status Nullable(String)', allow_suspicious_low_cardinality_types = 1 - -Query id: 1752429f-1cd3-4d61-97af-8324ff97ee25 - -┌─name────┬─type────────────────────┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┬─ttl_expression─┐ -│ id │ Nullable(Int64) │ │ │ │ │ │ -│ age │ LowCardinality(UInt8) │ │ │ │ │ │ -│ name │ Nullable(String) │ │ │ │ │ │ -│ status │ Nullable(String) │ │ │ │ │ │ -│ hobbies │ Array(Nullable(String)) │ │ │ │ │ │ -└─────────┴─────────────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘ - - ``` ```response ┌─name────┬─type────────────────────┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┬─ttl_expression─┐ From 5b6e581c1a86933bf640ceb075f449282500cb3b Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Mon, 27 Feb 2023 14:18:41 +0100 Subject: [PATCH 716/717] Preset description on the tweak reset --- tests/ci/release.py | 2 ++ tests/ci/version_helper.py | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/tests/ci/release.py b/tests/ci/release.py index f7b5155e750..e221bd3d008 100755 --- a/tests/ci/release.py +++ b/tests/ci/release.py @@ -273,7 +273,9 @@ class Release: self, version: ClickHouseVersion, reset_tweak: bool = True ) -> None: if reset_tweak: + desc = version.description version = version.reset_tweak() + version.with_description(desc) update_cmake_version(version) update_contributors(raise_error=True) if self.dry_run: diff --git a/tests/ci/version_helper.py b/tests/ci/version_helper.py index 1ede2c90b55..372974f4eda 100755 --- a/tests/ci/version_helper.py +++ b/tests/ci/version_helper.py @@ -58,6 +58,7 @@ class ClickHouseVersion: elif self._git is not None: self._tweak = self._git.tweak self._describe = "" + self._description = "" def update(self, part: Literal["major", "minor", "patch"]) -> "ClickHouseVersion": """If part is valid, returns a new version""" @@ -125,6 +126,10 @@ class ClickHouseVersion: def describe(self): return self._describe + @property + def description(self) -> str: + return self._description + @property def string(self): return ".".join( @@ -149,6 +154,7 @@ class ClickHouseVersion: def with_description(self, version_type): if version_type not in VersionType.VALID: raise ValueError(f"version type {version_type} not in {VersionType.VALID}") + self._description = version_type self._describe = f"v{self.string}-{version_type}" def __eq__(self, other: Any) -> bool: From 463dbdc838e019cb505dedf9c4446d922867ce9b Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 27 Feb 2023 13:29:21 +0000 Subject: [PATCH 717/717] Update version_date.tsv and changelogs after v22.3.19.6-lts --- docs/changelogs/v22.3.19.6-lts.md | 17 +++++++++++++++++ utils/list-versions/version_date.tsv | 1 + 2 files changed, 18 insertions(+) create mode 100644 docs/changelogs/v22.3.19.6-lts.md diff --git a/docs/changelogs/v22.3.19.6-lts.md b/docs/changelogs/v22.3.19.6-lts.md new file mode 100644 index 00000000000..d5b45f4ce66 --- /dev/null +++ b/docs/changelogs/v22.3.19.6-lts.md @@ -0,0 +1,17 @@ +--- +sidebar_position: 1 +sidebar_label: 2023 +--- + +# 2023 Changelog + +### ClickHouse release v22.3.19.6-lts (467e0a7bd77) FIXME as compared to v22.3.18.37-lts (fe512717551) + +#### Bug Fix (user-visible misbehavior in official stable or prestable release) + +* Backported in [#46440](https://github.com/ClickHouse/ClickHouse/issues/46440): Fix possible `LOGICAL_ERROR` in asynchronous inserts with invalid data sent in format `VALUES`. [#46350](https://github.com/ClickHouse/ClickHouse/pull/46350) ([Anton Popov](https://github.com/CurtizJ)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Get rid of legacy DocsReleaseChecks [#46665](https://github.com/ClickHouse/ClickHouse/pull/46665) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index d4abbdebee1..3814e94bf24 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -63,6 +63,7 @@ v22.4.5.9-stable 2022-05-06 v22.4.4.7-stable 2022-04-29 v22.4.3.3-stable 2022-04-26 v22.4.2.1-stable 2022-04-22 +v22.3.19.6-lts 2023-02-27 v22.3.18.37-lts 2023-02-15 v22.3.17.13-lts 2023-01-12 v22.3.16.1190-lts 2023-01-09
Deprecated Method for installing rpm-packages From 2b6c088eb04057980e93afdc1ce9ef9c8d4ccc53 Mon Sep 17 00:00:00 2001 From: DanRoscigno Date: Fri, 17 Feb 2023 16:04:15 -0500 Subject: [PATCH 383/717] add keeper to tgz list --- docs/en/getting-started/install.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/getting-started/install.md b/docs/en/getting-started/install.md index a0b4efb3ddd..434917ffc82 100644 --- a/docs/en/getting-started/install.md +++ b/docs/en/getting-started/install.md @@ -248,7 +248,7 @@ case $(uname -m) in *) echo "Unknown architecture $(uname -m)"; exit 1 ;; esac -for PKG in clickhouse-common-static clickhouse-common-static-dbg clickhouse-server clickhouse-client +for PKG in clickhouse-common-static clickhouse-common-static-dbg clickhouse-server clickhouse-client clickhouse-keeper do curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION-${ARCH}.tgz" \ || curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION.tgz" From 16bd07067f8590b98d02744005b14a7a91140010 Mon Sep 17 00:00:00 2001 From: Dan Roscigno Date: Fri, 17 Feb 2023 16:08:12 -0500 Subject: [PATCH 384/717] Update docs/en/getting-started/install.md --- docs/en/getting-started/install.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/en/getting-started/install.md b/docs/en/getting-started/install.md index 434917ffc82..404c6c6f227 100644 --- a/docs/en/getting-started/install.md +++ b/docs/en/getting-started/install.md @@ -181,7 +181,9 @@ sudo yum install -y clickhouse-server clickhouse-client #### Start ClickHouse server ```bash -sudo /etc/init.d/clickhouse-server start +sudo systemctl enable clickhouse-server +sudo systemctl start clickhouse-server +sudo systemctl status clickhouse-server clickhouse-client # or "clickhouse-client --password" if you set up a password. ``` From 55e932261219bb18d27381c54a746b758f2dbf50 Mon Sep 17 00:00:00 2001 From: bkuschel Date: Fri, 17 Feb 2023 16:18:30 -0500 Subject: [PATCH 385/717] Fixes for OpenSSL and s390x --- CMakeLists.txt | 2 +- base/poco/Crypto/CMakeLists.txt | 1 + base/poco/Crypto/src/RSAKeyImpl.cpp | 4 ++-- base/poco/Crypto/src/X509Certificate.cpp | 2 +- cmake/target.cmake | 3 +++ 5 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b422d7d807b..0b1ffbd54bb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -458,7 +458,7 @@ endif () set (CMAKE_POSTFIX_VARIABLE "CMAKE_${CMAKE_BUILD_TYPE_UC}_POSTFIX") set (CMAKE_POSITION_INDEPENDENT_CODE OFF) -if (OS_LINUX AND NOT ARCH_AARCH64) +if (OS_LINUX AND NOT (ARCH_AARCH64 OR ARCH_S390X)) # Slightly more efficient code can be generated # It's disabled for ARM because otherwise ClickHouse cannot run on Android. set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -fno-pie") diff --git a/base/poco/Crypto/CMakeLists.txt b/base/poco/Crypto/CMakeLists.txt index d11a437fc02..67171afadfb 100644 --- a/base/poco/Crypto/CMakeLists.txt +++ b/base/poco/Crypto/CMakeLists.txt @@ -21,6 +21,7 @@ if (ENABLE_SSL) -Wno-unreachable-code-return -Wno-unused-parameter -Wno-zero-as-null-pointer-constant + -Wno-used-but-marked-unused ) target_include_directories (_poco_crypto SYSTEM PUBLIC "include") target_link_libraries (_poco_crypto PUBLIC Poco::Foundation OpenSSL::SSL OpenSSL::Crypto) diff --git a/base/poco/Crypto/src/RSAKeyImpl.cpp b/base/poco/Crypto/src/RSAKeyImpl.cpp index 86089b828b1..eb6e758343a 100644 --- a/base/poco/Crypto/src/RSAKeyImpl.cpp +++ b/base/poco/Crypto/src/RSAKeyImpl.cpp @@ -102,7 +102,7 @@ RSAKeyImpl::RSAKeyImpl(const std::string& publicKeyFile, { BIO* bio = BIO_new(BIO_s_file()); if (!bio) throw Poco::IOException("Cannot create BIO for reading public key", publicKeyFile); - int rc = BIO_read_filename(bio, publicKeyFile.c_str()); + int rc = BIO_read_filename(bio, const_cast(publicKeyFile.c_str())); if (rc) { RSA* pubKey = PEM_read_bio_RSAPublicKey(bio, &_pRSA, 0, 0); @@ -132,7 +132,7 @@ RSAKeyImpl::RSAKeyImpl(const std::string& publicKeyFile, { BIO* bio = BIO_new(BIO_s_file()); if (!bio) throw Poco::IOException("Cannot create BIO for reading private key", privateKeyFile); - int rc = BIO_read_filename(bio, privateKeyFile.c_str()); + int rc = BIO_read_filename(bio, const_cast(privateKeyFile.c_str())); if (rc) { RSA* privKey = 0; diff --git a/base/poco/Crypto/src/X509Certificate.cpp b/base/poco/Crypto/src/X509Certificate.cpp index 8c2ba698d7b..16458d0477b 100644 --- a/base/poco/Crypto/src/X509Certificate.cpp +++ b/base/poco/Crypto/src/X509Certificate.cpp @@ -129,7 +129,7 @@ void X509Certificate::load(const std::string& path) BIO *pBIO = BIO_new(BIO_s_file()); if (!pBIO) throw Poco::IOException("Cannot create BIO for reading certificate file", path); - if (!BIO_read_filename(pBIO, path.c_str())) + if (!BIO_read_filename(pBIO, const_cast(path.c_str()))) { BIO_free(pBIO); throw Poco::OpenFileException("Cannot open certificate file for reading", path); diff --git a/cmake/target.cmake b/cmake/target.cmake index b7806319980..5ef45576fb7 100644 --- a/cmake/target.cmake +++ b/cmake/target.cmake @@ -33,6 +33,9 @@ if (CMAKE_CROSSCOMPILING) elseif (ARCH_PPC64LE) set (ENABLE_GRPC OFF CACHE INTERNAL "") set (ENABLE_SENTRY OFF CACHE INTERNAL "") + elseif (ARCH_S390X) + set (ENABLE_GRPC OFF CACHE INTERNAL "") + set (ENABLE_SENTRY OFF CACHE INTERNAL "") endif () elseif (OS_FREEBSD) # FIXME: broken dependencies From f78efe91f6b9165ef208310cd8a960984a5acc35 Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Fri, 17 Feb 2023 18:46:30 -0400 Subject: [PATCH 386/717] Update summingmergetree.md --- .../mergetree-family/summingmergetree.md | 55 +++++++++++++++++-- 1 file changed, 51 insertions(+), 4 deletions(-) diff --git a/docs/en/engines/table-engines/mergetree-family/summingmergetree.md b/docs/en/engines/table-engines/mergetree-family/summingmergetree.md index 2df3af42691..b2b6272c58e 100644 --- a/docs/en/engines/table-engines/mergetree-family/summingmergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/summingmergetree.md @@ -130,10 +130,57 @@ then this nested table is interpreted as a mapping of `key => (values...)`, and Examples: ``` text -[(1, 100)] + [(2, 150)] -> [(1, 100), (2, 150)] -[(1, 100)] + [(1, 150)] -> [(1, 250)] -[(1, 100)] + [(1, 150), (2, 150)] -> [(1, 250), (2, 150)] -[(1, 100), (2, 150)] + [(1, -100)] -> [(2, 150)] +DROP TABLE IF EXISTS nested_sum; +CREATE TABLE nested_sum +( + date Date, + site UInt32, + hitsMap Nested( + browser String, + imps UInt32, + clicks UInt32 + ) +) ENGINE = SummingMergeTree +PRIMARY KEY (date, site); + +INSERT INTO nested_sum VALUES ('2020-01-01', 12, ['Firefox', 'Opera'], [10, 5], [2, 1]); +INSERT INTO nested_sum VALUES ('2020-01-01', 12, ['Chrome', 'Firefox'], [20, 1], [1, 1]); +INSERT INTO nested_sum VALUES ('2020-01-01', 12, ['IE'], [22], [0]); +INSERT INTO nested_sum VALUES ('2020-01-01', 10, ['Chrome'], [4], [3]); + +OPTIMIZE TABLE nested_sum FINAL; -- emulate merge + +SELECT * FROM nested_sum; +┌───────date─┬─site─┬─hitsMap.browser───────────────────┬─hitsMap.imps─┬─hitsMap.clicks─┐ +│ 2020-01-01 │ 10 │ ['Chrome'] │ [4] │ [3] │ +│ 2020-01-01 │ 12 │ ['Chrome','Firefox','IE','Opera'] │ [20,11,22,5] │ [1,3,0,1] │ +└────────────┴──────┴───────────────────────────────────┴──────────────┴────────────────┘ + +SELECT + site, + browser, + impressions, + clicks +FROM +( + SELECT + site, + sumMap(hitsMap.browser, hitsMap.imps, hitsMap.clicks) AS imps_map + FROM nested_sum + GROUP BY site +) +ARRAY JOIN + imps_map.1 AS browser, + imps_map.2 AS impressions, + imps_map.3 AS clicks; + +┌─site─┬─browser─┬─impressions─┬─clicks─┐ +│ 12 │ Chrome │ 20 │ 1 │ +│ 12 │ Firefox │ 11 │ 3 │ +│ 12 │ IE │ 22 │ 0 │ +│ 12 │ Opera │ 5 │ 1 │ +│ 10 │ Chrome │ 4 │ 3 │ +└──────┴─────────┴─────────────┴────────┘ ``` When requesting data, use the [sumMap(key, value)](../../../sql-reference/aggregate-functions/reference/summap.md) function for aggregation of `Map`. From b76df1401ec678f47fb046b2f805fc71ce1e698d Mon Sep 17 00:00:00 2001 From: Han Fei Date: Sat, 18 Feb 2023 01:08:28 +0100 Subject: [PATCH 387/717] enable async-insert-max-query-number only if async_insert_deduplicate is true --- src/Interpreters/AsynchronousInsertQueue.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/AsynchronousInsertQueue.cpp b/src/Interpreters/AsynchronousInsertQueue.cpp index fa3e9915e8f..293849b88b7 100644 --- a/src/Interpreters/AsynchronousInsertQueue.cpp +++ b/src/Interpreters/AsynchronousInsertQueue.cpp @@ -246,7 +246,7 @@ std::future AsynchronousInsertQueue::push(ASTPtr query, ContextPtr query_c /// And use setting from query context. /// It works, because queries with the same set of settings are already grouped together. if (data->size_in_bytes >= key.settings.async_insert_max_data_size - || data->query_number >= key.settings.async_insert_max_query_number) + || (data->query_number >= key.settings.async_insert_max_query_number && key.settings.async_insert_deduplicate)) { data_to_process = std::move(data); shard.iterators.erase(it); From f9cd66db363f52473ad962c2c534800d61c1897d Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 18 Feb 2023 10:53:47 +0100 Subject: [PATCH 388/717] Remove extra try/catch for LocalQueryState reset Signed-off-by: Azat Khuzhin --- src/Client/LocalConnection.cpp | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/Client/LocalConnection.cpp b/src/Client/LocalConnection.cpp index 77a0a846300..712ff5f5a31 100644 --- a/src/Client/LocalConnection.cpp +++ b/src/Client/LocalConnection.cpp @@ -34,14 +34,7 @@ LocalConnection::LocalConnection(ContextPtr context_, bool send_progress_, bool LocalConnection::~LocalConnection() { - try - { - state.reset(); - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__); - } + state.reset(); } bool LocalConnection::hasReadPendingData() const From 6f7b6e92066ac1fb31e49ef9742db2986e0dab0b Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 18 Feb 2023 10:54:22 +0100 Subject: [PATCH 389/717] Remove extra try/catch for QueryState reset It was possible back when destuctors may throw, in a pre C++11. I've found one place where it may throw from 2012: ~UnionBlockInputStream() { ... if (exception && !std::uncaught_exception()) exception->rethrow(); ... } Refs: 8a053aba54c Signed-off-by: Azat Khuzhin --- src/Server/TCPHandler.cpp | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 84587af47bb..f76c342fa9a 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -601,23 +601,12 @@ void TCPHandler::runImpl() LOG_DEBUG(log, "Processed in {} sec.", state.watch.elapsedSeconds()); } - try - { - /// QueryState should be cleared before QueryScope, since otherwise - /// the MemoryTracker will be wrong for possible deallocations. - /// (i.e. deallocations from the Aggregator with two-level aggregation) - state.reset(); - query_scope.reset(); - thread_trace_context.reset(); - } - catch (...) - { - /** During the processing of request, there was an exception that we caught and possibly sent to client. - * When destroying the request pipeline execution there was a second exception. - * For example, a pipeline could run in multiple threads, and an exception could occur in each of them. - * Ignore it. - */ - } + /// QueryState should be cleared before QueryScope, since otherwise + /// the MemoryTracker will be wrong for possible deallocations. + /// (i.e. deallocations from the Aggregator with two-level aggregation) + state.reset(); + query_scope.reset(); + thread_trace_context.reset(); /// It is important to destroy query context here. We do not want it to live arbitrarily longer than the query. query_context.reset(); From fb35a6852ea55a693250449dcc5a07fb30ac4c2e Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 18 Feb 2023 11:07:04 +0100 Subject: [PATCH 390/717] More flakiness fiexes for test_backup_restore_on_cluster/test_disallow_concurrency Signed-off-by: Azat Khuzhin --- .../test_disallow_concurrency.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py b/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py index d689e44e4d4..ad16725e266 100644 --- a/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py +++ b/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py @@ -186,10 +186,14 @@ def test_concurrent_restores_on_same_node(): ) nodes[0].query(f"DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY") - nodes[0].query(f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_name} ASYNC") + restore_id = ( + nodes[0] + .query(f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_name} ASYNC") + .split("\t")[0] + ) assert_eq_with_retry( nodes[0], - f"SELECT status FROM system.backups WHERE status == 'RESTORING'", + f"SELECT status FROM system.backups WHERE status == 'RESTORING' AND id == '{restore_id}'", "RESTORING", ) assert "Concurrent restores not supported" in nodes[0].query_and_get_error( @@ -220,7 +224,11 @@ def test_concurrent_restores_on_different_node(): ) nodes[0].query(f"DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY") - nodes[0].query(f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_name} ASYNC") + restore_id = ( + nodes[0] + .query(f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_name} ASYNC") + .split("\t")[0] + ) assert_eq_with_retry( nodes[0], f"SELECT status FROM system.backups WHERE status == 'RESTORING'", @@ -232,6 +240,6 @@ def test_concurrent_restores_on_different_node(): assert_eq_with_retry( nodes[0], - f"SELECT status FROM system.backups WHERE status == 'RESTORED'", + f"SELECT status FROM system.backups WHERE status == 'RESTORED' AND id == '{restore_id}'", "RESTORED", ) From 19e0b26a7c66ab9db29c70627470efdc32838dda Mon Sep 17 00:00:00 2001 From: Han Fei Date: Sat, 18 Feb 2023 11:38:09 +0100 Subject: [PATCH 391/717] fix --- tests/queries/0_stateless/02015_async_inserts_2.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02015_async_inserts_2.sh b/tests/queries/0_stateless/02015_async_inserts_2.sh index 8934dcc66e0..48523ccd9a9 100755 --- a/tests/queries/0_stateless/02015_async_inserts_2.sh +++ b/tests/queries/0_stateless/02015_async_inserts_2.sh @@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -url="${CLICKHOUSE_URL}&async_insert=1&wait_for_async_insert=1&async_insert_busy_timeout_ms=600000&async_insert_max_query_number=3" +url="${CLICKHOUSE_URL}&async_insert=1&wait_for_async_insert=1&async_insert_busy_timeout_ms=600000&async_insert_max_query_number=3&async_insert_deduplicate=1" ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS async_inserts" ${CLICKHOUSE_CLIENT} -q "CREATE TABLE async_inserts (id UInt32, s String) ENGINE = MergeTree ORDER BY id" From bb001078c9a922fbf370462d9f68706d7998f284 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 18 Feb 2023 14:01:47 +0300 Subject: [PATCH 392/717] Update sampling-query-profiler.md --- .../optimizing-performance/sampling-query-profiler.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/en/operations/optimizing-performance/sampling-query-profiler.md b/docs/en/operations/optimizing-performance/sampling-query-profiler.md index 7c63d4a9174..72eb655101f 100644 --- a/docs/en/operations/optimizing-performance/sampling-query-profiler.md +++ b/docs/en/operations/optimizing-performance/sampling-query-profiler.md @@ -60,7 +60,3 @@ GROUP BY trace ORDER BY count() DESC LIMIT 10 ``` - -``` text -{% include "examples/sampling_query_profiler_result.txt" %} -``` From ee7eb8a7dcffef7f0380fb60026ffb81ffc5aedc Mon Sep 17 00:00:00 2001 From: kssenii Date: Sat, 18 Feb 2023 12:28:55 +0100 Subject: [PATCH 393/717] Fix test --- .../test.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/tests/integration/test_postgresql_replica_database_engine_1/test.py b/tests/integration/test_postgresql_replica_database_engine_1/test.py index 019617cdc39..c7a503f8f16 100644 --- a/tests/integration/test_postgresql_replica_database_engine_1/test.py +++ b/tests/integration/test_postgresql_replica_database_engine_1/test.py @@ -581,18 +581,6 @@ def test_virtual_columns(started_cluster): ) print(result) - cursor.execute(f"ALTER TABLE {table_name} ADD COLUMN value2 integer") - instance.query( - f"INSERT INTO postgres_database.{table_name} SELECT number, number, number from numbers(10, 10)" - ) - assert_number_of_columns(instance, 3, table_name) - check_tables_are_synchronized(instance, table_name) - - result = instance.query( - "SELECT key, value, value2, _sign, _version FROM test_database.postgresql_replica_0;" - ) - print(result) - instance.query( f"INSERT INTO postgres_database.{table_name} SELECT number, number, number from numbers(20, 10)" ) From c2f5331a2deb291925521d64a3382937b5a9ab37 Mon Sep 17 00:00:00 2001 From: Joanna Hulboj Date: Sat, 18 Feb 2023 12:54:52 +0000 Subject: [PATCH 394/717] Revert "FIXUP: Into FunctionArrayMapped: Simplify the dispatch in case of positional params" This reverts commit d48bdcc7549af1aedd5e8bb4915d8bad56929aef. --- src/Common/tuple.h | 19 -------- src/Functions/array/FunctionArrayMapped.h | 54 +++++++++++------------ src/Functions/array/arraySort.cpp | 22 ++++----- 3 files changed, 35 insertions(+), 60 deletions(-) delete mode 100644 src/Common/tuple.h diff --git a/src/Common/tuple.h b/src/Common/tuple.h deleted file mode 100644 index f17de09ee25..00000000000 --- a/src/Common/tuple.h +++ /dev/null @@ -1,19 +0,0 @@ -#pragma once - -#include -#include - -namespace detail -{ -template -auto to_tuple_impl(C && c, std::index_sequence) -{ - return std::tie(c[Is]...); -} -} - -template -auto to_tuple(C && c) -{ - return detail::to_tuple_impl(c, std::make_index_sequence()); -} diff --git a/src/Functions/array/FunctionArrayMapped.h b/src/Functions/array/FunctionArrayMapped.h index 7a201df4638..42b7ae69a40 100644 --- a/src/Functions/array/FunctionArrayMapped.h +++ b/src/Functions/array/FunctionArrayMapped.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -10,7 +11,6 @@ #include #include -#include #include #include @@ -79,7 +79,6 @@ const IColumn::Offsets & getOffsets(const T & column) * It is possible for the functions to require fixed number of positional arguments: * arrayPartialSort(limit, arr) * arrayPartialSort(x -> predicate, limit, arr) - * See the arraySort.cpp for details. * * For some functions arrayCount, arrayExists, arrayAll, an overload of the form f(array) is available, * which works in the same way as f(x -> x, array). @@ -195,13 +194,8 @@ public: arguments[num_fixed_params].type->getName()); if constexpr (num_fixed_params) - { - // Arguments are (f0, f1, ..., arr) - // Delegate checking of (f0, f1, ...) to Impl - std::apply( - [this](auto &&... args) { Impl::checkArguments(getName(), std::forward(args)...); }, - to_tuple(std::begin(arguments))); - } + Impl::checkArguments( + std::span(std::begin(arguments), num_fixed_params), getName()); DataTypePtr nested_type = data_type->getNestedType(); @@ -235,13 +229,8 @@ public: arguments[0].type->getName()); if constexpr (num_fixed_params) - { - // Arguments are (lambda, f0, f1, ..., arr0, arr1,...) - // Delegate checking of (f0, f1, ...) to Impl - std::apply( - [this](auto &&... args) { Impl::checkArguments(getName(), std::forward(args)...); }, - to_tuple(std::begin(arguments) + 1)); - } + Impl::checkArguments( + std::span(std::begin(arguments) + 1, num_fixed_params), getName()); /// The types of the remaining arguments are already checked in getLambdaArgumentTypes. @@ -301,17 +290,23 @@ public: if constexpr (std::is_same_v) { - // Invoke Impl::execute, add the fixed arguments as trailing ones if applicable - return std::apply( - [&](auto &&... args) { return Impl::execute(*column_array, column_array->getNestedColumn().getDataPtr(), std::forward(args)...); }, - to_tuple(std::begin(arguments))); + if constexpr (num_fixed_params) + return Impl::execute( + *column_array, + column_array->getNestedColumn().getDataPtr(), + std::span(std::begin(arguments), num_fixed_params)); + else + return Impl::execute(*column_array, column_array->getNestedColumn().getDataPtr()); } else { - // Invoke Impl::execute, add the fixed arguments as trailing ones if applicable - return std::apply( - [&](auto &&... args) { return Impl::execute(*column_array, column_array->getDataPtr(), std::forward(args)...); }, - to_tuple(std::begin(arguments))); + if constexpr (num_fixed_params) + return Impl::execute( + *column_array, + column_array->getDataPtr(), + std::span(std::begin(arguments), num_fixed_params)); + else + return Impl::execute(*column_array, column_array->getDataPtr()); } } else @@ -440,10 +435,13 @@ public: } } - // Invoke Impl::execute, add the fixed arguments as trailing ones if applicable - return std::apply( - [&](auto &&... args) { return Impl::execute(*column_first_array, lambda_result.column, std::forward(args)...); }, - to_tuple(std::begin(arguments) + 1)); + if constexpr (num_fixed_params) + return Impl::execute( + *column_first_array, + lambda_result.column, + std::span(std::begin(arguments) + 1, num_fixed_params)); + else + return Impl::execute(*column_first_array, lambda_result.column); } } }; diff --git a/src/Functions/array/arraySort.cpp b/src/Functions/array/arraySort.cpp index 93c0b294168..ebdc408dbbd 100644 --- a/src/Functions/array/arraySort.cpp +++ b/src/Functions/array/arraySort.cpp @@ -45,39 +45,35 @@ struct ArraySortImpl } }; - static void checkArguments(const String & name, const ColumnWithTypeAndName& arg) + static void checkArguments(std::span arguments, const String & name) requires(num_fixed_params) { - WhichDataType which(arg.type.get()); + if (arguments.size() != 1) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} needs limit argument", name); + + WhichDataType which(arguments[0].type.get()); if (!which.isUInt() && !which.isInt()) throw Exception( ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of limit argument of function {} (must be UInt or Int)", - arg.type->getName(), + arguments[0].type->getName(), name); } static ColumnPtr execute( const ColumnArray & array, ColumnPtr mapped, - const ColumnWithTypeAndName & limit_arg) + std::span arguments [[maybe_unused]] = {}) { - const auto limit = [&]() -> size_t + [[maybe_unused]] const auto limit = [&]() -> size_t { if constexpr (is_partial) { - return limit_arg.column.get()->getUInt(0); + return arguments[0].column.get()->getUInt(0); } return 0; }(); - return execute(array, mapped, limit); - } - static ColumnPtr execute( - const ColumnArray & array, - ColumnPtr mapped, - size_t limit [[maybe_unused]] = 0) - { const ColumnArray::Offsets & offsets = array.getOffsets(); size_t size = offsets.size(); From 30921b211b0e233a48f619c0994089ed9a3e9815 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 18 Feb 2023 16:49:55 +0300 Subject: [PATCH 395/717] Update wikistat.md --- .../example-datasets/wikistat.md | 62 ++++++++++++++----- 1 file changed, 48 insertions(+), 14 deletions(-) diff --git a/docs/en/getting-started/example-datasets/wikistat.md b/docs/en/getting-started/example-datasets/wikistat.md index 53f7cec2ef4..ea8a1e1847c 100644 --- a/docs/en/getting-started/example-datasets/wikistat.md +++ b/docs/en/getting-started/example-datasets/wikistat.md @@ -5,27 +5,61 @@ sidebar_label: WikiStat # WikiStat -See http://dumps.wikimedia.org/other/pagecounts-raw/ for details. +The dataset contains 0.5 trillion records. + +See the video from FOSDEM 2023: https://www.youtube.com/watch?v=JlcI2Vfz_uk +And the presentation: https://presentations.clickhouse.com/fosdem2023/ + +Data source: https://dumps.wikimedia.org/other/pageviews/ + +Getting the list of links: +``` +for i in {2015..2023}; do + for j in {01..12}; do + echo "${i}-${j}" >&2 + curl -sSL "https://dumps.wikimedia.org/other/pageviews/$i/$i-$j/" \ + | grep -oE 'pageviews-[0-9]+-[0-9]+\.gz' + done +done | sort | uniq | tee links.txt +``` + +Downloading the data: +``` +sed -r 's!pageviews-([0-9]{4})([0-9]{2})[0-9]{2}-[0-9]+\.gz!https://dumps.wikimedia.org/other/pageviews/\1/\1-\2/\0!' \ + links.txt | xargs -P3 wget --continue +``` + +(it will take about 3 days) Creating a table: ``` sql CREATE TABLE wikistat ( - date Date, - time DateTime, - project String, - subproject String, - path String, - hits UInt64, - size UInt64 -) ENGINE = MergeTree(date, (path, time), 8192); + time DateTime CODEC(Delta, ZSTD(3)), + project LowCardinality(String), + subproject LowCardinality(String), + path String CODEC(ZSTD(3)), + hits UInt64 CODEC(ZSTD(3)) +) +ENGINE = MergeTree +ORDER BY (path, time); ``` -Loading data: +Loading the data: -``` bash -$ for i in {2007..2016}; do for j in {01..12}; do echo $i-$j >&2; curl -sSL "http://dumps.wikimedia.org/other/pagecounts-raw/$i/$i-$j/" | grep -oE 'pagecounts-[0-9]+-[0-9]+\.gz'; done; done | sort | uniq | tee links.txt -$ cat links.txt | while read link; do wget http://dumps.wikimedia.org/other/pagecounts-raw/$(echo $link | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})[0-9]{2}-[0-9]+\.gz/\1/')/$(echo $link | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})[0-9]{2}-[0-9]+\.gz/\1-\2/')/$link; done -$ ls -1 /opt/wikistat/ | grep gz | while read i; do echo $i; gzip -cd /opt/wikistat/$i | ./wikistat-loader --time="$(echo -n $i | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})([0-9]{2})-([0-9]{2})([0-9]{2})([0-9]{2})\.gz/\1-\2-\3 \4-00-00/')" | clickhouse-client --query="INSERT INTO wikistat FORMAT TabSeparated"; done +``` +clickhouse-local --query " + WITH replaceRegexpOne(_path, '^.+pageviews-(\\d{4})(\\d{2})(\\d{2})-(\\d{2})(\\d{2})(\\d{2}).gz$', '\1-\2-\3 \4-\5-\6')::DateTime AS time, + extractGroups(line, '^([^ \\.]+)(\\.[^ ]+)? +([^ ]+) +(\\d+) +(\\d+)$') AS values + SELECT + time, + values[1] AS project, + values[2] AS subproject, + values[3] AS path, + (values[4])::UInt64 AS hits, + (values[5])::UInt64 AS size + FROM file('pageviews*.gz', LineAsString) + WHERE length(values) = 5 FORMAT Native +" | clickhouse-client --query "INSERT INTO wikistat FORMAT Native" ``` From 325b728bc991c42a4514108095136e9ea5aec7c9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 18 Feb 2023 16:50:14 +0300 Subject: [PATCH 396/717] Update wikistat.md --- docs/en/getting-started/example-datasets/wikistat.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/en/getting-started/example-datasets/wikistat.md b/docs/en/getting-started/example-datasets/wikistat.md index ea8a1e1847c..9d0760efe94 100644 --- a/docs/en/getting-started/example-datasets/wikistat.md +++ b/docs/en/getting-started/example-datasets/wikistat.md @@ -8,6 +8,7 @@ sidebar_label: WikiStat The dataset contains 0.5 trillion records. See the video from FOSDEM 2023: https://www.youtube.com/watch?v=JlcI2Vfz_uk + And the presentation: https://presentations.clickhouse.com/fosdem2023/ Data source: https://dumps.wikimedia.org/other/pageviews/ From ef61c221671832deb8635317faf57d406dbd8714 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Sat, 18 Feb 2023 14:26:24 +0000 Subject: [PATCH 397/717] another review fixes --- .../useDataParallelAggregation.cpp | 22 +++++++------------ .../Transforms/AggregatingTransform.cpp | 4 +++- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp b/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp index f39e3a511c6..13a749bd0b6 100644 --- a/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp +++ b/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp @@ -89,22 +89,16 @@ bool allOutputsDependsOnlyOnAllowedNodes( if (visited.contains(node)) return visited[node]; - auto has_match_in_group_by_actions_that_is_also_irreducible = [&irreducible_nodes, &matches, &node]() + bool res = false; + /// `matches` maps partition key nodes into nodes in group by actions + if (matches.contains(node)) { - /// `matches` maps partition key nodes into nodes in group by actions - if (matches.contains(node)) - { - if (const auto * node_in_gb_actions = matches.at(node).node; - /// Double-check is needed because function can be mapped into its arg (see matchTrees) - node_in_gb_actions && node_in_gb_actions->result_name == node->result_name) - { - return irreducible_nodes.contains(node_in_gb_actions); - } - } - return false; - }; + const auto & match = matches.at(node); + /// Function could be mapped into its argument. In this case .monotonicity != std::nullopt (see matchTrees) + if (match.node && match.node->result_name == node->result_name && !match.monotonicity) + res = irreducible_nodes.contains(match.node); + } - bool res = has_match_in_group_by_actions_that_is_also_irreducible(); if (!res) { switch (node->type) diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index e06c53d2175..69eeb76bf53 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -743,7 +743,9 @@ void AggregatingTransform::initGenerate() pipe.addSimpleTransform( [this](const Block & header) { - return std::make_shared(header, params->params.max_block_size, 1024 * 1024); + /// Just a reasonable constant, matches default value for the setting `preferred_block_size_bytes` + static constexpr size_t oneMB = 1024 * 1024; + return std::make_shared(header, params->params.max_block_size, oneMB); }); } /// AggregatingTransform::expandPipeline expects single output port. From 1f1cd1b22ce829447dd38637f88bf8fa3b7bd8d9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 18 Feb 2023 15:27:09 +0100 Subject: [PATCH 398/717] Whitespaces --- base/base/bit_cast.h | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/base/base/bit_cast.h b/base/base/bit_cast.h index 8198991e98e..5373ead36e8 100644 --- a/base/base/bit_cast.h +++ b/base/base/bit_cast.h @@ -12,21 +12,22 @@ template std::decay_t bit_cast(const From & from) { - /** - * Assume the source value is 0xAABBCCDD (i.e. sizeof(from) == 4). - * Its BE representation is 0xAABBCCDD, the LE representation is 0xDDCCBBAA. - * Further assume, sizeof(res) == 8 and that res is initially zeroed out. - * With LE, the result after bit_cast will be 0xDDCCBBAA00000000 --> input value == output value. - * With BE, the result after bit_cast will be 0x00000000AABBCCDD --> input value == output value. - */ + /** Assume the source value is 0xAABBCCDD (i.e. sizeof(from) == 4). + * Its BE representation is 0xAABBCCDD, the LE representation is 0xDDCCBBAA. + * Further assume, sizeof(res) == 8 and that res is initially zeroed out. + * With LE, the result after bit_cast will be 0xDDCCBBAA00000000 --> input value == output value. + * With BE, the result after bit_cast will be 0x00000000AABBCCDD --> input value == output value. + */ To res {}; if constexpr (std::endian::native == std::endian::little) - memcpy(static_cast(&res), &from, std::min(sizeof(res), sizeof(from))); + { + memcpy(static_cast(&res), &from, std::min(sizeof(res), sizeof(from))); + } else { - uint32_t offset_to = (sizeof(res) > sizeof(from)) ? (sizeof(res) - sizeof(from)) : 0; - uint32_t offset_from = (sizeof(from) > sizeof(res)) ? (sizeof(from) - sizeof(res)) : 0; - memcpy(reinterpret_cast(&res) + offset_to, reinterpret_cast(&from) + offset_from, std::min(sizeof(res), sizeof(from))); + uint32_t offset_to = (sizeof(res) > sizeof(from)) ? (sizeof(res) - sizeof(from)) : 0; + uint32_t offset_from = (sizeof(from) > sizeof(res)) ? (sizeof(from) - sizeof(res)) : 0; + memcpy(reinterpret_cast(&res) + offset_to, reinterpret_cast(&from) + offset_from, std::min(sizeof(res), sizeof(from))); } return res; } From eee39b7bd1a47b9d7487bfd6f5f1b5d803a5698d Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Sat, 18 Feb 2023 14:30:49 +0000 Subject: [PATCH 399/717] bring back lost lines --- src/Processors/QueryPlan/ReadFromMergeTree.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 7365b697b6e..25f0f3ee171 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -1507,6 +1507,8 @@ Pipe ReadFromMergeTree::spreadMarkRanges( std::vector add_columns = metadata_for_reading->getColumnsRequiredForSortingKey(); column_names_to_read.insert(column_names_to_read.end(), add_columns.begin(), add_columns.end()); + if (!data.merging_params.is_deleted_column.empty()) + column_names_to_read.push_back(data.merging_params.is_deleted_column); if (!data.merging_params.sign_column.empty()) column_names_to_read.push_back(data.merging_params.sign_column); if (!data.merging_params.version_column.empty()) From cbd961de98c2d788e14387c7ee25e9518580c3e3 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sat, 18 Feb 2023 17:06:00 +0100 Subject: [PATCH 400/717] Fixed code review issues --- src/Analyzer/AggregationUtils.cpp | 5 +- src/Analyzer/ConstantNode.cpp | 6 +- src/Analyzer/ConstantNode.h | 10 +- src/Analyzer/IQueryTreeNode.cpp | 10 + src/Analyzer/IQueryTreeNode.h | 2 +- src/Analyzer/Passes/QueryAnalysisPass.cpp | 246 +-------------- src/Analyzer/Utils.cpp | 64 +--- src/Analyzer/ValidationUtils.cpp | 282 ++++++++++++++++++ src/Analyzer/ValidationUtils.h | 29 ++ src/Analyzer/WindowFunctionsUtils.cpp | 5 +- .../InterpreterSelectQueryAnalyzer.cpp | 10 +- .../InterpreterSelectQueryAnalyzer.h | 6 +- src/Interpreters/MutationsInterpreter.cpp | 24 +- .../getHeaderForProcessingStage.cpp | 2 +- src/Planner/CollectColumnIdentifiers.cpp | 5 +- src/Planner/CollectSets.cpp | 5 +- src/Planner/Planner.cpp | 11 +- src/Planner/Planner.h | 2 +- src/Planner/PlannerJoinTree.cpp | 6 +- .../QueryPlanOptimizationSettings.h | 2 +- .../Optimizations/filterPushDown.cpp | 15 +- .../Transforms/buildPushingToViewsChain.cpp | 2 +- src/Storages/LiveView/StorageLiveView.cpp | 8 +- src/Storages/LiveView/StorageLiveView.h | 18 +- src/Storages/StorageDistributed.cpp | 4 +- .../TableFunctionExecutable.cpp | 5 +- ...479_analyzer_join_with_constants.reference | 5 - .../02479_analyzer_join_with_constants.sql | 2 +- ...yzer_optimize_grouping_sets_keys.reference | 36 +-- 29 files changed, 411 insertions(+), 416 deletions(-) create mode 100644 src/Analyzer/ValidationUtils.cpp create mode 100644 src/Analyzer/ValidationUtils.h diff --git a/src/Analyzer/AggregationUtils.cpp b/src/Analyzer/AggregationUtils.cpp index fd35d0cea84..0f134b00794 100644 --- a/src/Analyzer/AggregationUtils.cpp +++ b/src/Analyzer/AggregationUtils.cpp @@ -51,14 +51,11 @@ public: has_aggregate_functions = true; } - bool needChildVisit(const QueryTreeNodePtr & parent_node, const QueryTreeNodePtr & child_node) + bool needChildVisit(const QueryTreeNodePtr &, const QueryTreeNodePtr & child_node) { if (only_check && has_aggregate_functions) return false; - if (parent_node->getNodeType() == QueryTreeNodeType::CONSTANT) - return false; - auto child_node_type = child_node->getNodeType(); return !(child_node_type == QueryTreeNodeType::QUERY || child_node_type == QueryTreeNodeType::UNION); } diff --git a/src/Analyzer/ConstantNode.cpp b/src/Analyzer/ConstantNode.cpp index 1c55039abbc..79fc38cd617 100644 --- a/src/Analyzer/ConstantNode.cpp +++ b/src/Analyzer/ConstantNode.cpp @@ -17,12 +17,12 @@ namespace DB { -ConstantNode::ConstantNode(ConstantValuePtr constant_value_, QueryTreeNodePtr source_expression) +ConstantNode::ConstantNode(ConstantValuePtr constant_value_, QueryTreeNodePtr source_expression_) : IQueryTreeNode(children_size) , constant_value(std::move(constant_value_)) , value_string(applyVisitor(FieldVisitorToString(), constant_value->getValue())) { - children[source_child_index] = std::move(source_expression); + source_expression = std::move(source_expression_); } ConstantNode::ConstantNode(ConstantValuePtr constant_value_) @@ -72,7 +72,7 @@ void ConstantNode::updateTreeHashImpl(HashState & hash_state) const QueryTreeNodePtr ConstantNode::cloneImpl() const { - return std::make_shared(constant_value); + return std::make_shared(constant_value, source_expression); } ASTPtr ConstantNode::toASTImpl() const diff --git a/src/Analyzer/ConstantNode.h b/src/Analyzer/ConstantNode.h index 30e5c064167..6b58533a701 100644 --- a/src/Analyzer/ConstantNode.h +++ b/src/Analyzer/ConstantNode.h @@ -49,19 +49,19 @@ public: /// Returns true if constant node has source expression, false otherwise bool hasSourceExpression() const { - return children[source_child_index] != nullptr; + return source_expression != nullptr; } /// Get source expression const QueryTreeNodePtr & getSourceExpression() const { - return children[source_child_index]; + return source_expression; } /// Get source expression QueryTreeNodePtr & getSourceExpression() { - return children[source_child_index]; + return source_expression; } QueryTreeNodeType getNodeType() const override @@ -88,9 +88,9 @@ protected: private: ConstantValuePtr constant_value; String value_string; + QueryTreeNodePtr source_expression; - static constexpr size_t children_size = 1; - static constexpr size_t source_child_index = 0; + static constexpr size_t children_size = 0; }; } diff --git a/src/Analyzer/IQueryTreeNode.cpp b/src/Analyzer/IQueryTreeNode.cpp index a8759318fda..6b9b8b53d81 100644 --- a/src/Analyzer/IQueryTreeNode.cpp +++ b/src/Analyzer/IQueryTreeNode.cpp @@ -164,6 +164,16 @@ bool IQueryTreeNode::isEqual(const IQueryTreeNode & rhs, CompareOptions compare_ IQueryTreeNode::Hash IQueryTreeNode::getTreeHash() const { + /** Compute tree hash with this node as root. + * + * Some nodes can contain weak pointers to other nodes. Such weak nodes are not necessary + * part of tree that we try to hash, but we need to update hash state with their content. + * + * Algorithm + * For each node in tree we update hash state with their content. + * For weak nodes there is special handling. If we visit weak node first time we update hash state with weak node content and register + * identifier for this node, for subsequent visits of this weak node we hash weak node identifier instead of content. + */ HashState hash_state; std::unordered_map weak_node_to_identifier; diff --git a/src/Analyzer/IQueryTreeNode.h b/src/Analyzer/IQueryTreeNode.h index 748a947274e..157bbd1b951 100644 --- a/src/Analyzer/IQueryTreeNode.h +++ b/src/Analyzer/IQueryTreeNode.h @@ -100,7 +100,7 @@ public: * With default compare options aliases of query tree nodes are compared during isEqual call. * Original ASTs of query tree nodes are not compared during isEqual call. */ - bool isEqual(const IQueryTreeNode & rhs, CompareOptions compare_options = {true}) const; + bool isEqual(const IQueryTreeNode & rhs, CompareOptions compare_options = { .compare_aliases = true }) const; using Hash = std::pair; using HashState = SipHash; diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index f58e1c619db..45d8154f39b 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -52,6 +52,8 @@ #include #include #include +#include +#include #include #include #include @@ -71,7 +73,6 @@ #include #include #include -#include namespace ProfileEvents { @@ -99,7 +100,6 @@ namespace ErrorCodes extern const int EMPTY_LIST_OF_COLUMNS_QUERIED; extern const int TOO_DEEP_SUBQUERIES; extern const int UNKNOWN_AGGREGATE_FUNCTION; - extern const int NOT_AN_AGGREGATE; extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; extern const int TOO_MANY_ARGUMENTS_FOR_FUNCTION; extern const int ILLEGAL_FINAL; @@ -2991,13 +2991,7 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromJoin(const IdentifierLoo { resolved_identifier = resolved_identifier->clone(); auto & resolved_column = resolved_identifier->as(); - const auto & resolved_column_type = resolved_column.getColumnType(); - const auto & resolved_column_name = resolved_column.getColumnName(); - - auto to_nullable_function_resolver = FunctionFactory::instance().get("toNullable", scope.context); - auto to_nullable_function_arguments = {ColumnWithTypeAndName(nullptr, resolved_column_type, resolved_column_name)}; - auto to_nullable_function = to_nullable_function_resolver->build(to_nullable_function_arguments); - resolved_column.setColumnType(to_nullable_function->getResultType()); + resolved_column.setColumnType(makeNullableSafe(resolved_column.getColumnType())); } return resolved_identifier; @@ -5891,8 +5885,7 @@ void QueryAnalyzer::resolveTableFunction(QueryTreeNodePtr & table_function_node, continue; } - - if (auto * table_function_argument_function = table_function_argument->as()) + else if (auto * table_function_argument_function = table_function_argument->as()) { const auto & table_function_argument_function_name = table_function_argument_function->getFunctionName(); if (TableFunctionFactory::instance().isTableFunctionName(table_function_argument_function_name)) @@ -5912,8 +5905,8 @@ void QueryAnalyzer::resolveTableFunction(QueryTreeNodePtr & table_function_node, * We cannot skip analysis for such arguments, because some table functions cannot provide * information if analysis for argument should be skipped until other arguments will be resolved. * - * Example: SELECT key from remote('127.0.0.{1,2}', view(select number AS key from numbers(2)), key); - * Example: SELECT id from remote('127.0.0.{1,2}', 'default', 'test_table', id); + * Example: SELECT key from remote('127.0.0.{1,2}', view(select number AS key from numbers(2)), cityHash64(key)); + * Example: SELECT id from remote('127.0.0.{1,2}', 'default', 'test_table', cityHash64(id)); */ try { @@ -6240,7 +6233,7 @@ void QueryAnalyzer::resolveQueryJoinTreeNode(QueryTreeNodePtr & join_tree_node, auto [it, inserted] = scope.alias_name_to_table_expression_node.emplace(alias_name, table_expression_node); if (!inserted) - throw Exception(ErrorCodes::UNSUPPORTED_METHOD, + throw Exception(ErrorCodes::MULTIPLE_EXPRESSIONS_FOR_ALIAS, "Duplicate aliases {} for table expressions in FROM section are not allowed. Try to register {}. Already registered {}.", alias_name, table_expression_node->formatASTForErrorMessage(), @@ -6251,108 +6244,6 @@ void QueryAnalyzer::resolveQueryJoinTreeNode(QueryTreeNodePtr & join_tree_node, scope.table_expressions_in_resolve_process.erase(join_tree_node.get()); } -class ValidateGroupByColumnsVisitor : public ConstInDepthQueryTreeVisitor -{ -public: - ValidateGroupByColumnsVisitor(const QueryTreeNodes & group_by_keys_nodes_, const IdentifierResolveScope & scope_) - : group_by_keys_nodes(group_by_keys_nodes_) - , scope(scope_) - {} - - void visitImpl(const QueryTreeNodePtr & node) - { - auto query_tree_node_type = node->getNodeType(); - if (query_tree_node_type == QueryTreeNodeType::CONSTANT || - query_tree_node_type == QueryTreeNodeType::SORT || - query_tree_node_type == QueryTreeNodeType::INTERPOLATE) - return; - - if (nodeIsAggregateFunctionOrInGroupByKeys(node)) - return; - - auto * function_node = node->as(); - if (function_node && function_node->getFunctionName() == "grouping") - { - auto & grouping_function_arguments_nodes = function_node->getArguments().getNodes(); - for (auto & grouping_function_arguments_node : grouping_function_arguments_nodes) - { - bool found_argument_in_group_by_keys = false; - - for (const auto & group_by_key_node : group_by_keys_nodes) - { - if (grouping_function_arguments_node->isEqual(*group_by_key_node)) - { - found_argument_in_group_by_keys = true; - break; - } - } - - if (!found_argument_in_group_by_keys) - throw Exception(ErrorCodes::NOT_AN_AGGREGATE, - "GROUPING function argument {} is not in GROUP BY. In scope {}", - grouping_function_arguments_node->formatASTForErrorMessage(), - scope.scope_node->formatASTForErrorMessage()); - } - - return; - } - - auto * column_node = node->as(); - if (!column_node) - return; - - auto column_node_source = column_node->getColumnSource(); - if (column_node_source->getNodeType() == QueryTreeNodeType::LAMBDA) - return; - - std::string column_name; - - if (column_node_source->hasAlias()) - column_name = column_node_source->getAlias(); - else if (auto * table_node = column_node_source->as()) - column_name = table_node->getStorageID().getFullTableName(); - - if (!column_name.empty()) - column_name += '.'; - - column_name += column_node->getColumnName(); - - throw Exception(ErrorCodes::NOT_AN_AGGREGATE, - "Column {} is not under aggregate function and not in GROUP BY. In scope {}", - column_name, - scope.scope_node->formatASTForErrorMessage()); - } - - bool needChildVisit(const QueryTreeNodePtr & parent_node, const QueryTreeNodePtr & child_node) - { - if (nodeIsAggregateFunctionOrInGroupByKeys(parent_node)) - return false; - - if (parent_node->getNodeType() == QueryTreeNodeType::CONSTANT) - return false; - - auto child_node_type = child_node->getNodeType(); - return !(child_node_type == QueryTreeNodeType::QUERY || child_node_type == QueryTreeNodeType::UNION); - } - -private: - bool nodeIsAggregateFunctionOrInGroupByKeys(const QueryTreeNodePtr & node) const - { - if (auto * function_node = node->as()) - if (function_node->isAggregateFunction()) - return true; - - for (const auto & group_by_key_node : group_by_keys_nodes) - if (node->isEqual(*group_by_key_node, {.compare_aliases = false})) - return true; - - return false; - } - - const QueryTreeNodes & group_by_keys_nodes; - const IdentifierResolveScope & scope; -}; - /** Resolve query. * This function modifies query node during resolve. It is caller responsibility to clone query node before resolve * if it is needed for later use. @@ -6674,128 +6565,7 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier "ARRAY JOIN", "in PREWHERE"); - /** Validate aggregates - * - * 1. Check that there are no aggregate functions and GROUPING function in JOIN TREE, WHERE, PREWHERE, in another aggregate functions. - * 2. Check that there are no window functions in JOIN TREE, WHERE, PREWHERE, HAVING, WINDOW, inside another aggregate function, - * inside window function arguments, inside window function window definition. - * 3. Check that there are no columns that are not specified in GROUP BY keys. - * 4. Validate GROUP BY modifiers. - */ - auto join_tree_node_type = query_node_typed.getJoinTree()->getNodeType(); - bool join_tree_is_subquery = join_tree_node_type == QueryTreeNodeType::QUERY || join_tree_node_type == QueryTreeNodeType::UNION; - - if (!join_tree_is_subquery) - { - assertNoAggregateFunctionNodes(query_node_typed.getJoinTree(), "in JOIN TREE"); - assertNoGroupingFunctionNodes(query_node_typed.getJoinTree(), "in JOIN TREE"); - assertNoWindowFunctionNodes(query_node_typed.getJoinTree(), "in JOIN TREE"); - } - - if (query_node_typed.hasWhere()) - { - assertNoAggregateFunctionNodes(query_node_typed.getWhere(), "in WHERE"); - assertNoGroupingFunctionNodes(query_node_typed.getWhere(), "in WHERE"); - assertNoWindowFunctionNodes(query_node_typed.getWhere(), "in WHERE"); - } - - if (query_node_typed.hasPrewhere()) - { - assertNoAggregateFunctionNodes(query_node_typed.getPrewhere(), "in PREWHERE"); - assertNoGroupingFunctionNodes(query_node_typed.getPrewhere(), "in PREWHERE"); - assertNoWindowFunctionNodes(query_node_typed.getPrewhere(), "in PREWHERE"); - } - - if (query_node_typed.hasHaving()) - assertNoWindowFunctionNodes(query_node_typed.getHaving(), "in HAVING"); - - if (query_node_typed.hasWindow()) - assertNoWindowFunctionNodes(query_node_typed.getWindowNode(), "in WINDOW"); - - QueryTreeNodes aggregate_function_nodes; - QueryTreeNodes window_function_nodes; - - collectAggregateFunctionNodes(query_node, aggregate_function_nodes); - collectWindowFunctionNodes(query_node, window_function_nodes); - - if (query_node_typed.hasGroupBy()) - { - assertNoAggregateFunctionNodes(query_node_typed.getGroupByNode(), "in GROUP BY"); - assertNoGroupingFunctionNodes(query_node_typed.getGroupByNode(), "in GROUP BY"); - assertNoWindowFunctionNodes(query_node_typed.getGroupByNode(), "in GROUP BY"); - } - - for (auto & aggregate_function_node : aggregate_function_nodes) - { - auto & aggregate_function_node_typed = aggregate_function_node->as(); - - assertNoAggregateFunctionNodes(aggregate_function_node_typed.getArgumentsNode(), "inside another aggregate function"); - assertNoGroupingFunctionNodes(aggregate_function_node_typed.getArgumentsNode(), "inside another aggregate function"); - assertNoWindowFunctionNodes(aggregate_function_node_typed.getArgumentsNode(), "inside an aggregate function"); - } - - for (auto & window_function_node : window_function_nodes) - { - auto & window_function_node_typed = window_function_node->as(); - assertNoWindowFunctionNodes(window_function_node_typed.getArgumentsNode(), "inside another window function"); - - if (query_node_typed.hasWindow()) - assertNoWindowFunctionNodes(window_function_node_typed.getWindowNode(), "inside window definition"); - } - - QueryTreeNodes group_by_keys_nodes; - group_by_keys_nodes.reserve(query_node_typed.getGroupBy().getNodes().size()); - - for (auto & node : query_node_typed.getGroupBy().getNodes()) - { - if (query_node_typed.isGroupByWithGroupingSets()) - { - auto & grouping_set_keys = node->as(); - for (auto & grouping_set_key : grouping_set_keys.getNodes()) - { - if (grouping_set_key->as()) - continue; - - group_by_keys_nodes.push_back(grouping_set_key); - } - } - else - { - if (node->as()) - continue; - - group_by_keys_nodes.push_back(node); - } - } - - if (query_node_typed.getGroupBy().getNodes().empty()) - { - if (query_node_typed.hasHaving()) - assertNoGroupingFunctionNodes(query_node_typed.getHaving(), "in HAVING without GROUP BY"); - - if (query_node_typed.hasOrderBy()) - assertNoGroupingFunctionNodes(query_node_typed.getOrderByNode(), "in ORDER BY without GROUP BY"); - - assertNoGroupingFunctionNodes(query_node_typed.getProjectionNode(), "in SELECT without GROUP BY"); - } - - bool has_aggregation = !query_node_typed.getGroupBy().getNodes().empty() || !aggregate_function_nodes.empty(); - - if (has_aggregation) - { - ValidateGroupByColumnsVisitor validate_group_by_columns_visitor(group_by_keys_nodes, scope); - - if (query_node_typed.hasHaving()) - validate_group_by_columns_visitor.visit(query_node_typed.getHaving()); - - if (query_node_typed.hasOrderBy()) - validate_group_by_columns_visitor.visit(query_node_typed.getOrderByNode()); - - validate_group_by_columns_visitor.visit(query_node_typed.getProjectionNode()); - } - - if (!has_aggregation && (query_node_typed.isGroupByWithGroupingSets() || is_rollup_or_cube || query_node_typed.isGroupByWithTotals())) - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "WITH TOTALS, ROLLUP, CUBE or GROUPING SETS are not supported without aggregation"); + validateAggregates(query_node); /** WITH section can be safely removed, because WITH section only can provide aliases to query expressions * and CTE for other sections to use. diff --git a/src/Analyzer/Utils.cpp b/src/Analyzer/Utils.cpp index 5356ba308e4..f638ff9c5ce 100644 --- a/src/Analyzer/Utils.cpp +++ b/src/Analyzer/Utils.cpp @@ -253,7 +253,7 @@ QueryTreeNodePtr extractLeftTableExpression(const QueryTreeNodePtr & join_tree_n std::deque nodes_to_process; nodes_to_process.push_back(join_tree_node); - while (!nodes_to_process.empty()) + while (!result) { auto node_to_process = std::move(nodes_to_process.front()); nodes_to_process.pop_front(); @@ -379,58 +379,6 @@ bool nestedIdentifierCanBeResolved(const DataTypePtr & compound_type, Identifier namespace { -class ValidateFunctionNodesVisitor : public ConstInDepthQueryTreeVisitor -{ -public: - explicit ValidateFunctionNodesVisitor(std::string_view function_name_, - int exception_code_, - std::string_view exception_function_name_, - std::string_view exception_place_message_) - : function_name(function_name_) - , exception_code(exception_code_) - , exception_function_name(exception_function_name_) - , exception_place_message(exception_place_message_) - {} - - void visitImpl(const QueryTreeNodePtr & node) - { - auto * function_node = node->as(); - if (function_node && function_node->getFunctionName() == function_name) - throw Exception(exception_code, - "{} function {} is found {} in query", - exception_function_name, - function_node->formatASTForErrorMessage(), - exception_place_message); - } - - static bool needChildVisit(const QueryTreeNodePtr &, const QueryTreeNodePtr & child_node) - { - auto child_node_type = child_node->getNodeType(); - return !(child_node_type == QueryTreeNodeType::QUERY || child_node_type == QueryTreeNodeType::UNION); - } - -private: - std::string_view function_name; - int exception_code = 0; - std::string_view exception_function_name; - std::string_view exception_place_message; -}; - -} - -void assertNoFunctionNodes(const QueryTreeNodePtr & node, - std::string_view function_name, - int exception_code, - std::string_view exception_function_name, - std::string_view exception_place_message) -{ - ValidateFunctionNodesVisitor visitor(function_name, exception_code, exception_function_name, exception_place_message); - visitor.visit(node); -} - -namespace -{ - class CheckFunctionExistsVisitor : public ConstInDepthQueryTreeVisitor { public: @@ -450,14 +398,11 @@ public: has_function = function_node->getFunctionName() == function_name; } - bool needChildVisit(const QueryTreeNodePtr & parent_node, const QueryTreeNodePtr & child_node) + bool needChildVisit(const QueryTreeNodePtr &, const QueryTreeNodePtr & child_node) { if (has_function) return false; - if (parent_node->getNodeType() == QueryTreeNodeType::CONSTANT) - return false; - auto child_node_type = child_node->getNodeType(); return !(child_node_type == QueryTreeNodeType::QUERY || child_node_type == QueryTreeNodeType::UNION); } @@ -511,11 +456,8 @@ public: node = node_it->second; } - bool needChildVisit(const QueryTreeNodePtr & parent_node, const QueryTreeNodePtr & child_node) + bool needChildVisit(const QueryTreeNodePtr &, const QueryTreeNodePtr & child_node) { - if (parent_node->getNodeType() == QueryTreeNodeType::CONSTANT) - return false; - auto child_node_type = child_node->getNodeType(); return !(child_node_type == QueryTreeNodeType::QUERY || child_node_type == QueryTreeNodeType::UNION); } diff --git a/src/Analyzer/ValidationUtils.cpp b/src/Analyzer/ValidationUtils.cpp new file mode 100644 index 00000000000..feb2f8a890b --- /dev/null +++ b/src/Analyzer/ValidationUtils.cpp @@ -0,0 +1,282 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int NOT_AN_AGGREGATE; + extern const int NOT_IMPLEMENTED; +} + +class ValidateGroupByColumnsVisitor : public ConstInDepthQueryTreeVisitor +{ +public: + explicit ValidateGroupByColumnsVisitor(const QueryTreeNodes & group_by_keys_nodes_, const QueryTreeNodePtr & query_node_) + : group_by_keys_nodes(group_by_keys_nodes_) + , query_node(query_node_) + {} + + void visitImpl(const QueryTreeNodePtr & node) + { + auto query_tree_node_type = node->getNodeType(); + if (query_tree_node_type == QueryTreeNodeType::CONSTANT || + query_tree_node_type == QueryTreeNodeType::SORT || + query_tree_node_type == QueryTreeNodeType::INTERPOLATE) + return; + + if (nodeIsAggregateFunctionOrInGroupByKeys(node)) + return; + + auto * function_node = node->as(); + if (function_node && function_node->getFunctionName() == "grouping") + { + auto & grouping_function_arguments_nodes = function_node->getArguments().getNodes(); + for (auto & grouping_function_arguments_node : grouping_function_arguments_nodes) + { + bool found_argument_in_group_by_keys = false; + + for (const auto & group_by_key_node : group_by_keys_nodes) + { + if (grouping_function_arguments_node->isEqual(*group_by_key_node)) + { + found_argument_in_group_by_keys = true; + break; + } + } + + if (!found_argument_in_group_by_keys) + throw Exception(ErrorCodes::NOT_AN_AGGREGATE, + "GROUPING function argument {} is not in GROUP BY keys. In query {}", + grouping_function_arguments_node->formatASTForErrorMessage(), + query_node->formatASTForErrorMessage()); + } + + return; + } + + auto * column_node = node->as(); + if (!column_node) + return; + + auto column_node_source = column_node->getColumnSource(); + if (column_node_source->getNodeType() == QueryTreeNodeType::LAMBDA) + return; + + throw Exception(ErrorCodes::NOT_AN_AGGREGATE, + "Column {} is not under aggregate function and not in GROUP BY keys. In query {}", + column_node->formatConvertedASTForErrorMessage(), + query_node->formatASTForErrorMessage()); + } + + bool needChildVisit(const QueryTreeNodePtr & parent_node, const QueryTreeNodePtr & child_node) + { + if (nodeIsAggregateFunctionOrInGroupByKeys(parent_node)) + return false; + + auto child_node_type = child_node->getNodeType(); + return !(child_node_type == QueryTreeNodeType::QUERY || child_node_type == QueryTreeNodeType::UNION); + } + +private: + bool nodeIsAggregateFunctionOrInGroupByKeys(const QueryTreeNodePtr & node) const + { + if (auto * function_node = node->as()) + if (function_node->isAggregateFunction()) + return true; + + for (const auto & group_by_key_node : group_by_keys_nodes) + if (node->isEqual(*group_by_key_node, {.compare_aliases = false})) + return true; + + return false; + } + + const QueryTreeNodes & group_by_keys_nodes; + const QueryTreeNodePtr & query_node; +}; + +void validateAggregates(const QueryTreeNodePtr & query_node) +{ + const auto & query_node_typed = query_node->as(); + auto join_tree_node_type = query_node_typed.getJoinTree()->getNodeType(); + bool join_tree_is_subquery = join_tree_node_type == QueryTreeNodeType::QUERY || join_tree_node_type == QueryTreeNodeType::UNION; + + if (!join_tree_is_subquery) + { + assertNoAggregateFunctionNodes(query_node_typed.getJoinTree(), "in JOIN TREE"); + assertNoGroupingFunctionNodes(query_node_typed.getJoinTree(), "in JOIN TREE"); + assertNoWindowFunctionNodes(query_node_typed.getJoinTree(), "in JOIN TREE"); + } + + if (query_node_typed.hasWhere()) + { + assertNoAggregateFunctionNodes(query_node_typed.getWhere(), "in WHERE"); + assertNoGroupingFunctionNodes(query_node_typed.getWhere(), "in WHERE"); + assertNoWindowFunctionNodes(query_node_typed.getWhere(), "in WHERE"); + } + + if (query_node_typed.hasPrewhere()) + { + assertNoAggregateFunctionNodes(query_node_typed.getPrewhere(), "in PREWHERE"); + assertNoGroupingFunctionNodes(query_node_typed.getPrewhere(), "in PREWHERE"); + assertNoWindowFunctionNodes(query_node_typed.getPrewhere(), "in PREWHERE"); + } + + if (query_node_typed.hasHaving()) + assertNoWindowFunctionNodes(query_node_typed.getHaving(), "in HAVING"); + + if (query_node_typed.hasWindow()) + assertNoWindowFunctionNodes(query_node_typed.getWindowNode(), "in WINDOW"); + + QueryTreeNodes aggregate_function_nodes; + QueryTreeNodes window_function_nodes; + + collectAggregateFunctionNodes(query_node, aggregate_function_nodes); + collectWindowFunctionNodes(query_node, window_function_nodes); + + if (query_node_typed.hasGroupBy()) + { + assertNoAggregateFunctionNodes(query_node_typed.getGroupByNode(), "in GROUP BY"); + assertNoGroupingFunctionNodes(query_node_typed.getGroupByNode(), "in GROUP BY"); + assertNoWindowFunctionNodes(query_node_typed.getGroupByNode(), "in GROUP BY"); + } + + for (auto & aggregate_function_node : aggregate_function_nodes) + { + auto & aggregate_function_node_typed = aggregate_function_node->as(); + + assertNoAggregateFunctionNodes(aggregate_function_node_typed.getArgumentsNode(), "inside another aggregate function"); + assertNoGroupingFunctionNodes(aggregate_function_node_typed.getArgumentsNode(), "inside another aggregate function"); + assertNoWindowFunctionNodes(aggregate_function_node_typed.getArgumentsNode(), "inside an aggregate function"); + } + + for (auto & window_function_node : window_function_nodes) + { + auto & window_function_node_typed = window_function_node->as(); + assertNoWindowFunctionNodes(window_function_node_typed.getArgumentsNode(), "inside another window function"); + + if (query_node_typed.hasWindow()) + assertNoWindowFunctionNodes(window_function_node_typed.getWindowNode(), "inside window definition"); + } + + QueryTreeNodes group_by_keys_nodes; + group_by_keys_nodes.reserve(query_node_typed.getGroupBy().getNodes().size()); + + for (const auto & node : query_node_typed.getGroupBy().getNodes()) + { + if (query_node_typed.isGroupByWithGroupingSets()) + { + auto & grouping_set_keys = node->as(); + for (auto & grouping_set_key : grouping_set_keys.getNodes()) + { + if (grouping_set_key->as()) + continue; + + group_by_keys_nodes.push_back(grouping_set_key); + } + } + else + { + if (node->as()) + continue; + + group_by_keys_nodes.push_back(node); + } + } + + if (query_node_typed.getGroupBy().getNodes().empty()) + { + if (query_node_typed.hasHaving()) + assertNoGroupingFunctionNodes(query_node_typed.getHaving(), "in HAVING without GROUP BY"); + + if (query_node_typed.hasOrderBy()) + assertNoGroupingFunctionNodes(query_node_typed.getOrderByNode(), "in ORDER BY without GROUP BY"); + + assertNoGroupingFunctionNodes(query_node_typed.getProjectionNode(), "in SELECT without GROUP BY"); + } + + bool has_aggregation = !query_node_typed.getGroupBy().getNodes().empty() || !aggregate_function_nodes.empty(); + + if (has_aggregation) + { + ValidateGroupByColumnsVisitor validate_group_by_columns_visitor(group_by_keys_nodes, query_node); + + if (query_node_typed.hasHaving()) + validate_group_by_columns_visitor.visit(query_node_typed.getHaving()); + + if (query_node_typed.hasOrderBy()) + validate_group_by_columns_visitor.visit(query_node_typed.getOrderByNode()); + + validate_group_by_columns_visitor.visit(query_node_typed.getProjectionNode()); + } + + bool aggregation_with_rollup_or_cube_or_grouping_sets = query_node_typed.isGroupByWithRollup() || + query_node_typed.isGroupByWithCube() || + query_node_typed.isGroupByWithGroupingSets(); + if (!has_aggregation && (query_node_typed.isGroupByWithTotals() || aggregation_with_rollup_or_cube_or_grouping_sets)) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "WITH TOTALS, ROLLUP, CUBE or GROUPING SETS are not supported without aggregation"); +} + +namespace +{ + +class ValidateFunctionNodesVisitor : public ConstInDepthQueryTreeVisitor +{ +public: + explicit ValidateFunctionNodesVisitor(std::string_view function_name_, + int exception_code_, + std::string_view exception_function_name_, + std::string_view exception_place_message_) + : function_name(function_name_) + , exception_code(exception_code_) + , exception_function_name(exception_function_name_) + , exception_place_message(exception_place_message_) + {} + + void visitImpl(const QueryTreeNodePtr & node) + { + auto * function_node = node->as(); + if (function_node && function_node->getFunctionName() == function_name) + throw Exception(exception_code, + "{} function {} is found {} in query", + exception_function_name, + function_node->formatASTForErrorMessage(), + exception_place_message); + } + + static bool needChildVisit(const QueryTreeNodePtr &, const QueryTreeNodePtr & child_node) + { + auto child_node_type = child_node->getNodeType(); + return !(child_node_type == QueryTreeNodeType::QUERY || child_node_type == QueryTreeNodeType::UNION); + } + +private: + std::string_view function_name; + int exception_code = 0; + std::string_view exception_function_name; + std::string_view exception_place_message; +}; + +} + +void assertNoFunctionNodes(const QueryTreeNodePtr & node, + std::string_view function_name, + int exception_code, + std::string_view exception_function_name, + std::string_view exception_place_message) +{ + ValidateFunctionNodesVisitor visitor(function_name, exception_code, exception_function_name, exception_place_message); + visitor.visit(node); +} + +} diff --git a/src/Analyzer/ValidationUtils.h b/src/Analyzer/ValidationUtils.h new file mode 100644 index 00000000000..b511dc9514b --- /dev/null +++ b/src/Analyzer/ValidationUtils.h @@ -0,0 +1,29 @@ +#pragma once + +#include + +namespace DB +{ + +/** Validate aggregates in query node. + * + * 1. Check that there are no aggregate functions and GROUPING function in JOIN TREE, WHERE, PREWHERE, in another aggregate functions. + * 2. Check that there are no window functions in JOIN TREE, WHERE, PREWHERE, HAVING, WINDOW, inside another aggregate function, + * inside window function arguments, inside window function window definition. + * 3. Check that there are no columns that are not specified in GROUP BY keys in HAVING, ORDER BY, PROJECTION. + * 4. Check that there are no GROUPING functions that have arguments that are not specified in GROUP BY keys in HAVING, ORDER BY, + * PROJECTION. + * 5. Throws exception if there is GROUPING SETS or ROLLUP or CUBE or WITH TOTALS without aggregation. + */ +void validateAggregates(const QueryTreeNodePtr & query_node); + +/** Assert that there are no function nodes with specified function name in node children. + * Do not visit subqueries. + */ +void assertNoFunctionNodes(const QueryTreeNodePtr & node, + std::string_view function_name, + int exception_code, + std::string_view exception_function_name, + std::string_view exception_place_message); + +} diff --git a/src/Analyzer/WindowFunctionsUtils.cpp b/src/Analyzer/WindowFunctionsUtils.cpp index 200e0d098a9..10deec31cda 100644 --- a/src/Analyzer/WindowFunctionsUtils.cpp +++ b/src/Analyzer/WindowFunctionsUtils.cpp @@ -51,14 +51,11 @@ public: has_window_functions = true; } - bool needChildVisit(const QueryTreeNodePtr & parent_node, const QueryTreeNodePtr & child_node) + bool needChildVisit(const QueryTreeNodePtr &, const QueryTreeNodePtr & child_node) { if (only_check && has_window_functions) return false; - if (parent_node->getNodeType() == QueryTreeNodeType::CONSTANT) - return false; - auto child_node_type = child_node->getNodeType(); return !(child_node_type == QueryTreeNodeType::QUERY || child_node_type == QueryTreeNodeType::UNION); } diff --git a/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp b/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp index 1a83d93c87d..0536ee10f7c 100644 --- a/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp +++ b/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp @@ -163,8 +163,8 @@ InterpreterSelectQueryAnalyzer::InterpreterSelectQueryAnalyzer( InterpreterSelectQueryAnalyzer::InterpreterSelectQueryAnalyzer( const ASTPtr & query_, const ContextPtr & context_, - const SelectQueryOptions & select_query_options_, - const StoragePtr & storage_) + const StoragePtr & storage_, + const SelectQueryOptions & select_query_options_) : query(normalizeAndValidateQuery(query_)) , context(buildContext(context_, select_query_options_)) , select_query_options(select_query_options_) @@ -181,7 +181,7 @@ InterpreterSelectQueryAnalyzer::InterpreterSelectQueryAnalyzer( , context(buildContext(context_, select_query_options_)) , select_query_options(select_query_options_) , query_tree(query_tree_) - , planner(query_tree, select_query_options) + , planner(query_tree_, select_query_options_) { } @@ -196,13 +196,13 @@ Block InterpreterSelectQueryAnalyzer::getSampleBlock(const ASTPtr & query, return interpreter.getSampleBlock(); } -Block InterpreterSelectQueryAnalyzer::getSampleBlock(const QueryTreeNodePtr & query, +Block InterpreterSelectQueryAnalyzer::getSampleBlock(const QueryTreeNodePtr & query_tree, const ContextPtr & context, const SelectQueryOptions & select_query_options) { auto select_query_options_copy = select_query_options; select_query_options_copy.only_analyze = true; - InterpreterSelectQueryAnalyzer interpreter(query, context, select_query_options_copy); + InterpreterSelectQueryAnalyzer interpreter(query_tree, context, select_query_options); return interpreter.getSampleBlock(); } diff --git a/src/Interpreters/InterpreterSelectQueryAnalyzer.h b/src/Interpreters/InterpreterSelectQueryAnalyzer.h index f5151505d68..681a9cfe5a3 100644 --- a/src/Interpreters/InterpreterSelectQueryAnalyzer.h +++ b/src/Interpreters/InterpreterSelectQueryAnalyzer.h @@ -26,8 +26,8 @@ public: */ InterpreterSelectQueryAnalyzer(const ASTPtr & query_, const ContextPtr & context_, - const SelectQueryOptions & select_query_options_, - const StoragePtr & storage_); + const StoragePtr & storage_, + const SelectQueryOptions & select_query_options_); /// Initialize interpreter with query tree InterpreterSelectQueryAnalyzer(const QueryTreeNodePtr & query_tree_, @@ -45,7 +45,7 @@ public: const ContextPtr & context, const SelectQueryOptions & select_query_options = {}); - static Block getSampleBlock(const QueryTreeNodePtr & query_, + static Block getSampleBlock(const QueryTreeNodePtr & query_tree, const ContextPtr & context_, const SelectQueryOptions & select_query_options = {}); diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp index 0b52a1a51bc..034f327db91 100644 --- a/src/Interpreters/MutationsInterpreter.cpp +++ b/src/Interpreters/MutationsInterpreter.cpp @@ -172,21 +172,6 @@ ASTPtr prepareQueryAffectedAST(const std::vector & commands, co return select; } -QueryTreeNodePtr prepareQueryAffectedQueryTree(const std::vector & commands, const StoragePtr & storage, ContextPtr context) -{ - auto ast = prepareQueryAffectedAST(commands, storage, context); - auto query_tree = buildQueryTree(ast, context); - - auto & query_node = query_tree->as(); - query_node.getJoinTree() = std::make_shared(storage, context); - - QueryTreePassManager query_tree_pass_manager(context); - addQueryTreePasses(query_tree_pass_manager); - query_tree_pass_manager.run(query_tree); - - return query_tree; -} - ColumnDependencies getAllColumnDependencies(const StorageMetadataPtr & metadata_snapshot, const NameSet & updated_columns) { NameSet new_updated_columns = updated_columns; @@ -246,15 +231,18 @@ bool isStorageTouchedByMutations( std::optional interpreter_select_query; BlockIO io; + ASTPtr select_query = prepareQueryAffectedAST(commands, storage.shared_from_this(), context); + if (context->getSettingsRef().allow_experimental_analyzer) { - auto select_query_tree = prepareQueryAffectedQueryTree(commands, storage.shared_from_this(), context); - InterpreterSelectQueryAnalyzer interpreter(select_query_tree, context, SelectQueryOptions().ignoreLimits().ignoreProjections()); + InterpreterSelectQueryAnalyzer interpreter(select_query, + context, + storage_from_part, + SelectQueryOptions().ignoreLimits().ignoreProjections()); io = interpreter.execute(); } else { - ASTPtr select_query = prepareQueryAffectedAST(commands, storage.shared_from_this(), context); /// Interpreter must be alive, when we use result of execute() method. /// For some reason it may copy context and give it into ExpressionTransform /// after that we will use context from destroyed stack frame in our stream. diff --git a/src/Interpreters/getHeaderForProcessingStage.cpp b/src/Interpreters/getHeaderForProcessingStage.cpp index d725a06ca16..ab10bd2024c 100644 --- a/src/Interpreters/getHeaderForProcessingStage.cpp +++ b/src/Interpreters/getHeaderForProcessingStage.cpp @@ -175,7 +175,7 @@ Block getHeaderForProcessingStage( if (context->getSettingsRef().allow_experimental_analyzer) { auto storage = std::make_shared(storage_snapshot->storage.getStorageID(), storage_snapshot->metadata->getColumns()); - InterpreterSelectQueryAnalyzer interpreter(query, context, SelectQueryOptions(processed_stage).analyze(), storage); + InterpreterSelectQueryAnalyzer interpreter(query, context, storage, SelectQueryOptions(processed_stage).analyze()); result = interpreter.getSampleBlock(); } else diff --git a/src/Planner/CollectColumnIdentifiers.cpp b/src/Planner/CollectColumnIdentifiers.cpp index ee9b1bf06be..95f1c7d53d8 100644 --- a/src/Planner/CollectColumnIdentifiers.cpp +++ b/src/Planner/CollectColumnIdentifiers.cpp @@ -20,11 +20,8 @@ public: , planner_context(planner_context_) {} - static bool needChildVisit(const QueryTreeNodePtr & parent_node, const QueryTreeNodePtr & child) + static bool needChildVisit(const QueryTreeNodePtr &, const QueryTreeNodePtr & child) { - if (parent_node->getNodeType() == QueryTreeNodeType::CONSTANT) - return false; - auto child_node_type = child->getNodeType(); return child_node_type != QueryTreeNodeType::TABLE && child_node_type != QueryTreeNodeType::TABLE_FUNCTION diff --git a/src/Planner/CollectSets.cpp b/src/Planner/CollectSets.cpp index 2bc8eb8f69d..bc4b0dd09f3 100644 --- a/src/Planner/CollectSets.cpp +++ b/src/Planner/CollectSets.cpp @@ -81,11 +81,8 @@ public: } } - static bool needChildVisit(const QueryTreeNodePtr & parent_node, const QueryTreeNodePtr & child_node) + static bool needChildVisit(const QueryTreeNodePtr &, const QueryTreeNodePtr & child_node) { - if (parent_node->getNodeType() == QueryTreeNodeType::CONSTANT) - return false; - auto child_node_type = child_node->getNodeType(); return !(child_node_type == QueryTreeNodeType::QUERY || child_node_type == QueryTreeNodeType::UNION); } diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp index a4714c05a23..f0fe44e368f 100644 --- a/src/Planner/Planner.cpp +++ b/src/Planner/Planner.cpp @@ -188,7 +188,9 @@ void extendQueryContextAndStoragesLifetime(QueryPlan & query_plan, const Planner class QueryAnalysisResult { public: - QueryAnalysisResult(const QueryTreeNodePtr & query_tree, const PlannerQueryProcessingInfo & query_processing_info, const PlannerContextPtr & planner_context) + QueryAnalysisResult(const QueryTreeNodePtr & query_tree, + const PlannerQueryProcessingInfo & query_processing_info, + const PlannerContextPtr & planner_context) { const auto & query_node = query_tree->as(); const auto & query_context = planner_context->getQueryContext(); @@ -198,7 +200,8 @@ public: && settings.group_by_overflow_mode == OverflowMode::ANY && settings.totals_mode != TotalsMode::AFTER_HAVING_EXCLUSIVE; aggregate_final = query_processing_info.getToStage() > QueryProcessingStage::WithMergeableState && !query_node.isGroupByWithTotals() && !query_node.isGroupByWithRollup() && !query_node.isGroupByWithCube(); - aggregate_with_grouping_set = query_node.isGroupByWithRollup() || query_node.isGroupByWithCube() || query_node.isGroupByWithGroupingSets(); + aggregation_with_rollup_or_cube_or_grouping_sets = query_node.isGroupByWithRollup() || query_node.isGroupByWithCube() || + query_node.isGroupByWithGroupingSets(); aggregation_should_produce_results_in_order_of_bucket_number = query_processing_info.getToStage() == QueryProcessingStage::WithMergeableState && settings.distributed_aggregation_memory_efficient; @@ -233,7 +236,7 @@ public: bool aggregate_overflow_row = false; bool aggregate_final = false; - bool aggregate_with_grouping_set = false; + bool aggregation_with_rollup_or_cube_or_grouping_sets = false; bool aggregation_should_produce_results_in_order_of_bucket_number = false; bool query_has_array_join_in_join_tree = false; bool query_has_with_totals_in_any_subquery_in_join_tree = false; @@ -409,7 +412,7 @@ void addMergingAggregatedStep(QueryPlan & query_plan, params, query_analysis_result.aggregate_final, /// Grouping sets don't work with distributed_aggregation_memory_efficient enabled (#43989) - settings.distributed_aggregation_memory_efficient && is_remote_storage && !query_analysis_result.aggregate_with_grouping_set, + settings.distributed_aggregation_memory_efficient && is_remote_storage && !query_analysis_result.aggregation_with_rollup_or_cube_or_grouping_sets, settings.max_threads, settings.aggregation_memory_efficient_merge_threads, query_analysis_result.aggregation_should_produce_results_in_order_of_bucket_number, diff --git a/src/Planner/Planner.h b/src/Planner/Planner.h index 4427e8cc504..443dfa114ee 100644 --- a/src/Planner/Planner.h +++ b/src/Planner/Planner.h @@ -58,9 +58,9 @@ private: void buildPlanForQueryNode(); QueryTreeNodePtr query_tree; - QueryPlan query_plan; SelectQueryOptions select_query_options; PlannerContextPtr planner_context; + QueryPlan query_plan; StorageLimitsList storage_limits; }; diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index 0c74479615a..59b09f91888 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -679,6 +679,7 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_ for (auto & column_from_joined_table : columns_from_joined_table) { + /// Add columns from joined table only if they are presented in outer scope, otherwise they can be dropped if (planner_context->getGlobalPlannerContext()->hasColumnIdentifier(column_from_joined_table.name) && outer_scope_columns.contains(column_from_joined_table.name)) table_join->addJoinedColumn(column_from_joined_table); @@ -928,7 +929,6 @@ JoinTreeQueryPlan buildJoinTreeQueryPlan(const QueryTreeNodePtr & query_node, } std::vector query_plans_stack; - bool has_remote_table = false; for (size_t i = 0; i < table_expressions_stack_size; ++i) { @@ -970,12 +970,10 @@ JoinTreeQueryPlan buildJoinTreeQueryPlan(const QueryTreeNodePtr & query_node, else { const auto & table_expression_data = planner_context->getTableExpressionDataOrThrow(table_expression); - if (table_expression_data.isRemote() && (has_remote_table || i != 0)) + if (table_expression_data.isRemote() && i != 0) throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "JOIN with multiple remote storages is unsupported"); - has_remote_table = table_expression_data.isRemote(); - query_plans_stack.push_back(buildQueryPlanForTableExpression(table_expression, select_query_info, select_query_options, diff --git a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h index 89464e01d3c..b894e5caf1d 100644 --- a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h +++ b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h @@ -31,7 +31,7 @@ struct QueryPlanOptimizationSettings bool aggregation_in_order = false; /// If removing redundant sorting is enabled, for example, ORDER BY clauses in subqueries - bool remove_redundant_sorting = false; + bool remove_redundant_sorting = true; static QueryPlanOptimizationSettings fromSettings(const Settings & from); static QueryPlanOptimizationSettings fromContext(ContextPtr from); diff --git a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp index 46fe3055e32..dbf389163be 100644 --- a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp +++ b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp @@ -333,19 +333,8 @@ size_t tryPushDownFilter(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes // { // } - if (auto * sorting = typeid_cast(child.get())) - { - const auto & sort_description = sorting->getSortDescription(); - auto sort_description_it = std::find_if(sort_description.begin(), sort_description.end(), [&](auto & sort_column_description) - { - return sort_column_description.column_name == filter->getFilterColumnName(); - }); - bool can_remove_filter = sort_description_it == sort_description.end(); - - Names allowed_inputs = child->getOutputStream().header.getNames(); - if (auto updated_steps = tryAddNewFilterStep(parent_node, nodes, allowed_inputs, can_remove_filter)) - return updated_steps; - } + if (auto updated_steps = simplePushDownOverStep(parent_node, nodes, child)) + return updated_steps; if (auto updated_steps = simplePushDownOverStep(parent_node, nodes, child)) return updated_steps; diff --git a/src/Processors/Transforms/buildPushingToViewsChain.cpp b/src/Processors/Transforms/buildPushingToViewsChain.cpp index c57eb90c46b..33d6a1f2403 100644 --- a/src/Processors/Transforms/buildPushingToViewsChain.cpp +++ b/src/Processors/Transforms/buildPushingToViewsChain.cpp @@ -486,7 +486,7 @@ static QueryPipeline process(Block block, ViewRuntimeData & view, const ViewsDat if (local_context->getSettingsRef().allow_experimental_analyzer) { - InterpreterSelectQueryAnalyzer interpreter(view.query, local_context, SelectQueryOptions(), local_context->getViewSource()); + InterpreterSelectQueryAnalyzer interpreter(view.query, local_context, local_context->getViewSource(), SelectQueryOptions()); pipeline = interpreter.buildQueryPipeline(); } else diff --git a/src/Storages/LiveView/StorageLiveView.cpp b/src/Storages/LiveView/StorageLiveView.cpp index 5f398f39dcc..547becf3837 100644 --- a/src/Storages/LiveView/StorageLiveView.cpp +++ b/src/Storages/LiveView/StorageLiveView.cpp @@ -481,7 +481,7 @@ void StorageLiveView::refresh() refreshImpl(lock); } -void StorageLiveView::refreshImpl(std::lock_guard & lock) +void StorageLiveView::refreshImpl(const std::lock_guard & lock) { if (getNewBlocks(lock)) condition.notify_all(); @@ -553,7 +553,7 @@ ASTPtr StorageLiveView::getInnerBlocksQuery() return inner_blocks_query->clone(); } -MergeableBlocksPtr StorageLiveView::collectMergeableBlocks(ContextPtr local_context, std::lock_guard &) const +MergeableBlocksPtr StorageLiveView::collectMergeableBlocks(ContextPtr local_context, const std::lock_guard &) const { MergeableBlocksPtr new_mergeable_blocks = std::make_shared(); BlocksPtrs new_blocks = std::make_shared>(); @@ -672,7 +672,7 @@ QueryPipelineBuilder StorageLiveView::completeQuery(Pipes pipes) return builder; } -bool StorageLiveView::getNewBlocks(std::lock_guard & lock) +bool StorageLiveView::getNewBlocks(const std::lock_guard & lock) { SipHash hash; UInt128 key; @@ -751,7 +751,7 @@ void StorageLiveView::periodicRefreshTaskFunc() scheduleNextPeriodicRefresh(lock); } -void StorageLiveView::scheduleNextPeriodicRefresh(std::lock_guard & lock) +void StorageLiveView::scheduleNextPeriodicRefresh(const std::lock_guard & lock) { Seconds current_time = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()); Seconds blocks_time = std::chrono::duration_cast(getBlocksTime(lock).time_since_epoch()); diff --git a/src/Storages/LiveView/StorageLiveView.h b/src/Storages/LiveView/StorageLiveView.h index fbb1dbe027c..002cbf96ebe 100644 --- a/src/Storages/LiveView/StorageLiveView.h +++ b/src/Storages/LiveView/StorageLiveView.h @@ -104,7 +104,7 @@ public: /// Get blocks hash /// must be called with mutex locked - String getBlocksHashKey(std::lock_guard &) + String getBlocksHashKey(const std::lock_guard &) { if (*blocks_metadata_ptr) return (*blocks_metadata_ptr)->hash; @@ -112,7 +112,7 @@ public: } /// Get blocks version /// must be called with mutex locked - UInt64 getBlocksVersion(std::lock_guard &) + UInt64 getBlocksVersion(const std::lock_guard &) { if (*blocks_metadata_ptr) return (*blocks_metadata_ptr)->version; @@ -124,7 +124,7 @@ public: void refresh(); private: - void refreshImpl(std::lock_guard & lock); + void refreshImpl(const std::lock_guard & lock); String getBlocksTableName() const { @@ -139,14 +139,14 @@ private: /// Check we have any active readers /// must be called with mutex locked - bool hasActiveUsers(std::lock_guard &) const + bool hasActiveUsers(const std::lock_guard &) const { return active_ptr.use_count() > 1; } /// Get blocks time /// must be called with mutex locked - Time getBlocksTime(std::lock_guard &) + Time getBlocksTime(const std::lock_guard &) { if (*blocks_metadata_ptr) return (*blocks_metadata_ptr)->time; @@ -155,7 +155,7 @@ private: /// Reset blocks /// must be called with mutex locked - void reset(std::lock_guard &) + void reset(const std::lock_guard &) { (*blocks_ptr).reset(); if (*blocks_metadata_ptr) @@ -164,18 +164,18 @@ private: } /// Collect mergeable blocks and their sample. Must be called holding mutex - MergeableBlocksPtr collectMergeableBlocks(ContextPtr context, std::lock_guard & lock) const; + MergeableBlocksPtr collectMergeableBlocks(ContextPtr context, const std::lock_guard & lock) const; /// Complete query using input streams from mergeable blocks QueryPipelineBuilder completeQuery(Pipes pipes); /// Read new data blocks that store query result - bool getNewBlocks(std::lock_guard & lock); + bool getNewBlocks(const std::lock_guard & lock); void periodicRefreshTaskFunc(); /// Must be called with mutex locked - void scheduleNextPeriodicRefresh(std::lock_guard & lock); + void scheduleNextPeriodicRefresh(const std::lock_guard & lock); SelectQueryDescription select_query_description; diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 2719b95ba5d..5516d6cadf0 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -689,9 +689,7 @@ QueryTreeNodePtr buildQueryTreeDistributedTableReplacedWithLocalTable(const Sele if (!storage) storage = std::make_shared(resolved_remote_storage_id, distributed_storage_snapshot->metadata->getColumns()); - auto storage_lock = storage->lockForShare(query_context->getInitialQueryId(), query_context->getSettingsRef().lock_acquire_timeout); - auto storage_snapshot = storage->getStorageSnapshot(storage->getInMemoryMetadataPtr(), query_context); - replacement_table_expression = std::make_shared(std::move(storage), std::move(storage_lock), std::move(storage_snapshot)); + replacement_table_expression = std::make_shared(std::move(storage), query_context); } replacement_table_expression->setAlias(query_info.table_expression->getAlias()); diff --git a/src/TableFunctions/TableFunctionExecutable.cpp b/src/TableFunctions/TableFunctionExecutable.cpp index ca769a2ff0a..ae0946f8a50 100644 --- a/src/TableFunctions/TableFunctionExecutable.cpp +++ b/src/TableFunctions/TableFunctionExecutable.cpp @@ -34,8 +34,11 @@ std::vector TableFunctionExecutable::skipAnalysisForArguments(const Quer const auto & table_function_node_arguments = table_function_node.getArguments().getNodes(); size_t table_function_node_arguments_size = table_function_node_arguments.size(); - std::vector result_indexes; + if (table_function_node_arguments_size <= 3) + return {}; + std::vector result_indexes; + result_indexes.reserve(table_function_node_arguments_size - 3); for (size_t i = 3; i < table_function_node_arguments_size; ++i) result_indexes.push_back(i); diff --git a/tests/queries/0_stateless/02479_analyzer_join_with_constants.reference b/tests/queries/0_stateless/02479_analyzer_join_with_constants.reference index 2a428d5d927..3a23cce46b4 100644 --- a/tests/queries/0_stateless/02479_analyzer_join_with_constants.reference +++ b/tests/queries/0_stateless/02479_analyzer_join_with_constants.reference @@ -8,8 +8,3 @@ -- 1 1 0 0 -- -0 -0 -0 -0 -0 diff --git a/tests/queries/0_stateless/02479_analyzer_join_with_constants.sql b/tests/queries/0_stateless/02479_analyzer_join_with_constants.sql index 0cc3ff3ab00..99f20290ff0 100644 --- a/tests/queries/0_stateless/02479_analyzer_join_with_constants.sql +++ b/tests/queries/0_stateless/02479_analyzer_join_with_constants.sql @@ -24,4 +24,4 @@ SELECT * FROM (SELECT 1 AS id, 1 AS value) AS t1 ASOF LEFT JOIN (SELECT 1 AS id, SELECT '--'; -SELECT b.dt FROM (SELECT NULL > NULL AS pk, 1 AS dt FROM numbers(5)) AS a ASOF LEFT JOIN (SELECT NULL AS pk, 1 AS dt) AS b ON (a.pk = b.pk) AND 1 != 1 AND (a.dt >= b.dt); +SELECT b.dt FROM (SELECT NULL > NULL AS pk, 1 AS dt FROM numbers(5)) AS a ASOF LEFT JOIN (SELECT NULL AS pk, 1 AS dt) AS b ON (a.pk = b.pk) AND 1 != 1 AND (a.dt >= b.dt); -- { serverError 403 } diff --git a/tests/queries/0_stateless/02481_analyzer_optimize_grouping_sets_keys.reference b/tests/queries/0_stateless/02481_analyzer_optimize_grouping_sets_keys.reference index d01bb5715ad..03722034708 100644 --- a/tests/queries/0_stateless/02481_analyzer_optimize_grouping_sets_keys.reference +++ b/tests/queries/0_stateless/02481_analyzer_optimize_grouping_sets_keys.reference @@ -76,14 +76,14 @@ QUERY id: 0, group_by_type: grouping_sets LIST id: 48, nodes: 2 CONSTANT id: 49, constant_value: Float64_0.6931471805599453, constant_value_type: Float64 EXPRESSION - FUNCTION id: 50, function_name: log, function_type: ordinary, result_type: Float64 + FUNCTION id: 5, function_name: log, function_type: ordinary, result_type: Float64 ARGUMENTS - LIST id: 51, nodes: 1 - CONSTANT id: 52, constant_value: UInt64_2, constant_value_type: UInt8 - FUNCTION id: 53, function_name: avg, function_type: aggregate, result_type: Float64 + LIST id: 6, nodes: 1 + CONSTANT id: 7, constant_value: UInt64_2, constant_value_type: UInt8 + FUNCTION id: 50, function_name: avg, function_type: aggregate, result_type: Float64 ARGUMENTS - LIST id: 54, nodes: 1 - COLUMN id: 55, column_name: number, result_type: UInt64, source_id: 11 + LIST id: 51, nodes: 1 + COLUMN id: 52, column_name: number, result_type: UInt64, source_id: 11 QUERY id: 0, group_by_type: grouping_sets PROJECTION COLUMNS k Float64 @@ -153,14 +153,14 @@ QUERY id: 0, group_by_type: grouping_sets LIST id: 42, nodes: 2 CONSTANT id: 43, constant_value: Float64_0.6931471805599453, constant_value_type: Float64 EXPRESSION - FUNCTION id: 44, function_name: log, function_type: ordinary, result_type: Float64 + FUNCTION id: 5, function_name: log, function_type: ordinary, result_type: Float64 ARGUMENTS - LIST id: 45, nodes: 1 - CONSTANT id: 46, constant_value: UInt64_2, constant_value_type: UInt8 - FUNCTION id: 47, function_name: avg, function_type: aggregate, result_type: Float64 + LIST id: 6, nodes: 1 + CONSTANT id: 7, constant_value: UInt64_2, constant_value_type: UInt8 + FUNCTION id: 44, function_name: avg, function_type: aggregate, result_type: Float64 ARGUMENTS - LIST id: 48, nodes: 1 - COLUMN id: 49, column_name: number, result_type: UInt64, source_id: 11 + LIST id: 45, nodes: 1 + COLUMN id: 46, column_name: number, result_type: UInt64, source_id: 11 QUERY id: 0, group_by_type: grouping_sets PROJECTION COLUMNS k Float64 @@ -238,14 +238,14 @@ QUERY id: 0, group_by_type: grouping_sets LIST id: 47, nodes: 2 CONSTANT id: 48, constant_value: Float64_0.6931471805599453, constant_value_type: Float64 EXPRESSION - FUNCTION id: 49, function_name: log, function_type: ordinary, result_type: Float64 + FUNCTION id: 5, function_name: log, function_type: ordinary, result_type: Float64 ARGUMENTS - LIST id: 50, nodes: 1 - CONSTANT id: 51, constant_value: UInt64_2, constant_value_type: UInt8 - FUNCTION id: 52, function_name: avg, function_type: aggregate, result_type: Float64 + LIST id: 6, nodes: 1 + CONSTANT id: 7, constant_value: UInt64_2, constant_value_type: UInt8 + FUNCTION id: 49, function_name: avg, function_type: aggregate, result_type: Float64 ARGUMENTS - LIST id: 53, nodes: 1 - COLUMN id: 54, column_name: number, result_type: UInt64, source_id: 11 + LIST id: 50, nodes: 1 + COLUMN id: 51, column_name: number, result_type: UInt64, source_id: 11 QUERY id: 0, group_by_type: grouping_sets PROJECTION COLUMNS count() UInt64 From 4b0beb938890e55784654ed5f8e59eb722fd45c7 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Sat, 18 Feb 2023 17:59:12 +0100 Subject: [PATCH 401/717] Test case for incorrect combined PREWHERE column --- .../02559_multiple_read_steps_in_prewhere.reference | 4 ++++ .../0_stateless/02559_multiple_read_steps_in_prewhere.sql | 2 ++ 2 files changed, 6 insertions(+) diff --git a/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.reference b/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.reference index f2da126fc92..1bc2ac005a8 100644 --- a/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.reference +++ b/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.reference @@ -36,6 +36,10 @@ SELECT cast(id1 as UInt16) AS cond1, (id2 % 40000) AS cond2, (cond1 AND cond2) A 8 8 1 9 9 1 SELECT cast(id1 as UInt16) AS cond1 FROM test_02559 PREWHERE cond1 LIMIT 10; -- { serverError ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER } +SELECT * FROM test_02559 PREWHERE id1 <= 3 AND id2 > 0 WHERE (id1 + id2 < 15) LIMIT 10; +1 1 +2 2 +3 3 SELECT count() FROM test_02559 PREWHERE id2>=0 AND (1 OR ignore(id1)) WHERE ignore(id1)=0; 10 CREATE ROW POLICY 02559_filter_1 ON test_02559 USING id2=2 AS permissive TO ALL; diff --git a/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.sql b/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.sql index 0b434eae7df..829aea9f865 100644 --- a/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.sql +++ b/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.sql @@ -23,6 +23,8 @@ SELECT cast(id1 as UInt16) AS cond1, (id2 % 40000) AS cond2, (cond1 AND cond2) A SELECT cast(id1 as UInt16) AS cond1 FROM test_02559 PREWHERE cond1 LIMIT 10; -- { serverError ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER } +SELECT * FROM test_02559 PREWHERE id1 <= 3 AND id2 > 0 WHERE (id1 + id2 < 15) LIMIT 10; + SELECT count() FROM test_02559 PREWHERE id2>=0 AND (1 OR ignore(id1)) WHERE ignore(id1)=0; CREATE ROW POLICY 02559_filter_1 ON test_02559 USING id2=2 AS permissive TO ALL; From a3ee9b604a899ed0ee0a599309dbdc105549a32b Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sat, 18 Feb 2023 18:10:10 +0100 Subject: [PATCH 402/717] Fixed tests --- ...01300_group_by_other_keys_having.reference | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/queries/0_stateless/01300_group_by_other_keys_having.reference b/tests/queries/0_stateless/01300_group_by_other_keys_having.reference index 1f72535c40c..a9be79800c1 100644 --- a/tests/queries/0_stateless/01300_group_by_other_keys_having.reference +++ b/tests/queries/0_stateless/01300_group_by_other_keys_having.reference @@ -92,14 +92,14 @@ QUERY id: 0 LIST id: 37, nodes: 2 CONSTANT id: 38, constant_value: Float64_0.6931471805599453, constant_value_type: Float64 EXPRESSION - FUNCTION id: 39, function_name: log, function_type: ordinary, result_type: Float64 + FUNCTION id: 5, function_name: log, function_type: ordinary, result_type: Float64 ARGUMENTS - LIST id: 40, nodes: 1 - CONSTANT id: 41, constant_value: UInt64_2, constant_value_type: UInt8 - FUNCTION id: 42, function_name: avg, function_type: aggregate, result_type: Float64 + LIST id: 6, nodes: 1 + CONSTANT id: 7, constant_value: UInt64_2, constant_value_type: UInt8 + FUNCTION id: 39, function_name: avg, function_type: aggregate, result_type: Float64 ARGUMENTS - LIST id: 43, nodes: 1 - COLUMN id: 44, column_name: number, result_type: UInt64, source_id: 11 + LIST id: 40, nodes: 1 + COLUMN id: 41, column_name: number, result_type: UInt64, source_id: 11 SELECT avg(log(2) * number) AS k FROM numbers(10000000) WHERE ((number % 5) * (number % 5)) < 5 @@ -162,14 +162,14 @@ QUERY id: 0 LIST id: 33, nodes: 2 CONSTANT id: 34, constant_value: Float64_0.6931471805599453, constant_value_type: Float64 EXPRESSION - FUNCTION id: 35, function_name: log, function_type: ordinary, result_type: Float64 + FUNCTION id: 5, function_name: log, function_type: ordinary, result_type: Float64 ARGUMENTS - LIST id: 36, nodes: 1 - CONSTANT id: 37, constant_value: UInt64_2, constant_value_type: UInt8 - FUNCTION id: 38, function_name: avg, function_type: aggregate, result_type: Float64 + LIST id: 6, nodes: 1 + CONSTANT id: 7, constant_value: UInt64_2, constant_value_type: UInt8 + FUNCTION id: 35, function_name: avg, function_type: aggregate, result_type: Float64 ARGUMENTS - LIST id: 39, nodes: 1 - COLUMN id: 40, column_name: number, result_type: UInt64, source_id: 11 + LIST id: 36, nodes: 1 + COLUMN id: 37, column_name: number, result_type: UInt64, source_id: 11 SELECT (number % 5) * (number % 5) AS k FROM numbers(10000000) WHERE ((number % 5) * (number % 5)) < 5 @@ -332,14 +332,14 @@ QUERY id: 0 LIST id: 45, nodes: 2 CONSTANT id: 46, constant_value: Float64_0.6931471805599453, constant_value_type: Float64 EXPRESSION - FUNCTION id: 47, function_name: log, function_type: ordinary, result_type: Float64 + FUNCTION id: 5, function_name: log, function_type: ordinary, result_type: Float64 ARGUMENTS - LIST id: 48, nodes: 1 - CONSTANT id: 49, constant_value: UInt64_2, constant_value_type: UInt8 - FUNCTION id: 50, function_name: avg, function_type: aggregate, result_type: Float64 + LIST id: 6, nodes: 1 + CONSTANT id: 7, constant_value: UInt64_2, constant_value_type: UInt8 + FUNCTION id: 47, function_name: avg, function_type: aggregate, result_type: Float64 ARGUMENTS - LIST id: 51, nodes: 1 - COLUMN id: 52, column_name: number, result_type: UInt64, source_id: 11 + LIST id: 48, nodes: 1 + COLUMN id: 49, column_name: number, result_type: UInt64, source_id: 11 SELECT avg(log(2) * number) AS k FROM numbers(10000000) WHERE ((number % 5) * (number % 5)) < 5 From cae064822c7cbaae035ffd4ccf2b8843d7ee01ab Mon Sep 17 00:00:00 2001 From: Joanna Hulboj Date: Sat, 18 Feb 2023 17:20:57 +0000 Subject: [PATCH 403/717] FIXUP: FunctionArrayMapped - Simpler arg passing for check/exec --- src/Functions/array/FunctionArrayMapped.h | 12 +++++------- src/Functions/array/arraySort.cpp | 24 +++++++++++++++-------- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/src/Functions/array/FunctionArrayMapped.h b/src/Functions/array/FunctionArrayMapped.h index 42b7ae69a40..3c5cd29fff5 100644 --- a/src/Functions/array/FunctionArrayMapped.h +++ b/src/Functions/array/FunctionArrayMapped.h @@ -194,8 +194,7 @@ public: arguments[num_fixed_params].type->getName()); if constexpr (num_fixed_params) - Impl::checkArguments( - std::span(std::begin(arguments), num_fixed_params), getName()); + Impl::checkArguments(getName(), arguments.data()); DataTypePtr nested_type = data_type->getNestedType(); @@ -229,8 +228,7 @@ public: arguments[0].type->getName()); if constexpr (num_fixed_params) - Impl::checkArguments( - std::span(std::begin(arguments) + 1, num_fixed_params), getName()); + Impl::checkArguments(getName(), arguments.data() + 1); /// The types of the remaining arguments are already checked in getLambdaArgumentTypes. @@ -294,7 +292,7 @@ public: return Impl::execute( *column_array, column_array->getNestedColumn().getDataPtr(), - std::span(std::begin(arguments), num_fixed_params)); + arguments.data()); else return Impl::execute(*column_array, column_array->getNestedColumn().getDataPtr()); } @@ -304,7 +302,7 @@ public: return Impl::execute( *column_array, column_array->getDataPtr(), - std::span(std::begin(arguments), num_fixed_params)); + arguments.data()); else return Impl::execute(*column_array, column_array->getDataPtr()); } @@ -439,7 +437,7 @@ public: return Impl::execute( *column_first_array, lambda_result.column, - std::span(std::begin(arguments) + 1, num_fixed_params)); + arguments.data() + 1); else return Impl::execute(*column_first_array, lambda_result.column); } diff --git a/src/Functions/array/arraySort.cpp b/src/Functions/array/arraySort.cpp index ebdc408dbbd..0188324bc19 100644 --- a/src/Functions/array/arraySort.cpp +++ b/src/Functions/array/arraySort.cpp @@ -9,6 +9,7 @@ namespace DB namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int LOGICAL_ERROR; } /** Sort arrays, by values of its elements, or by values of corresponding elements of calculated expression (known as "schwartzsort"). @@ -45,31 +46,38 @@ struct ArraySortImpl } }; - static void checkArguments(std::span arguments, const String & name) + static void checkArguments(const String & name, const ColumnWithTypeAndName * fixed_arguments) requires(num_fixed_params) { - if (arguments.size() != 1) - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} needs limit argument", name); - - WhichDataType which(arguments[0].type.get()); + if (!fixed_arguments) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Expected fixed arguments to get the limit for partial array sort" + ); + WhichDataType which(fixed_arguments[0].type.get()); if (!which.isUInt() && !which.isInt()) throw Exception( ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of limit argument of function {} (must be UInt or Int)", - arguments[0].type->getName(), + fixed_arguments[0].type->getName(), name); } static ColumnPtr execute( const ColumnArray & array, ColumnPtr mapped, - std::span arguments [[maybe_unused]] = {}) + const ColumnWithTypeAndName * fixed_arguments [[maybe_unused]] = nullptr) { [[maybe_unused]] const auto limit = [&]() -> size_t { if constexpr (is_partial) { - return arguments[0].column.get()->getUInt(0); + if (!fixed_arguments) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Expected fixed arguments to get the limit for partial array sort" + ); + return fixed_arguments[0].column.get()->getUInt(0); } return 0; }(); From 88fb6c3c25aa4eb0a48a51152fbbacfe236cbd54 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Sat, 18 Feb 2023 18:29:16 +0100 Subject: [PATCH 404/717] Fix for incorrect combined PREWHERE column --- .../MergeTreeSplitPrewhereIntoReadSteps.cpp | 31 +++++++++++++++---- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp b/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp index 268ae5509d1..0dd91536bb4 100644 --- a/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp +++ b/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp @@ -133,6 +133,9 @@ const ActionsDAG::Node & addCast( const String & type_name, OriginalToNewNodeMap & node_remap) { + if (node_to_cast.result_type->getName() == type_name) + return node_to_cast; + Field cast_type_constant_value(type_name); ColumnWithTypeAndName column; @@ -271,12 +274,28 @@ bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionAction else if (output->result_name == prewhere_info->prewhere_column_name) { /// Special case for final PREWHERE column: it is an AND combination of all conditions, - /// but we have only the condition for the last step here. - /// However we know that the ultimate result after filtering is constant 1 for the PREWHERE column. - auto const_true = output->result_type->createColumnConst(0, Field{1}); - const auto & prewhere_result_node = - steps.back().actions->addColumn(ColumnWithTypeAndName(const_true, output->result_type, output->result_name)); - steps.back().actions->addOrReplaceInOutputs(prewhere_result_node); + /// but we have only the condition for the last step here. We know that the combined filter is equivalent to + /// to the last condition after filters from previous steps are applied. We just need to CAST the last condition + /// to the type of combined filter. We do this in 2 steps: + /// 1. AND the last condition with constant True. This is needed to make sure that in the last step filter has UInt8 type + /// but containes values other than 0 and 1 (e.g. if it is (number%5) it contains 2,3,4) + /// 2. CAST the result to the exact type of the PREWHERE column from the original DAG + const auto & last_step_result_node_info = node_remap[steps.back().column_name]; + auto & last_step_dag = steps.back().actions; + /// Build AND(last_step_result_node, true) + Field true_constant_value(true); + ColumnWithTypeAndName column; + column.column = DataTypeUInt8().createColumnConst(0, true_constant_value); + column.type = std::make_shared(); + const auto * cast_type_constant_node = &last_step_dag->addColumn(std::move(column)); + ActionsDAG::NodeRawConstPtrs children = {last_step_result_node_info.node, cast_type_constant_node}; + FunctionOverloadResolverPtr func_builder_and = std::make_unique(std::make_shared()); + const auto& and_node = addFunction(last_step_dag, func_builder_and, children, node_remap); + /// Build CAST(and_node, type of PREWHERE column) + const auto & cast_node = addCast(last_step_dag, and_node, output->result_type->getName(), node_remap); + /// Add alias for the result with the name of the PREWHERE column + const auto & prewhere_result_node = last_step_dag->addAlias(cast_node, output->result_name); + last_step_dag->addOrReplaceInOutputs(prewhere_result_node); } else { From dbacc57e4b1c53b6995280813ea393d45bb29886 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Sat, 18 Feb 2023 18:31:18 +0100 Subject: [PATCH 405/717] Test case for Nullable step filter column --- .../02559_multiple_read_steps_in_prewhere.reference | 7 +++++++ .../0_stateless/02559_multiple_read_steps_in_prewhere.sql | 2 ++ 2 files changed, 9 insertions(+) diff --git a/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.reference b/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.reference index 1bc2ac005a8..580e7d4fef3 100644 --- a/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.reference +++ b/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.reference @@ -20,6 +20,13 @@ SELECT cast(id1 as UInt16) AS cond1, (id2 % 40000) AS cond2, (cond1 AND cond2) A 7 7 1 8 8 1 9 9 1 +SELECT cast(id1 as UInt16) AS cond1, (if(id2 > 3, id2, NULL) % 40000) AS cond2, (cond1 AND cond2) AS cond FROM test_02559 PREWHERE cond LIMIT 10; +4 4 1 +5 5 1 +6 6 1 +7 7 1 +8 8 1 +9 9 1 SELECT cast(id1 as UInt16) AS cond1, (id2 % 40000) AS cond2, (cond1 AND cond2) AS cond FROM test_02559 PREWHERE cond AND id2 > 4 LIMIT 10; 5 5 1 6 6 1 diff --git a/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.sql b/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.sql index 829aea9f865..ee11b6edd55 100644 --- a/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.sql +++ b/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.sql @@ -15,6 +15,8 @@ SELECT cast(id1 as UInt16) AS id16 FROM test_02559 PREWHERE id16 and (id2 % 4000 SELECT cast(id1 as UInt16) AS cond1, (id2 % 40000) AS cond2, (cond1 AND cond2) AS cond FROM test_02559 PREWHERE cond LIMIT 10; +SELECT cast(id1 as UInt16) AS cond1, (if(id2 > 3, id2, NULL) % 40000) AS cond2, (cond1 AND cond2) AS cond FROM test_02559 PREWHERE cond LIMIT 10; + SELECT cast(id1 as UInt16) AS cond1, (id2 % 40000) AS cond2, (cond1 AND cond2) AS cond FROM test_02559 PREWHERE cond AND id2 > 4 LIMIT 10; SELECT cast(id1 as UInt16) AS cond1, (id2 % 40000) AS cond2, (cond1 AND cond2) AS cond FROM test_02559 PREWHERE id2 > 5 AND cond LIMIT 10; From 830398f0bd46e66c20e0fca0df94c9213ffa6247 Mon Sep 17 00:00:00 2001 From: flynn Date: Sat, 18 Feb 2023 17:14:25 +0000 Subject: [PATCH 406/717] fix build without avro fix --- src/Storages/StorageIceberg.cpp | 3 ++- src/Storages/StorageIceberg.h | 3 ++- src/TableFunctions/TableFunctionIceberg.cpp | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Storages/StorageIceberg.cpp b/src/Storages/StorageIceberg.cpp index 7626f4dbe82..dd7923fa76a 100644 --- a/src/Storages/StorageIceberg.cpp +++ b/src/Storages/StorageIceberg.cpp @@ -1,5 +1,6 @@ #include "config.h" -#if USE_AWS_S3 + +#if USE_AWS_S3 && USE_AVRO # include # include diff --git a/src/Storages/StorageIceberg.h b/src/Storages/StorageIceberg.h index 20afde337c6..e985ce5ad15 100644 --- a/src/Storages/StorageIceberg.h +++ b/src/Storages/StorageIceberg.h @@ -2,7 +2,8 @@ #include "config.h" -#if USE_AWS_S3 +// StorageIceberd depending on Avro to parse metadata with Avro format. +#if USE_AWS_S3 && USE_AVRO # include # include diff --git a/src/TableFunctions/TableFunctionIceberg.cpp b/src/TableFunctions/TableFunctionIceberg.cpp index 16119b8ea78..5600e028527 100644 --- a/src/TableFunctions/TableFunctionIceberg.cpp +++ b/src/TableFunctions/TableFunctionIceberg.cpp @@ -1,6 +1,6 @@ #include "config.h" -#if USE_AWS_S3 +#if USE_AWS_S3 && USE_AVRO # include # include From 177261a5ac80939666365f42e8a58be0b697325f Mon Sep 17 00:00:00 2001 From: flynn Date: Sat, 18 Feb 2023 17:34:37 +0000 Subject: [PATCH 407/717] fix typo --- src/Storages/StorageIceberg.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/StorageIceberg.h b/src/Storages/StorageIceberg.h index e985ce5ad15..d4906d472b2 100644 --- a/src/Storages/StorageIceberg.h +++ b/src/Storages/StorageIceberg.h @@ -2,7 +2,7 @@ #include "config.h" -// StorageIceberd depending on Avro to parse metadata with Avro format. +// StorageIceberg depending on Avro to parse metadata with Avro format. #if USE_AWS_S3 && USE_AVRO # include From 0ed0b191f8a7edd142ec396fe26d9efe3dea13d3 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Sat, 18 Feb 2023 18:45:38 +0100 Subject: [PATCH 408/717] Fix for Nullable step filter column --- .../MergeTreeSplitPrewhereIntoReadSteps.cpp | 44 ++++++++++++++----- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp b/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp index 0dd91536bb4..b42900d239d 100644 --- a/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp +++ b/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp @@ -149,6 +149,27 @@ const ActionsDAG::Node & addCast( return addFunction(dag, func_builder_cast, std::move(children), node_remap); } +/// Normalizes the filter node by adding AND with a constant true. +/// This: +/// 1. produces a result with the proper Nullable or non-Nullable UInt8 type and +/// 2. makes sure that the result contains only 0 or 1 values even if the source column contains non-boolean values. +const ActionsDAG::Node & addAndTrue( + ActionsDAGPtr dag, + const ActionsDAG::Node & filter_node_to_normalize, + OriginalToNewNodeMap & node_remap) +{ + Field const_true_value(true); + + ColumnWithTypeAndName const_true_column; + const_true_column.column = DataTypeUInt8().createColumnConst(0, const_true_value); + const_true_column.type = std::make_shared(); + + const auto * const_true_node = &dag->addColumn(std::move(const_true_column)); + ActionsDAG::NodeRawConstPtrs children = {&filter_node_to_normalize, const_true_node}; + FunctionOverloadResolverPtr func_builder_and = std::make_unique(std::make_shared()); + return addFunction(dag, func_builder_and, children, node_remap); +} + } /// We want to build a sequence of steps that will compute parts of the prewhere condition. @@ -241,15 +262,21 @@ bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionAction else { const auto & result_node = *new_condition_nodes.front(); - /// Add cast to UInt8 if needed - if (result_node.result_type->getTypeId() == TypeIndex::UInt8) + /// Check if explicit cast is needed for the condition to serve as a filter. + const auto result_type_name = result_node.result_type->getName(); + if (result_type_name == "UInt8" || + result_type_name == "Nullable(UInt8)" || + result_type_name == "LowCardinality(UInt8)" || + result_type_name == "LowCardinality(Nullable(UInt8))") { + /// No need to cast step_dag->addOrReplaceInOutputs(result_node); result_name = result_node.result_name; } else { - const auto & cast_node = addCast(step_dag, result_node, "UInt8", node_remap); + /// Build "condition AND True" expression to "cast" the condition to UInt8 or Nullable(UInt8) depending on its type. + const auto & cast_node = addAndTrue(step_dag, result_node, node_remap); step_dag->addOrReplaceInOutputs(cast_node); result_name = cast_node.result_name; } @@ -278,19 +305,12 @@ bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionAction /// to the last condition after filters from previous steps are applied. We just need to CAST the last condition /// to the type of combined filter. We do this in 2 steps: /// 1. AND the last condition with constant True. This is needed to make sure that in the last step filter has UInt8 type - /// but containes values other than 0 and 1 (e.g. if it is (number%5) it contains 2,3,4) + /// but contains values other than 0 and 1 (e.g. if it is (number%5) it contains 2,3,4) /// 2. CAST the result to the exact type of the PREWHERE column from the original DAG const auto & last_step_result_node_info = node_remap[steps.back().column_name]; auto & last_step_dag = steps.back().actions; /// Build AND(last_step_result_node, true) - Field true_constant_value(true); - ColumnWithTypeAndName column; - column.column = DataTypeUInt8().createColumnConst(0, true_constant_value); - column.type = std::make_shared(); - const auto * cast_type_constant_node = &last_step_dag->addColumn(std::move(column)); - ActionsDAG::NodeRawConstPtrs children = {last_step_result_node_info.node, cast_type_constant_node}; - FunctionOverloadResolverPtr func_builder_and = std::make_unique(std::make_shared()); - const auto& and_node = addFunction(last_step_dag, func_builder_and, children, node_remap); + const auto & and_node = addAndTrue(last_step_dag, *last_step_result_node_info.node, node_remap); /// Build CAST(and_node, type of PREWHERE column) const auto & cast_node = addCast(last_step_dag, and_node, output->result_type->getName(), node_remap); /// Add alias for the result with the name of the PREWHERE column From 7cc10008e393b5dd254eb11189e96249e0029825 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sat, 18 Feb 2023 20:34:12 +0100 Subject: [PATCH 409/717] Fixed build --- src/Analyzer/AggregationUtils.cpp | 2 +- src/Analyzer/QueryTreePassManager.cpp | 2 +- src/Analyzer/Utils.cpp | 4 ++-- src/Analyzer/WindowFunctionsUtils.cpp | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Analyzer/AggregationUtils.cpp b/src/Analyzer/AggregationUtils.cpp index 0f134b00794..77b71d07bcf 100644 --- a/src/Analyzer/AggregationUtils.cpp +++ b/src/Analyzer/AggregationUtils.cpp @@ -51,7 +51,7 @@ public: has_aggregate_functions = true; } - bool needChildVisit(const QueryTreeNodePtr &, const QueryTreeNodePtr & child_node) + bool needChildVisit(const QueryTreeNodePtr &, const QueryTreeNodePtr & child_node) const { if (only_check && has_aggregate_functions) return false; diff --git a/src/Analyzer/QueryTreePassManager.cpp b/src/Analyzer/QueryTreePassManager.cpp index ee68d58bb33..588457f90f7 100644 --- a/src/Analyzer/QueryTreePassManager.cpp +++ b/src/Analyzer/QueryTreePassManager.cpp @@ -63,7 +63,7 @@ public: : pass_name(std::move(pass_name_)) {} - bool needChildVisit(VisitQueryTreeNodeType & parent, VisitQueryTreeNodeType &) + static bool needChildVisit(VisitQueryTreeNodeType & parent, VisitQueryTreeNodeType &) { if (parent->getNodeType() == QueryTreeNodeType::TABLE_FUNCTION) return false; diff --git a/src/Analyzer/Utils.cpp b/src/Analyzer/Utils.cpp index f638ff9c5ce..e746a8ff570 100644 --- a/src/Analyzer/Utils.cpp +++ b/src/Analyzer/Utils.cpp @@ -398,7 +398,7 @@ public: has_function = function_node->getFunctionName() == function_name; } - bool needChildVisit(const QueryTreeNodePtr &, const QueryTreeNodePtr & child_node) + bool needChildVisit(const QueryTreeNodePtr &, const QueryTreeNodePtr & child_node) const { if (has_function) return false; @@ -456,7 +456,7 @@ public: node = node_it->second; } - bool needChildVisit(const QueryTreeNodePtr &, const QueryTreeNodePtr & child_node) + static bool needChildVisit(const QueryTreeNodePtr &, const QueryTreeNodePtr & child_node) { auto child_node_type = child_node->getNodeType(); return !(child_node_type == QueryTreeNodeType::QUERY || child_node_type == QueryTreeNodeType::UNION); diff --git a/src/Analyzer/WindowFunctionsUtils.cpp b/src/Analyzer/WindowFunctionsUtils.cpp index 10deec31cda..811ccc2cd1a 100644 --- a/src/Analyzer/WindowFunctionsUtils.cpp +++ b/src/Analyzer/WindowFunctionsUtils.cpp @@ -51,7 +51,7 @@ public: has_window_functions = true; } - bool needChildVisit(const QueryTreeNodePtr &, const QueryTreeNodePtr & child_node) + bool needChildVisit(const QueryTreeNodePtr &, const QueryTreeNodePtr & child_node) const { if (only_check && has_window_functions) return false; From 91b15caa8bdd92a87b6795948a48dd25540cce59 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Sat, 18 Feb 2023 20:47:30 +0100 Subject: [PATCH 410/717] Simplify arguments validation --- src/Functions/regexpExtract.cpp | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/src/Functions/regexpExtract.cpp b/src/Functions/regexpExtract.cpp index 4e5d5819c5c..37eece3a9f2 100644 --- a/src/Functions/regexpExtract.cpp +++ b/src/Functions/regexpExtract.cpp @@ -15,7 +15,6 @@ namespace DB namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; - extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int ILLEGAL_COLUMN; extern const int INDEX_OF_POSITIONAL_ARGUMENT_IS_OUT_OF_RANGE; } @@ -38,7 +37,7 @@ namespace bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } - DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { if (arguments.size() != 2 && arguments.size() != 3) throw Exception( @@ -47,26 +46,15 @@ namespace getName(), arguments.size()); - if (!isString(arguments[0])) - throw Exception( - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "Illegal type {} of first argument of function {}", - arguments[0]->getName(), - getName()); + FunctionArgumentDescriptors args{ + {"haystack", &isString, nullptr, "String"}, + {"pattern", &isString, isColumnConst, "const String"}, + }; - if (!isString(arguments[1])) - throw Exception( - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "Illegal type {} of second argument of function {}", - arguments[1]->getName(), - getName()); + if (arguments.size() == 3) + args.emplace_back(FunctionArgumentDescriptor{"index", &isInteger, nullptr, "Integer"}); - if (arguments.size() > 2 && !isInteger(arguments[2])) - throw Exception( - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "Illegal type {} of third argument of function {}", - arguments[2]->getName(), - getName()); + validateFunctionArgumentTypes(*this, arguments, args); return std::make_shared(); } From 1480fce68884257409b90758f3691016083c4db2 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Sat, 18 Feb 2023 20:50:01 +0100 Subject: [PATCH 411/717] Removed extra identation --- src/Functions/regexpExtract.cpp | 373 ++++++++++++++++---------------- 1 file changed, 187 insertions(+), 186 deletions(-) diff --git a/src/Functions/regexpExtract.cpp b/src/Functions/regexpExtract.cpp index 37eece3a9f2..22f1b3599ba 100644 --- a/src/Functions/regexpExtract.cpp +++ b/src/Functions/regexpExtract.cpp @@ -21,119 +21,169 @@ namespace ErrorCodes namespace { - class FunctionRegexpExtract : public IFunction +class FunctionRegexpExtract : public IFunction +{ +public: + static constexpr auto name = "regexpExtract"; + static FunctionPtr create(ContextPtr) { return std::make_shared(); } + + String getName() const override { return name; } + + bool isVariadic() const override { return true; } + size_t getNumberOfArguments() const override { return 0; } + + bool useDefaultImplementationForConstants() const override { return true; } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; } + + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { - public: - static constexpr auto name = "regexpExtract"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + if (arguments.size() != 2 && arguments.size() != 3) + throw Exception( + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Number of arguments for function {} doesn't match: passed {}", + getName(), + arguments.size()); - String getName() const override { return name; } + FunctionArgumentDescriptors args{ + {"haystack", &isString, nullptr, "String"}, + {"pattern", &isString, isColumnConst, "const String"}, + }; - bool isVariadic() const override { return true; } - size_t getNumberOfArguments() const override { return 0; } + if (arguments.size() == 3) + args.emplace_back(FunctionArgumentDescriptor{"index", &isInteger, nullptr, "Integer"}); - bool useDefaultImplementationForConstants() const override { return true; } - ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; } + validateFunctionArgumentTypes(*this, arguments, args); - bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } + return std::make_shared(); + } - DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override + { + const ColumnPtr column = arguments[0].column; + const ColumnPtr column_pattern = arguments[1].column; + const ColumnPtr column_index = arguments.size() > 2 ? arguments[2].column : nullptr; + + /// Check if the second argument is const column + const ColumnConst * col_pattern = typeid_cast(column_pattern.get()); + if (!col_pattern) + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Second argument of function {} must be constant string", getName()); + + /// Check if the first argument is string column(const or not) + const ColumnConst * col_const = typeid_cast(column.get()); + const ColumnString * col = nullptr; + if (col_const) + col = typeid_cast(&col_const->getDataColumn()); + else + col = typeid_cast(column.get()); + if (!col) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of argument of function {}", arguments[0].column->getName(), getName()); + + auto col_res = ColumnString::create(); + ColumnString::Chars & vec_res = col_res->getChars(); + ColumnString::Offsets & offsets_res = col_res->getOffsets(); + + if (col_const) + constantVector(col_const->getValue(), col_pattern->getValue(), column_index, vec_res, offsets_res); + else if (!column_index || isColumnConst(*column_index)) { - if (arguments.size() != 2 && arguments.size() != 3) - throw Exception( - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, - "Number of arguments for function {} doesn't match: passed {}", - getName(), - arguments.size()); - - FunctionArgumentDescriptors args{ - {"haystack", &isString, nullptr, "String"}, - {"pattern", &isString, isColumnConst, "const String"}, - }; - - if (arguments.size() == 3) - args.emplace_back(FunctionArgumentDescriptor{"index", &isInteger, nullptr, "Integer"}); - - validateFunctionArgumentTypes(*this, arguments, args); - - return std::make_shared(); + const auto * col_const_index = typeid_cast(column_index.get()); + ssize_t index = !col_const_index ? 1 : col_const_index->getInt(0); + vectorConstant(col->getChars(), col->getOffsets(), col_pattern->getValue(), index, vec_res, offsets_res); } + else + vectorVector(col->getChars(), col->getOffsets(), col_pattern->getValue(), column_index, vec_res, offsets_res); - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override + return col_res; + } + +private: + static void saveMatch( + const OptimizedRegularExpression::MatchVec & matches, + size_t match_index, + const ColumnString::Chars & data, + size_t data_offset, + ColumnString::Chars & res_data, + ColumnString::Offsets & res_offsets, + size_t & res_offset) + { + if (match_index < matches.size() && matches[match_index].offset != std::string::npos) { - const ColumnPtr column = arguments[0].column; - const ColumnPtr column_pattern = arguments[1].column; - const ColumnPtr column_index = arguments.size() > 2 ? arguments[2].column : nullptr; - - /// Check if the second argument is const column - const ColumnConst * col_pattern = typeid_cast(column_pattern.get()); - if (!col_pattern) - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Second argument of function {} must be constant string", getName()); - - /// Check if the first argument is string column(const or not) - const ColumnConst * col_const = typeid_cast(column.get()); - const ColumnString * col = nullptr; - if (col_const) - col = typeid_cast(&col_const->getDataColumn()); - else - col = typeid_cast(column.get()); - if (!col) - throw Exception( - ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of argument of function {}", arguments[0].column->getName(), getName()); - - auto col_res = ColumnString::create(); - ColumnString::Chars & vec_res = col_res->getChars(); - ColumnString::Offsets & offsets_res = col_res->getOffsets(); - - if (col_const) - constantVector(col_const->getValue(), col_pattern->getValue(), column_index, vec_res, offsets_res); - else if (!column_index || isColumnConst(*column_index)) - { - const auto * col_const_index = typeid_cast(column_index.get()); - ssize_t index = !col_const_index ? 1 : col_const_index->getInt(0); - vectorConstant(col->getChars(), col->getOffsets(), col_pattern->getValue(), index, vec_res, offsets_res); - } - else - vectorVector(col->getChars(), col->getOffsets(), col_pattern->getValue(), column_index, vec_res, offsets_res); - - return col_res; + const auto & match = matches[match_index]; + res_data.resize(res_offset + match.length + 1); + memcpySmallAllowReadWriteOverflow15(&res_data[res_offset], &data[data_offset + match.offset], match.length); + res_offset += match.length; } + else + res_data.resize(res_offset + 1); - private: - static void saveMatch( - const OptimizedRegularExpression::MatchVec & matches, - size_t match_index, - const ColumnString::Chars & data, - size_t data_offset, - ColumnString::Chars & res_data, - ColumnString::Offsets & res_offsets, - size_t & res_offset) + res_data[res_offset] = 0; + ++res_offset; + res_offsets.push_back(res_offset); + } + + static void vectorConstant( + const ColumnString::Chars & data, + const ColumnString::Offsets & offsets, + const std::string & pattern, + ssize_t index, + ColumnString::Chars & res_data, + ColumnString::Offsets & res_offsets) + { + const Regexps::Regexp regexp = Regexps::createRegexp(pattern); + unsigned capture = regexp.getNumberOfSubpatterns(); + if (index < 0 || index >= capture + 1) + throw Exception( + ErrorCodes::INDEX_OF_POSITIONAL_ARGUMENT_IS_OUT_OF_RANGE, + "Index value {} is out of range, should be in [0, {})", + index, + capture + 1); + + OptimizedRegularExpression::MatchVec matches; + matches.reserve(index + 1); + + res_data.reserve(data.size() / 5); + res_offsets.reserve(offsets.size()); + size_t prev_offset = 0; + size_t res_offset = 0; + for (size_t cur_offset : offsets) { - if (match_index < matches.size() && matches[match_index].offset != std::string::npos) - { - const auto & match = matches[match_index]; - res_data.resize(res_offset + match.length + 1); - memcpySmallAllowReadWriteOverflow15(&res_data[res_offset], &data[data_offset + match.offset], match.length); - res_offset += match.length; - } - else - res_data.resize(res_offset + 1); + regexp.match( + reinterpret_cast(&data[prev_offset]), + cur_offset - prev_offset - 1, + matches, + static_cast(index + 1)); - res_data[res_offset] = 0; - ++res_offset; - res_offsets.push_back(res_offset); + saveMatch(matches, index, data, prev_offset, res_data, res_offsets, res_offset); + prev_offset = cur_offset; } + } - static void vectorConstant( - const ColumnString::Chars & data, - const ColumnString::Offsets & offsets, - const std::string & pattern, - ssize_t index, - ColumnString::Chars & res_data, - ColumnString::Offsets & res_offsets) + static void vectorVector( + const ColumnString::Chars & data, + const ColumnString::Offsets & offsets, + const std::string & pattern, + const ColumnPtr & column_index, + ColumnString::Chars & res_data, + ColumnString::Offsets & res_offsets) + { + res_data.reserve(data.size() / 5); + res_offsets.reserve(offsets.size()); + + const Regexps::Regexp regexp = Regexps::createRegexp(pattern); + unsigned capture = regexp.getNumberOfSubpatterns(); + + OptimizedRegularExpression::MatchVec matches; + matches.reserve(capture + 1); + size_t prev_offset = 0; + size_t res_offset = 0; + for (size_t i = 0; i < offsets.size(); ++i) { - const Regexps::Regexp regexp = Regexps::createRegexp(pattern); - unsigned capture = regexp.getNumberOfSubpatterns(); + size_t cur_offset = offsets[i]; + + ssize_t index = column_index->getInt(i); if (index < 0 || index >= capture + 1) throw Exception( ErrorCodes::INDEX_OF_POSITIONAL_ARGUMENT_IS_OUT_OF_RANGE, @@ -141,103 +191,54 @@ namespace index, capture + 1); - OptimizedRegularExpression::MatchVec matches; - matches.reserve(index + 1); + regexp.match( + reinterpret_cast(&data[prev_offset]), + cur_offset - prev_offset - 1, + matches, + static_cast(index + 1)); - res_data.reserve(data.size() / 5); - res_offsets.reserve(offsets.size()); - size_t prev_offset = 0; - size_t res_offset = 0; - for (size_t cur_offset : offsets) - { - regexp.match( - reinterpret_cast(&data[prev_offset]), - cur_offset - prev_offset - 1, - matches, - static_cast(index + 1)); - - saveMatch(matches, index, data, prev_offset, res_data, res_offsets, res_offset); - prev_offset = cur_offset; - } + saveMatch(matches, index, data, prev_offset, res_data, res_offsets, res_offset); + prev_offset = cur_offset; } + } - static void vectorVector( - const ColumnString::Chars & data, - const ColumnString::Offsets & offsets, - const std::string & pattern, - const ColumnPtr & column_index, - ColumnString::Chars & res_data, - ColumnString::Offsets & res_offsets) + static void constantVector( + const std::string & str, + const std::string & pattern, + const ColumnPtr & column_index, + ColumnString::Chars & res_data, + ColumnString::Offsets & res_offsets) + { + size_t rows = column_index->size(); + res_data.reserve(str.size() / 5); + res_offsets.reserve(rows); + + /// Copy data into padded array to be able to use memcpySmallAllowReadWriteOverflow15. + ColumnString::Chars padded_str; + padded_str.insert(str.begin(), str.end()); + + const Regexps::Regexp regexp = Regexps::createRegexp(pattern); + unsigned capture = regexp.getNumberOfSubpatterns(); + OptimizedRegularExpression::MatchVec matches; + matches.reserve(capture + 1); + regexp.match(reinterpret_cast(padded_str.data()), padded_str.size(), matches, static_cast(capture + 1)); + + size_t res_offset = 0; + for (size_t i = 0; i < rows; ++i) { - res_data.reserve(data.size() / 5); - res_offsets.reserve(offsets.size()); + ssize_t index = column_index->getInt(i); + if (index < 0 || index >= capture + 1) + throw Exception( + ErrorCodes::INDEX_OF_POSITIONAL_ARGUMENT_IS_OUT_OF_RANGE, + "Index value {} is out of range, should be in [0, {})", + index, + capture + 1); - const Regexps::Regexp regexp = Regexps::createRegexp(pattern); - unsigned capture = regexp.getNumberOfSubpatterns(); - - OptimizedRegularExpression::MatchVec matches; - matches.reserve(capture + 1); - size_t prev_offset = 0; - size_t res_offset = 0; - for (size_t i = 0; i < offsets.size(); ++i) - { - size_t cur_offset = offsets[i]; - - ssize_t index = column_index->getInt(i); - if (index < 0 || index >= capture + 1) - throw Exception( - ErrorCodes::INDEX_OF_POSITIONAL_ARGUMENT_IS_OUT_OF_RANGE, - "Index value {} is out of range, should be in [0, {})", - index, - capture + 1); - - regexp.match( - reinterpret_cast(&data[prev_offset]), - cur_offset - prev_offset - 1, - matches, - static_cast(index + 1)); - - saveMatch(matches, index, data, prev_offset, res_data, res_offsets, res_offset); - prev_offset = cur_offset; - } + saveMatch(matches, index, padded_str, 0, res_data, res_offsets, res_offset); } + } +}; - static void constantVector( - const std::string & str, - const std::string & pattern, - const ColumnPtr & column_index, - ColumnString::Chars & res_data, - ColumnString::Offsets & res_offsets) - { - size_t rows = column_index->size(); - res_data.reserve(str.size() / 5); - res_offsets.reserve(rows); - - /// Copy data into padded array to be able to use memcpySmallAllowReadWriteOverflow15. - ColumnString::Chars padded_str; - padded_str.insert(str.begin(), str.end()); - - const Regexps::Regexp regexp = Regexps::createRegexp(pattern); - unsigned capture = regexp.getNumberOfSubpatterns(); - OptimizedRegularExpression::MatchVec matches; - matches.reserve(capture + 1); - regexp.match(reinterpret_cast(padded_str.data()), padded_str.size(), matches, static_cast(capture + 1)); - - size_t res_offset = 0; - for (size_t i = 0; i < rows; ++i) - { - ssize_t index = column_index->getInt(i); - if (index < 0 || index >= capture + 1) - throw Exception( - ErrorCodes::INDEX_OF_POSITIONAL_ARGUMENT_IS_OUT_OF_RANGE, - "Index value {} is out of range, should be in [0, {})", - index, - capture + 1); - - saveMatch(matches, index, padded_str, 0, res_data, res_offsets, res_offset); - } - } - }; } REGISTER_FUNCTION(RegexpExtract) From 0859bbb08b49442716f0339fd3ff2e9caeaa7cd8 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 17 Feb 2023 10:07:20 +0100 Subject: [PATCH 412/717] Compress tar archives with zstd in intergration tests Signed-off-by: Azat Khuzhin --- tests/integration/ci-runner.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py index 9da2a58cc3e..d6d17abe725 100755 --- a/tests/integration/ci-runner.py +++ b/tests/integration/ci-runner.py @@ -381,7 +381,9 @@ class ClickhouseIntegrationTestsRunner: def _compress_logs(self, dir, relpaths, result_path): retcode = subprocess.call( # STYLE_CHECK_ALLOW_SUBPROCESS_CHECK_CALL - "tar czf {} -C {} {}".format(result_path, dir, " ".join(relpaths)), + "tar --use-compress-program='zstd --threads=0' -cf {} -C {} {}".format( + result_path, dir, " ".join(relpaths) + ), shell=True, ) # tar return 1 when the files are changed on compressing, we ignore it @@ -711,7 +713,7 @@ class ClickhouseIntegrationTestsRunner: if extra_logs_names or test_data_dirs_diff: extras_result_path = os.path.join( str(self.path()), - "integration_run_" + test_group_str + "_" + str(i) + ".tar.gz", + "integration_run_" + test_group_str + "_" + str(i) + ".tar.zst", ) self._compress_logs( os.path.join(repo_path, "tests/integration"), From f91a12717535a0cebd8d269cc7cf80b52969ccfa Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Sat, 18 Feb 2023 23:42:36 +0100 Subject: [PATCH 413/717] Repro for bug found by fuzzer and more similar cases --- ...559_multiple_read_steps_in_prewhere.reference | 16 ++++++++++++++++ .../02559_multiple_read_steps_in_prewhere.sql | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.reference b/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.reference index 580e7d4fef3..e713c2d01e5 100644 --- a/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.reference +++ b/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.reference @@ -49,6 +49,22 @@ SELECT * FROM test_02559 PREWHERE id1 <= 3 AND id2 > 0 WHERE (id1 + id2 < 15) LI 3 3 SELECT count() FROM test_02559 PREWHERE id2>=0 AND (1 OR ignore(id1)) WHERE ignore(id1)=0; 10 +SELECT count() FROM test_02559 PREWHERE ignore(id1); +0 +SELECT count() FROM test_02559 PREWHERE 1 OR ignore(id1); +10 +SELECT count() FROM test_02559 PREWHERE ignore(id1) AND id2 > 0; +0 +SELECT count() FROM test_02559 PREWHERE (1 OR ignore(id1)) AND id2 > 0; +9 +SELECT count() FROM test_02559 PREWHERE (id1 <= 10 AND id2 > 0) AND ignore(id1); +0 +SELECT count() FROM test_02559 PREWHERE ignore(id1) AND (id1 <= 10 AND id2 > 0); +0 +SELECT count() FROM test_02559 PREWHERE (id1 <= 10 AND id2 > 0) AND (1 OR ignore(id1)); +9 +SELECT count() FROM test_02559 PREWHERE (1 OR ignore(id1)) AND (id1 <= 10 AND id2 > 0); +9 CREATE ROW POLICY 02559_filter_1 ON test_02559 USING id2=2 AS permissive TO ALL; SELECT * FROM test_02559; 2 2 diff --git a/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.sql b/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.sql index ee11b6edd55..1e969afac33 100644 --- a/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.sql +++ b/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere.sql @@ -29,6 +29,22 @@ SELECT * FROM test_02559 PREWHERE id1 <= 3 AND id2 > 0 WHERE (id1 + id2 < 15) LI SELECT count() FROM test_02559 PREWHERE id2>=0 AND (1 OR ignore(id1)) WHERE ignore(id1)=0; +SELECT count() FROM test_02559 PREWHERE ignore(id1); + +SELECT count() FROM test_02559 PREWHERE 1 OR ignore(id1); + +SELECT count() FROM test_02559 PREWHERE ignore(id1) AND id2 > 0; + +SELECT count() FROM test_02559 PREWHERE (1 OR ignore(id1)) AND id2 > 0; + +SELECT count() FROM test_02559 PREWHERE (id1 <= 10 AND id2 > 0) AND ignore(id1); + +SELECT count() FROM test_02559 PREWHERE ignore(id1) AND (id1 <= 10 AND id2 > 0); + +SELECT count() FROM test_02559 PREWHERE (id1 <= 10 AND id2 > 0) AND (1 OR ignore(id1)); + +SELECT count() FROM test_02559 PREWHERE (1 OR ignore(id1)) AND (id1 <= 10 AND id2 > 0); + CREATE ROW POLICY 02559_filter_1 ON test_02559 USING id2=2 AS permissive TO ALL; SELECT * FROM test_02559; From 7b4fbf33b3c26b577bbc75ae277bfd3835a56be5 Mon Sep 17 00:00:00 2001 From: flynn Date: Sun, 19 Feb 2023 03:47:14 +0000 Subject: [PATCH 414/717] fix --- src/Storages/registerStorages.cpp | 8 +++++++- src/TableFunctions/registerTableFunctions.cpp | 4 +++- src/TableFunctions/registerTableFunctions.h | 4 +++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/Storages/registerStorages.cpp b/src/Storages/registerStorages.cpp index 2f3703fb6ea..e690189c365 100644 --- a/src/Storages/registerStorages.cpp +++ b/src/Storages/registerStorages.cpp @@ -35,8 +35,10 @@ void registerStorageCOS(StorageFactory & factory); void registerStorageOSS(StorageFactory & factory); void registerStorageHudi(StorageFactory & factory); void registerStorageDeltaLake(StorageFactory & factory); +#if USE_AVRO void registerStorageIceberg(StorageFactory & factory); #endif +#endif #if USE_HDFS void registerStorageHDFS(StorageFactory & factory); @@ -125,8 +127,12 @@ void registerStorages() registerStorageOSS(factory); registerStorageHudi(factory); registerStorageDeltaLake(factory); + + #if USE_AVRO registerStorageIceberg(factory); -#endif + #endif + + #endif #if USE_HDFS registerStorageHDFS(factory); diff --git a/src/TableFunctions/registerTableFunctions.cpp b/src/TableFunctions/registerTableFunctions.cpp index 85c0c475bde..7b2b989e724 100644 --- a/src/TableFunctions/registerTableFunctions.cpp +++ b/src/TableFunctions/registerTableFunctions.cpp @@ -27,10 +27,12 @@ void registerTableFunctions() registerTableFunctionS3(factory); registerTableFunctionS3Cluster(factory); registerTableFunctionCOS(factory); + registerTableFunctionOSS(factory); registerTableFunctionHudi(factory); registerTableFunctionDeltaLake(factory); +#if USE_AVRO registerTableFunctionIceberg(factory); - registerTableFunctionOSS(factory); +#endif #endif diff --git a/src/TableFunctions/registerTableFunctions.h b/src/TableFunctions/registerTableFunctions.h index 88dae5233da..911aae199e2 100644 --- a/src/TableFunctions/registerTableFunctions.h +++ b/src/TableFunctions/registerTableFunctions.h @@ -24,10 +24,12 @@ void registerTableFunctionMeiliSearch(TableFunctionFactory & factory); void registerTableFunctionS3(TableFunctionFactory & factory); void registerTableFunctionS3Cluster(TableFunctionFactory & factory); void registerTableFunctionCOS(TableFunctionFactory & factory); +void registerTableFunctionOSS(TableFunctionFactory & factory); void registerTableFunctionHudi(TableFunctionFactory & factory); void registerTableFunctionDeltaLake(TableFunctionFactory & factory); +#if USE_AVRO void registerTableFunctionIceberg(TableFunctionFactory & factory); -void registerTableFunctionOSS(TableFunctionFactory & factory); +#endif #endif #if USE_HDFS From 1d28da3263c92a28a1dbfbedfe6f0baf3f8b9489 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Sun, 19 Feb 2023 11:56:14 +0100 Subject: [PATCH 415/717] Test for bug with missing column in PREWHERE step --- ...teps_in_prewhere_missing_columns.reference | 30 +++++++++++++++++++ ...read_steps_in_prewhere_missing_columns.sql | 30 +++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere_missing_columns.reference create mode 100644 tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere_missing_columns.sql diff --git a/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere_missing_columns.reference b/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere_missing_columns.reference new file mode 100644 index 00000000000..0c10fa885cd --- /dev/null +++ b/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere_missing_columns.reference @@ -0,0 +1,30 @@ +1 Hello, world! 0 +2 Goodbye. 3 +-- { echoOn } +SELECT s FROM test_02559 PREWHERE x AND y ORDER BY s; +Goodbye. +SELECT s, y FROM test_02559 PREWHERE y ORDER BY s; +Goodbye. 3 +SELECT s, y FROM test_02559 PREWHERE NOT y ORDER BY s; +Hello, world! 0 +SELECT s, y FROM test_02559 PREWHERE x AND y ORDER BY s; +Goodbye. 3 +SELECT s, y FROM test_02559 PREWHERE x AND NOT y ORDER BY s; +Hello, world! 0 +SELECT s, y FROM test_02559 PREWHERE y AND x ORDER BY s; +Goodbye. 3 +SELECT s, y FROM test_02559 PREWHERE (NOT y) AND x ORDER BY s; +Hello, world! 0 +ALTER TABLE test_02559 ADD COLUMN z UInt8 DEFAULT 10; +INSERT INTO test_02559 VALUES (3, 'So long, and thanks for all the fish.', 42, 0); +SELECT * FROM test_02559 ORDER BY x; +1 Hello, world! 0 10 +2 Goodbye. 3 10 +3 So long, and thanks for all the fish. 42 0 +SELECT s FROM test_02559 PREWHERE z ORDER BY s; +Goodbye. +Hello, world! +SELECT s FROM test_02559 PREWHERE y AND z ORDER BY s; +Goodbye. +SELECT s, z FROM test_02559 PREWHERE NOT y AND z ORDER BY s; +Hello, world! 10 diff --git a/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere_missing_columns.sql b/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere_missing_columns.sql new file mode 100644 index 00000000000..f6299122ef2 --- /dev/null +++ b/tests/queries/0_stateless/02559_multiple_read_steps_in_prewhere_missing_columns.sql @@ -0,0 +1,30 @@ +DROP TABLE IF EXISTS test_02559; +CREATE TABLE test_02559 (x UInt8, s String) ENGINE = MergeTree ORDER BY tuple(); + +INSERT INTO test_02559 VALUES (1, 'Hello, world!'); + +ALTER TABLE test_02559 ADD COLUMN y UInt8 DEFAULT 0; +INSERT INTO test_02559 VALUES (2, 'Goodbye.', 3); +SELECT * FROM test_02559 ORDER BY x; + +SET enable_multiple_prewhere_read_steps=true, move_all_conditions_to_prewhere=true; + +-- { echoOn } +SELECT s FROM test_02559 PREWHERE x AND y ORDER BY s; +SELECT s, y FROM test_02559 PREWHERE y ORDER BY s; +SELECT s, y FROM test_02559 PREWHERE NOT y ORDER BY s; +SELECT s, y FROM test_02559 PREWHERE x AND y ORDER BY s; +SELECT s, y FROM test_02559 PREWHERE x AND NOT y ORDER BY s; +SELECT s, y FROM test_02559 PREWHERE y AND x ORDER BY s; +SELECT s, y FROM test_02559 PREWHERE (NOT y) AND x ORDER BY s; + +ALTER TABLE test_02559 ADD COLUMN z UInt8 DEFAULT 10; +INSERT INTO test_02559 VALUES (3, 'So long, and thanks for all the fish.', 42, 0); +SELECT * FROM test_02559 ORDER BY x; + +SELECT s FROM test_02559 PREWHERE z ORDER BY s; +SELECT s FROM test_02559 PREWHERE y AND z ORDER BY s; +SELECT s, z FROM test_02559 PREWHERE NOT y AND z ORDER BY s; +-- { echoOff } + +DROP TABLE test_02559; From 72719584eacff24d74117703dae1c3aa43f72acc Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 15 Feb 2023 15:11:16 +0100 Subject: [PATCH 416/717] Planner filter push down optimization fix --- .../QueryPlan/Optimizations/filterPushDown.cpp | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp index dbf389163be..46fe3055e32 100644 --- a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp +++ b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp @@ -333,8 +333,19 @@ size_t tryPushDownFilter(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes // { // } - if (auto updated_steps = simplePushDownOverStep(parent_node, nodes, child)) - return updated_steps; + if (auto * sorting = typeid_cast(child.get())) + { + const auto & sort_description = sorting->getSortDescription(); + auto sort_description_it = std::find_if(sort_description.begin(), sort_description.end(), [&](auto & sort_column_description) + { + return sort_column_description.column_name == filter->getFilterColumnName(); + }); + bool can_remove_filter = sort_description_it == sort_description.end(); + + Names allowed_inputs = child->getOutputStream().header.getNames(); + if (auto updated_steps = tryAddNewFilterStep(parent_node, nodes, allowed_inputs, can_remove_filter)) + return updated_steps; + } if (auto updated_steps = simplePushDownOverStep(parent_node, nodes, child)) return updated_steps; From f5f220ed812190c1f033adcc3fdcabe1a9455fad Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sun, 19 Feb 2023 12:05:43 +0100 Subject: [PATCH 417/717] Added tests --- ..._predicate_push_down_sorting_fix.reference | 22 +++++++++++++++++++ .../25337_predicate_push_down_sorting_fix.sql | 3 +++ 2 files changed, 25 insertions(+) create mode 100644 tests/queries/0_stateless/25337_predicate_push_down_sorting_fix.reference create mode 100644 tests/queries/0_stateless/25337_predicate_push_down_sorting_fix.sql diff --git a/tests/queries/0_stateless/25337_predicate_push_down_sorting_fix.reference b/tests/queries/0_stateless/25337_predicate_push_down_sorting_fix.reference new file mode 100644 index 00000000000..3d169126eef --- /dev/null +++ b/tests/queries/0_stateless/25337_predicate_push_down_sorting_fix.reference @@ -0,0 +1,22 @@ +Expression ((Project names + (Projection + ))) +Header: number UInt64 +Actions: INPUT : 0 -> number_1 UInt64 : 0 + ALIAS number_1 :: 0 -> number UInt64 : 1 + ALIAS number :: 1 -> number_0 UInt64 : 0 + ALIAS number_0 :: 0 -> number UInt64 : 1 +Positions: 1 + Sorting (Sorting for ORDER BY) + Header: ignore(2_UInt8) UInt8 + number_1 UInt64 + Sort description: ignore(2_UInt8) ASC + Filter (( + (Before ORDER BY + (Projection + Change column names to column identifiers)))) + Header: ignore(2_UInt8) UInt8 + number_1 UInt64 + Filter column: ignore(2_UInt8) + Actions: INPUT : 0 -> number UInt64 : 0 + COLUMN Const(UInt8) -> 2_UInt8 UInt8 : 1 + ALIAS number :: 0 -> number_1 UInt64 : 2 + FUNCTION ignore(2_UInt8 :: 1) -> ignore(2_UInt8) UInt8 : 0 + Positions: 0 2 + ReadFromStorage (SystemNumbers) + Header: number UInt64 diff --git a/tests/queries/0_stateless/25337_predicate_push_down_sorting_fix.sql b/tests/queries/0_stateless/25337_predicate_push_down_sorting_fix.sql new file mode 100644 index 00000000000..2dade7837b7 --- /dev/null +++ b/tests/queries/0_stateless/25337_predicate_push_down_sorting_fix.sql @@ -0,0 +1,3 @@ +SET allow_experimental_analyzer = 1; + +EXPLAIN header = 1, actions = 1 SELECT number FROM (SELECT number FROM numbers(2) ORDER BY ignore(2)) WHERE ignore(2); From 520b3816758e1908170a123730bd1d52dccd7202 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Sun, 19 Feb 2023 12:14:23 +0100 Subject: [PATCH 418/717] Fixes for executing actions on block with unknown row count --- .../MergeTree/MergeTreeRangeReader.cpp | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/Storages/MergeTree/MergeTreeRangeReader.cpp b/src/Storages/MergeTree/MergeTreeRangeReader.cpp index 4036d352a54..44f6cf9f70d 100644 --- a/src/Storages/MergeTree/MergeTreeRangeReader.cpp +++ b/src/Storages/MergeTree/MergeTreeRangeReader.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -921,6 +922,39 @@ bool MergeTreeRangeReader::isCurrentRangeFinished() const return prev_reader ? prev_reader->isCurrentRangeFinished() : stream.isFinished(); } + +/// When executing ExpressionActions on an empty block, it is not possible to determine the number of rows +/// in the block for the new columns so the result block will have 0 rows and it will not match the rest of +/// the columns in the ReadResult. +/// The dummy column is added to maintain the information about the number of rows in the block and to produce +/// the result block with the correct number of rows. +String addDummyColumnWithRowCount(Block & block, size_t num_rows) +{ + bool has_columns = false; + for (const auto & column : block) + { + if (column.column) + { + assert(column.column->size() == num_rows); + has_columns = true; + break; + } + } + + if (has_columns) + return {}; + + ColumnWithTypeAndName dummy_column; + dummy_column.column = DataTypeUInt8().createColumnConst(num_rows, Field(1)); + dummy_column.type = std::make_shared(); + /// Generate a random name to avoid collisions with real columns. + dummy_column.name = "....dummy...." + toString(UUIDHelpers::generateV4()); + block.insert(dummy_column); + + return dummy_column.name; +} + + MergeTreeRangeReader::ReadResult MergeTreeRangeReader::read(size_t max_rows, MarkRanges & ranges) { if (max_rows == 0) @@ -988,6 +1022,7 @@ MergeTreeRangeReader::ReadResult MergeTreeRangeReader::read(size_t max_rows, Mar for (const auto & col : read_result.additional_columns) additional_columns.insert(col); + addDummyColumnWithRowCount(additional_columns, read_result.num_rows); merge_tree_reader->evaluateMissingDefaults(additional_columns, columns); } @@ -1309,8 +1344,17 @@ void MergeTreeRangeReader::executePrewhereActionsAndFilterColumns(ReadResult & r Block additional_columns = block; if (prewhere_info->actions) + { + const String dummy_column = addDummyColumnWithRowCount(block, result.num_rows); + + LOG_TEST(log, "Executing prewhere actions on block: {}", block.dumpStructure()); + prewhere_info->actions->execute(block); + if (!dummy_column.empty()) + block.erase(dummy_column); + } + result.additional_columns.clear(); /// Additional columns might only be needed if there are more steps in the chain. if (!last_reader_in_chain) From 2b9ce319b8a832f314e1ccec128077801c53792a Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy <99031427+yakov-olkhovskiy@users.noreply.github.com> Date: Sun, 19 Feb 2023 08:37:51 -0500 Subject: [PATCH 419/717] remove unnecessary include --- src/Functions/array/FunctionArrayMapped.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Functions/array/FunctionArrayMapped.h b/src/Functions/array/FunctionArrayMapped.h index 3c5cd29fff5..5092698d01c 100644 --- a/src/Functions/array/FunctionArrayMapped.h +++ b/src/Functions/array/FunctionArrayMapped.h @@ -1,6 +1,5 @@ #pragma once -#include #include #include From b27efa928f1cca4eeed5062d2ecb6a99ccb278df Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy <99031427+yakov-olkhovskiy@users.noreply.github.com> Date: Sun, 19 Feb 2023 08:40:00 -0500 Subject: [PATCH 420/717] add EOL --- tests/queries/0_stateless/00390_array_sort.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/00390_array_sort.sql b/tests/queries/0_stateless/00390_array_sort.sql index 563dfb4ed1b..8c2307eb3bd 100644 --- a/tests/queries/0_stateless/00390_array_sort.sql +++ b/tests/queries/0_stateless/00390_array_sort.sql @@ -51,4 +51,4 @@ SELECT arrayPartialSort(2, [1,2,3], [1]); -- { serverError ILLEGAL_TYPE_OF_ARGUM SELECT arrayPartialSort(2, [1,2,3], 3); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } SELECT arrayPartialSort(arraySort([1,2,3]), [1,2,3]); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } SELECT arrayMap(x -> range(x), [4, 1, 2, 3]) AS arr, 100 AS lim, arrayResize(arrayPartialSort(arrayPartialSort(lim, arr), arr), lim), arrayResize(arrayPartialReverseSort(lim, arr), lim), arrayResize(arrayPartialSort(x -> (-length(x)), lim, arr), lim); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } -SELECT arrayPartialReverseSort(arraySort((x, y) -> y, [NULL, NULL], [NULL, NULL]), arr), arrayMap(x -> toString(x), [257, -9223372036854775807, 2, -2147483648, 2147483648, NULL, 65536, -2147483648, 2, 65535]) AS arr, NULL, 100 AS lim, 65536, arrayResize(arrayPartialSort(x -> reverse(x), lim, arr), lim) GROUP BY [NULL, 1023, -2, NULL, 255, '0', NULL, 9223372036854775806] WITH ROLLUP; -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } \ No newline at end of file +SELECT arrayPartialReverseSort(arraySort((x, y) -> y, [NULL, NULL], [NULL, NULL]), arr), arrayMap(x -> toString(x), [257, -9223372036854775807, 2, -2147483648, 2147483648, NULL, 65536, -2147483648, 2, 65535]) AS arr, NULL, 100 AS lim, 65536, arrayResize(arrayPartialSort(x -> reverse(x), lim, arr), lim) GROUP BY [NULL, 1023, -2, NULL, 255, '0', NULL, 9223372036854775806] WITH ROLLUP; -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } From 3a88ec0bf5f58bb1993df34378b89e3ce98dfa57 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 19 Feb 2023 21:02:23 +0100 Subject: [PATCH 421/717] Inhibit randomization in test --- .../0_stateless/01551_mergetree_read_in_order_spread.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01551_mergetree_read_in_order_spread.sql b/tests/queries/0_stateless/01551_mergetree_read_in_order_spread.sql index 1d21d861e20..e374e012238 100644 --- a/tests/queries/0_stateless/01551_mergetree_read_in_order_spread.sql +++ b/tests/queries/0_stateless/01551_mergetree_read_in_order_spread.sql @@ -8,7 +8,7 @@ CREATE TABLE data_01551 ) engine=AggregatingMergeTree() PARTITION BY key%2 ORDER BY (key, key/2) -SETTINGS index_granularity=10; +SETTINGS index_granularity=10, index_granularity_bytes='10Mi'; INSERT INTO data_01551 SELECT number FROM numbers(100000); SET max_threads=3; From d8cda3dbb8f9c2bfcd9bff2993490f0ed1d7499b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 19 Feb 2023 23:15:09 +0100 Subject: [PATCH 422/717] Remove PVS-Studio --- base/base/types.h | 4 ---- base/glibc-compatibility/musl/lgamma.c | 3 --- base/glibc-compatibility/musl/lgammal.c | 3 --- base/glibc-compatibility/musl/libm.h | 6 +++--- base/glibc-compatibility/musl/powf.c | 3 --- base/pcg-random/pcg_extras.hpp | 2 +- base/pcg-random/pcg_random.hpp | 8 ++++---- .../Foundation/include/Poco/LRUStrategy.h | 2 +- src/Access/ExternalAuthenticators.cpp | 6 +++--- src/Access/GrantedRoles.cpp | 2 +- .../AggregateFunctionBoundingRatio.h | 2 +- .../AggregateFunctionHistogram.h | 2 +- .../AggregateFunctionMinMaxAny.h | 2 +- src/Columns/tests/gtest_weak_hash_32.cpp | 12 ++++++------ src/Common/DateLUT.h | 2 +- src/Common/Dwarf.cpp | 4 ++-- src/Common/HashTable/ClearableHashSet.h | 4 ++-- src/Common/HashTable/FixedClearableHashSet.h | 2 +- src/Common/HashTable/FixedHashMap.h | 6 +++--- src/Common/HashTable/FixedHashTable.h | 2 +- src/Common/HashTable/HashSet.h | 4 ++-- src/Common/HashTable/HashTable.h | 2 +- src/Common/HashTable/StringHashTable.h | 2 +- src/Common/JSONBuilder.h | 2 +- src/Common/OpenTelemetryTraceContext.cpp | 4 ++-- src/Common/PoolWithFailoverBase.h | 2 +- src/Common/SpaceSaving.h | 2 +- src/Common/ThreadProfileEvents.h | 2 +- src/Common/Volnitsky.h | 6 +++--- src/Common/ZooKeeper/TestKeeper.cpp | 16 ++++++++-------- src/Common/ZooKeeper/ZooKeeperImpl.cpp | 2 +- src/Common/malloc.cpp | 19 +++++++++---------- .../tests/gtest_thread_pool_global_full.cpp | 4 ++-- .../tests/gtest_compressionCodec.cpp | 2 +- src/Coordination/ZooKeeperDataReader.cpp | 2 +- src/Core/BaseSettings.h | 1 - src/Core/Field.h | 2 +- src/Core/MySQL/MySQLReplication.cpp | 6 +++--- src/Daemon/SentryWriter.cpp | 2 +- .../SerializationLowCardinality.cpp | 2 +- src/DataTypes/convertMySQLDataType.cpp | 4 ++-- src/DataTypes/getLeastSupertype.cpp | 2 +- src/Databases/DatabaseAtomic.cpp | 2 +- .../fetchPostgreSQLTableStructure.cpp | 6 +++--- src/Dictionaries/Embedded/RegionsNames.h | 2 +- src/Dictionaries/HTTPDictionarySource.cpp | 2 +- src/Dictionaries/MySQLDictionarySource.cpp | 2 +- .../PostgreSQLDictionarySource.cpp | 2 +- src/Dictionaries/XDBCDictionarySource.cpp | 2 +- src/Formats/NativeWriter.cpp | 2 +- src/Functions/FunctionHelpers.cpp | 2 +- .../FunctionsBinaryRepresentation.cpp | 4 ++-- src/Functions/FunctionsMiscellaneous.h | 2 +- src/Functions/ifNull.cpp | 2 +- src/Functions/isDecimalOverflow.cpp | 2 +- src/Functions/isNaN.cpp | 3 +-- src/Functions/isValidUTF8.cpp | 2 +- src/Functions/runningAccumulate.cpp | 2 +- src/IO/ReadHelpers.h | 4 ++-- src/IO/WriteHelpers.h | 2 +- src/IO/parseDateTimeBestEffort.cpp | 2 +- src/IO/readFloatText.h | 4 ++-- src/Interpreters/ActionsVisitor.cpp | 2 +- src/Interpreters/Context.cpp | 4 ++-- src/Interpreters/DuplicateOrderByVisitor.cpp | 3 +-- src/Interpreters/ExpressionActions.h | 2 +- src/Interpreters/GroupByFunctionKeysVisitor.h | 2 +- .../InJoinSubqueriesPreprocessor.cpp | 6 +++--- src/Interpreters/InterpreterCreateQuery.cpp | 2 +- src/Interpreters/InterpreterWatchQuery.cpp | 2 +- src/Interpreters/JoinUtils.cpp | 2 +- src/Interpreters/PartLog.cpp | 2 +- src/Interpreters/executeQuery.cpp | 4 ++-- src/Loggers/Loggers.cpp | 2 +- src/Parsers/ASTCreateQuery.cpp | 4 ++-- src/Parsers/IParser.h | 4 ++-- src/Parsers/IParserBase.h | 2 +- .../tests/gtest_alter_command_parser.cpp | 3 +-- src/Parsers/ParserExplainQuery.cpp | 4 ++-- src/Processors/Formats/IRowOutputFormat.cpp | 2 +- src/Processors/QueryPlan/QueryPlan.cpp | 2 +- src/Processors/QueryPlan/SortingStep.cpp | 2 +- ...gingAggregatedMemoryEfficientTransform.cpp | 4 ++-- .../Transforms/buildPushingToViewsChain.cpp | 2 +- src/QueryPipeline/QueryPipelineBuilder.h | 2 +- src/Server/TCPHandler.cpp | 2 +- src/Storages/IStorage.h | 2 +- src/Storages/MergeTree/DataPartsExchange.cpp | 2 +- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 2 +- src/Storages/MergeTree/MergeTask.h | 8 ++++---- src/Storages/MergeTree/MergeTreeData.cpp | 6 +++--- .../MergeTreeDataPartWriterCompact.cpp | 6 +++--- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 4 ++-- src/Storages/MergeTree/MergeTreeIndexSet.cpp | 2 +- src/Storages/MergeTree/MergeTreePartInfo.h | 2 +- .../MergeTree/SimpleMergeSelector.cpp | 2 +- src/Storages/StorageBuffer.cpp | 2 +- src/Storages/StorageFactory.cpp | 2 +- src/Storages/StorageJoin.cpp | 2 +- src/Storages/StorageLog.cpp | 2 +- src/Storages/StorageMergeTree.cpp | 2 +- src/Storages/StorageReplicatedMergeTree.cpp | 4 ++-- 102 files changed, 158 insertions(+), 176 deletions(-) diff --git a/base/base/types.h b/base/base/types.h index e178653f7c6..5825c8ae7ad 100644 --- a/base/base/types.h +++ b/base/base/types.h @@ -13,11 +13,7 @@ using char8_t = unsigned char; #endif /// This is needed for more strict aliasing. https://godbolt.org/z/xpJBSb https://stackoverflow.com/a/57453713 -#if !defined(PVS_STUDIO) /// But PVS-Studio does not treat it correctly. using UInt8 = char8_t; -#else -using UInt8 = uint8_t; -#endif using UInt16 = uint16_t; using UInt32 = uint32_t; diff --git a/base/glibc-compatibility/musl/lgamma.c b/base/glibc-compatibility/musl/lgamma.c index 5e959504e29..fb9d105d0fa 100644 --- a/base/glibc-compatibility/musl/lgamma.c +++ b/base/glibc-compatibility/musl/lgamma.c @@ -78,9 +78,6 @@ * */ -// Disable warnings by PVS-Studio -//-V::GA - static const double pi = 3.14159265358979311600e+00, /* 0x400921FB, 0x54442D18 */ a0 = 7.72156649015328655494e-02, /* 0x3FB3C467, 0xE37DB0C8 */ diff --git a/base/glibc-compatibility/musl/lgammal.c b/base/glibc-compatibility/musl/lgammal.c index 775559f13b6..b158748ce1f 100644 --- a/base/glibc-compatibility/musl/lgammal.c +++ b/base/glibc-compatibility/musl/lgammal.c @@ -85,9 +85,6 @@ * */ -// Disable warnings by PVS-Studio -//-V::GA - #include #include #include "libm.h" diff --git a/base/glibc-compatibility/musl/libm.h b/base/glibc-compatibility/musl/libm.h index e5029318693..55520c2fb03 100644 --- a/base/glibc-compatibility/musl/libm.h +++ b/base/glibc-compatibility/musl/libm.h @@ -155,7 +155,7 @@ static inline long double fp_barrierl(long double x) static inline void fp_force_evalf(float x) { volatile float y; - y = x; //-V1001 + y = x; } #endif @@ -164,7 +164,7 @@ static inline void fp_force_evalf(float x) static inline void fp_force_eval(double x) { volatile double y; - y = x; //-V1001 + y = x; } #endif @@ -173,7 +173,7 @@ static inline void fp_force_eval(double x) static inline void fp_force_evall(long double x) { volatile long double y; - y = x; //-V1001 + y = x; } #endif diff --git a/base/glibc-compatibility/musl/powf.c b/base/glibc-compatibility/musl/powf.c index 35dc3611b94..de8fab54554 100644 --- a/base/glibc-compatibility/musl/powf.c +++ b/base/glibc-compatibility/musl/powf.c @@ -3,9 +3,6 @@ * SPDX-License-Identifier: MIT */ -// Disable warnings by PVS-Studio -//-V::GA - #include #include #include "libm.h" diff --git a/base/pcg-random/pcg_extras.hpp b/base/pcg-random/pcg_extras.hpp index 78ce726d48b..cc11d907006 100644 --- a/base/pcg-random/pcg_extras.hpp +++ b/base/pcg-random/pcg_extras.hpp @@ -455,7 +455,7 @@ auto bounded_rand(RngType& rng, typename RngType::result_type upper_bound) typedef typename RngType::result_type rtype; rtype threshold = (RngType::max() - RngType::min() + rtype(1) - upper_bound) % upper_bound; - for (;;) { //-V1044 + for (;;) { rtype r = rng() - RngType::min(); if (r >= threshold) return r % upper_bound; diff --git a/base/pcg-random/pcg_random.hpp b/base/pcg-random/pcg_random.hpp index db7c3d7f66c..cebceee2d33 100644 --- a/base/pcg-random/pcg_random.hpp +++ b/base/pcg-random/pcg_random.hpp @@ -930,7 +930,7 @@ struct rxs_m_xs_mixin { constexpr bitcount_t shift = bits - xtypebits; constexpr bitcount_t mask = (1 << opbits) - 1; bitcount_t rshift = - opbits ? bitcount_t(internal >> (bits - opbits)) & mask : 0; //-V547 + opbits ? bitcount_t(internal >> (bits - opbits)) & mask : 0; internal ^= internal >> (opbits + rshift); internal *= mcg_multiplier::multiplier(); xtype result = internal >> shift; @@ -952,7 +952,7 @@ struct rxs_m_xs_mixin { internal *= mcg_unmultiplier::unmultiplier(); - bitcount_t rshift = opbits ? (internal >> (bits - opbits)) & mask : 0; //-V547 + bitcount_t rshift = opbits ? (internal >> (bits - opbits)) & mask : 0; internal = unxorshift(internal, bits, opbits + rshift); return internal; @@ -977,7 +977,7 @@ struct rxs_m_mixin { : 2; constexpr bitcount_t shift = bits - xtypebits; constexpr bitcount_t mask = (1 << opbits) - 1; - bitcount_t rshift = opbits ? (internal >> (bits - opbits)) & mask : 0; //-V547 + bitcount_t rshift = opbits ? (internal >> (bits - opbits)) & mask : 0; internal ^= internal >> (opbits + rshift); internal *= mcg_multiplier::multiplier(); xtype result = internal >> shift; @@ -1368,7 +1368,7 @@ void extended::selfinit() // - any strange correlations would only be apparent if we // were to backstep the generator so that the base generator // was generating the same values again - result_type xdiff = baseclass::operator()() - baseclass::operator()(); //-V501 + result_type xdiff = baseclass::operator()() - baseclass::operator()(); for (size_t i = 0; i < table_size; ++i) { data_[i] = baseclass::operator()() ^ xdiff; } diff --git a/base/poco/Foundation/include/Poco/LRUStrategy.h b/base/poco/Foundation/include/Poco/LRUStrategy.h index 04b946f6790..71e457a40f6 100644 --- a/base/poco/Foundation/include/Poco/LRUStrategy.h +++ b/base/poco/Foundation/include/Poco/LRUStrategy.h @@ -113,7 +113,7 @@ public: } std::size_t diff = curSize - _size; - Iterator it = --_keys.end(); //--keys can never be invoked on an empty list due to the minSize==1 requirement of LRU + Iterator it = --_keys.end(); /// --keys can never be invoked on an empty list due to the minSize==1 requirement of LRU std::size_t i = 0; while (i++ < diff) diff --git a/src/Access/ExternalAuthenticators.cpp b/src/Access/ExternalAuthenticators.cpp index a81b5c760d4..e4d4d2acd06 100644 --- a/src/Access/ExternalAuthenticators.cpp +++ b/src/Access/ExternalAuthenticators.cpp @@ -121,7 +121,7 @@ void parseLDAPServer(LDAPClient::Params & params, const Poco::Util::AbstractConf if (enable_tls_lc_str == "starttls") params.enable_tls = LDAPClient::Params::TLSEnable::YES_STARTTLS; else if (config.getBool(ldap_server_config + ".enable_tls")) - params.enable_tls = LDAPClient::Params::TLSEnable::YES; //-V1048 + params.enable_tls = LDAPClient::Params::TLSEnable::YES; else params.enable_tls = LDAPClient::Params::TLSEnable::NO; } @@ -140,7 +140,7 @@ void parseLDAPServer(LDAPClient::Params & params, const Poco::Util::AbstractConf else if (tls_minimum_protocol_version_lc_str == "tls1.1") params.tls_minimum_protocol_version = LDAPClient::Params::TLSProtocolVersion::TLS1_1; else if (tls_minimum_protocol_version_lc_str == "tls1.2") - params.tls_minimum_protocol_version = LDAPClient::Params::TLSProtocolVersion::TLS1_2; //-V1048 + params.tls_minimum_protocol_version = LDAPClient::Params::TLSProtocolVersion::TLS1_2; else throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad value for 'tls_minimum_protocol_version' entry, allowed values are: " @@ -159,7 +159,7 @@ void parseLDAPServer(LDAPClient::Params & params, const Poco::Util::AbstractConf else if (tls_require_cert_lc_str == "try") params.tls_require_cert = LDAPClient::Params::TLSRequireCert::TRY; else if (tls_require_cert_lc_str == "demand") - params.tls_require_cert = LDAPClient::Params::TLSRequireCert::DEMAND; //-V1048 + params.tls_require_cert = LDAPClient::Params::TLSRequireCert::DEMAND; else throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad value for 'tls_require_cert' entry, allowed values are: " diff --git a/src/Access/GrantedRoles.cpp b/src/Access/GrantedRoles.cpp index 4df6809e0fe..bb07cbd6951 100644 --- a/src/Access/GrantedRoles.cpp +++ b/src/Access/GrantedRoles.cpp @@ -137,7 +137,7 @@ GrantedRoles::Elements GrantedRoles::getElements() const boost::range::set_difference(roles, roles_with_admin_option, std::back_inserter(element.ids)); if (!element.empty()) { - element.admin_option = false; //-V1048 + element.admin_option = false; elements.emplace_back(std::move(element)); } diff --git a/src/AggregateFunctions/AggregateFunctionBoundingRatio.h b/src/AggregateFunctions/AggregateFunctionBoundingRatio.h index 04bcdabd3af..935adbf2b7d 100644 --- a/src/AggregateFunctions/AggregateFunctionBoundingRatio.h +++ b/src/AggregateFunctions/AggregateFunctionBoundingRatio.h @@ -20,7 +20,7 @@ namespace ErrorCodes /** Tracks the leftmost and rightmost (x, y) data points. */ -struct AggregateFunctionBoundingRatioData //-V730 +struct AggregateFunctionBoundingRatioData { struct Point { diff --git a/src/AggregateFunctions/AggregateFunctionHistogram.h b/src/AggregateFunctions/AggregateFunctionHistogram.h index 776503acf28..35e5f241ec9 100644 --- a/src/AggregateFunctions/AggregateFunctionHistogram.h +++ b/src/AggregateFunctions/AggregateFunctionHistogram.h @@ -221,7 +221,7 @@ private: } public: - AggregateFunctionHistogramData() //-V730 + AggregateFunctionHistogramData() : size(0) , lower_bound(std::numeric_limits::max()) , upper_bound(std::numeric_limits::lowest()) diff --git a/src/AggregateFunctions/AggregateFunctionMinMaxAny.h b/src/AggregateFunctions/AggregateFunctionMinMaxAny.h index a302c7de35d..b984772c8ea 100644 --- a/src/AggregateFunctions/AggregateFunctionMinMaxAny.h +++ b/src/AggregateFunctions/AggregateFunctionMinMaxAny.h @@ -481,7 +481,7 @@ struct Compatibility /** For strings. Short strings are stored in the object itself, and long strings are allocated separately. * NOTE It could also be suitable for arrays of numbers. */ -struct SingleValueDataString //-V730 +struct SingleValueDataString { private: using Self = SingleValueDataString; diff --git a/src/Columns/tests/gtest_weak_hash_32.cpp b/src/Columns/tests/gtest_weak_hash_32.cpp index 8027bd4d6cc..5755cc3af72 100644 --- a/src/Columns/tests/gtest_weak_hash_32.cpp +++ b/src/Columns/tests/gtest_weak_hash_32.cpp @@ -182,7 +182,7 @@ TEST(WeakHash32, ColumnVectorI32) for (int idx [[maybe_unused]] : {1, 2}) { for (int32_t i = -32768; i < 32768; ++i) - data.push_back(i << 16); //-V610 + data.push_back(i << 16); } WeakHash32 hash(col->size()); @@ -216,7 +216,7 @@ TEST(WeakHash32, ColumnVectorI64) for (int idx [[maybe_unused]] : {1, 2}) { for (int64_t i = -32768; i < 32768; ++i) - data.push_back(i << 32); //-V610 + data.push_back(i << 32); } WeakHash32 hash(col->size()); @@ -258,7 +258,7 @@ TEST(WeakHash32, ColumnVectorI128) for (int idx [[maybe_unused]] : {1, 2}) { for (int64_t i = -32768; i < 32768; ++i) - data.push_back(i << 32); //-V610 + data.push_back(i << 32); } WeakHash32 hash(col->size()); @@ -275,7 +275,7 @@ TEST(WeakHash32, ColumnDecimal32) for (int idx [[maybe_unused]] : {1, 2}) { for (int32_t i = -32768; i < 32768; ++i) - data.push_back(i << 16); //-V610 + data.push_back(i << 16); } WeakHash32 hash(col->size()); @@ -292,7 +292,7 @@ TEST(WeakHash32, ColumnDecimal64) for (int idx [[maybe_unused]] : {1, 2}) { for (int64_t i = -32768; i < 32768; ++i) - data.push_back(i << 32); //-V610 + data.push_back(i << 32); } WeakHash32 hash(col->size()); @@ -309,7 +309,7 @@ TEST(WeakHash32, ColumnDecimal128) for (int idx [[maybe_unused]] : {1, 2}) { for (int64_t i = -32768; i < 32768; ++i) - data.push_back(i << 32); //-V610 + data.push_back(i << 32); } WeakHash32 hash(col->size()); diff --git a/src/Common/DateLUT.h b/src/Common/DateLUT.h index b7ba37c2bec..fe259f0f3c3 100644 --- a/src/Common/DateLUT.h +++ b/src/Common/DateLUT.h @@ -17,7 +17,7 @@ class DateLUT : private boost::noncopyable { public: /// Return singleton DateLUTImpl instance for the default time zone. - static ALWAYS_INLINE const DateLUTImpl & instance() // -V1071 + static ALWAYS_INLINE const DateLUTImpl & instance() { const auto & date_lut = getInstance(); return *date_lut.default_impl.load(std::memory_order_acquire); diff --git a/src/Common/Dwarf.cpp b/src/Common/Dwarf.cpp index 3fca1278ccc..551ed93773f 100644 --- a/src/Common/Dwarf.cpp +++ b/src/Common/Dwarf.cpp @@ -1083,7 +1083,7 @@ bool Dwarf::findLocation( // file+line of the non-inlined outer function making the call. // locationInfo.name is already set by the caller by looking up the // non-inlined function @address belongs to. - info.has_file_and_line = true; //-V1048 + info.has_file_and_line = true; info.file = call_locations[0].file; info.line = call_locations[0].line; @@ -1783,7 +1783,7 @@ void Dwarf::LineNumberVM::init() lineRange_ = read(header); opcodeBase_ = read(header); SAFE_CHECK(opcodeBase_ != 0, "invalid opcode base"); - standardOpcodeLengths_ = reinterpret_cast(header.data()); //-V506 + standardOpcodeLengths_ = reinterpret_cast(header.data()); header.remove_prefix(opcodeBase_ - 1); if (version_ <= 4) diff --git a/src/Common/HashTable/ClearableHashSet.h b/src/Common/HashTable/ClearableHashSet.h index 292f0e4059a..4cbce1a5213 100644 --- a/src/Common/HashTable/ClearableHashSet.h +++ b/src/Common/HashTable/ClearableHashSet.h @@ -44,7 +44,7 @@ struct ClearableHashTableCell : public BaseCell /// Do I need to store the zero key separately (that is, can a zero key be inserted into the hash table). static constexpr bool need_zero_value_storage = false; - ClearableHashTableCell() {} //-V730 /// NOLINT + ClearableHashTableCell() {} /// NOLINT ClearableHashTableCell(const Key & key_, const State & state) : BaseCell(key_, state), version(state.version) {} }; @@ -68,7 +68,7 @@ struct ClearableHashTableCell : public StringRefBa /// Do I need to store the zero key separately (that is, can a zero key be inserted into the hash table). static constexpr bool need_zero_value_storage = true; - ClearableHashTableCell() { } //-V730 /// NOLINT + ClearableHashTableCell() { } /// NOLINT ClearableHashTableCell(const StringRef & key_, const State & state) : StringRefBaseCell(key_, state), version(state.version) { } }; diff --git a/src/Common/HashTable/FixedClearableHashSet.h b/src/Common/HashTable/FixedClearableHashSet.h index 1faac136359..30af7ebe0f5 100644 --- a/src/Common/HashTable/FixedClearableHashSet.h +++ b/src/Common/HashTable/FixedClearableHashSet.h @@ -13,7 +13,7 @@ struct FixedClearableHashTableCell using mapped_type = VoidMapped; UInt32 version; - FixedClearableHashTableCell() {} //-V730 /// NOLINT + FixedClearableHashTableCell() {} /// NOLINT FixedClearableHashTableCell(const Key &, const State & state) : version(state.version) {} const VoidKey getKey() const { return {}; } /// NOLINT diff --git a/src/Common/HashTable/FixedHashMap.h b/src/Common/HashTable/FixedHashMap.h index 40b76d9b4eb..e835a6fba94 100644 --- a/src/Common/HashTable/FixedHashMap.h +++ b/src/Common/HashTable/FixedHashMap.h @@ -16,7 +16,7 @@ struct FixedHashMapCell bool full; Mapped mapped; - FixedHashMapCell() {} //-V730 /// NOLINT + FixedHashMapCell() {} /// NOLINT FixedHashMapCell(const Key &, const State &) : full(true) {} FixedHashMapCell(const value_type & value_, const State &) : full(true), mapped(value_.second) {} @@ -31,7 +31,7 @@ struct FixedHashMapCell /// Note that we have to assemble a continuous layout for the value_type on each call of getValue(). struct CellExt { - CellExt() {} //-V730 /// NOLINT + CellExt() {} /// NOLINT CellExt(Key && key_, const FixedHashMapCell * ptr_) : key(key_), ptr(const_cast(ptr_)) {} void update(Key && key_, const FixedHashMapCell * ptr_) { @@ -76,7 +76,7 @@ struct FixedHashMapImplicitZeroCell /// Note that we have to assemble a continuous layout for the value_type on each call of getValue(). struct CellExt { - CellExt() {} //-V730 /// NOLINT + CellExt() {} /// NOLINT CellExt(Key && key_, const FixedHashMapImplicitZeroCell * ptr_) : key(key_), ptr(const_cast(ptr_)) {} void update(Key && key_, const FixedHashMapImplicitZeroCell * ptr_) { diff --git a/src/Common/HashTable/FixedHashTable.h b/src/Common/HashTable/FixedHashTable.h index 12e4a2b3dba..7df90fd98b9 100644 --- a/src/Common/HashTable/FixedHashTable.h +++ b/src/Common/HashTable/FixedHashTable.h @@ -19,7 +19,7 @@ struct FixedHashTableCell using mapped_type = VoidMapped; bool full; - FixedHashTableCell() {} //-V730 /// NOLINT + FixedHashTableCell() {} /// NOLINT FixedHashTableCell(const Key &, const State &) : full(true) {} const VoidKey getKey() const { return {}; } /// NOLINT diff --git a/src/Common/HashTable/HashSet.h b/src/Common/HashTable/HashSet.h index be4be078ee8..bac858b16a5 100644 --- a/src/Common/HashTable/HashSet.h +++ b/src/Common/HashTable/HashSet.h @@ -121,8 +121,8 @@ struct HashSetCellWithSavedHash : public HashTableCell size_t saved_hash; - HashSetCellWithSavedHash() : Base() {} //-V730 - HashSetCellWithSavedHash(const Key & key_, const typename Base::State & state) : Base(key_, state) {} //-V730 + HashSetCellWithSavedHash() : Base() {} + HashSetCellWithSavedHash(const Key & key_, const typename Base::State & state) : Base(key_, state) {} bool keyEquals(const Key & key_) const { return bitEquals(this->key, key_); } bool keyEquals(const Key & key_, size_t hash_) const { return saved_hash == hash_ && bitEquals(this->key, key_); } diff --git a/src/Common/HashTable/HashTable.h b/src/Common/HashTable/HashTable.h index 75f848b51c3..5c348f936d2 100644 --- a/src/Common/HashTable/HashTable.h +++ b/src/Common/HashTable/HashTable.h @@ -369,7 +369,7 @@ template struct ZeroValueStorage; template -struct ZeroValueStorage //-V730 +struct ZeroValueStorage { private: bool has_zero = false; diff --git a/src/Common/HashTable/StringHashTable.h b/src/Common/HashTable/StringHashTable.h index bbcfae1ca84..a9d078f6cf3 100644 --- a/src/Common/HashTable/StringHashTable.h +++ b/src/Common/HashTable/StringHashTable.h @@ -92,7 +92,7 @@ struct StringHashTableHash }; template -struct StringHashTableEmpty //-V730 +struct StringHashTableEmpty { using Self = StringHashTableEmpty; diff --git a/src/Common/JSONBuilder.h b/src/Common/JSONBuilder.h index 38d19da011d..1690ae7bb77 100644 --- a/src/Common/JSONBuilder.h +++ b/src/Common/JSONBuilder.h @@ -95,7 +95,7 @@ class JSONMap : public IItem }; public: - void add(std::string key, ItemPtr value) { values.emplace_back(Pair{.key = std::move(key), .value = std::move(value)}); } //-V1030 + void add(std::string key, ItemPtr value) { values.emplace_back(Pair{.key = std::move(key), .value = std::move(value)}); } void add(std::string key, std::string value) { add(std::move(key), std::make_unique(std::move(value))); } void add(std::string key, const char * value) { add(std::move(key), std::make_unique(value)); } void add(std::string key, std::string_view value) { add(std::move(key), std::make_unique(value)); } diff --git a/src/Common/OpenTelemetryTraceContext.cpp b/src/Common/OpenTelemetryTraceContext.cpp index 37455f2266d..b62822ceda2 100644 --- a/src/Common/OpenTelemetryTraceContext.cpp +++ b/src/Common/OpenTelemetryTraceContext.cpp @@ -320,8 +320,8 @@ TracingContextHolder::TracingContextHolder( while (_parent_trace_context.trace_id == UUID()) { // Make sure the random generated trace_id is not 0 which is an invalid id. - _parent_trace_context.trace_id.toUnderType().items[0] = thread_local_rng(); //-V656 - _parent_trace_context.trace_id.toUnderType().items[1] = thread_local_rng(); //-V656 + _parent_trace_context.trace_id.toUnderType().items[0] = thread_local_rng(); + _parent_trace_context.trace_id.toUnderType().items[1] = thread_local_rng(); } _parent_trace_context.span_id = 0; } diff --git a/src/Common/PoolWithFailoverBase.h b/src/Common/PoolWithFailoverBase.h index fabfd4517d6..0e8fbb4e6d1 100644 --- a/src/Common/PoolWithFailoverBase.h +++ b/src/Common/PoolWithFailoverBase.h @@ -381,7 +381,7 @@ void PoolWithFailoverBase::updateErrorCounts(PoolWithFailoverBase stats_getter; explicit TasksStatsCounters(UInt64 tid, MetricsProvider provider); diff --git a/src/Common/Volnitsky.h b/src/Common/Volnitsky.h index a27fd36f704..7073a9a4709 100644 --- a/src/Common/Volnitsky.h +++ b/src/Common/Volnitsky.h @@ -193,8 +193,8 @@ namespace VolnitskyTraits chars.c1 = seq_l[seq_ngram_offset + 1]; putNGramBase(n, offset); - chars.c0 = seq_r[seq_ngram_offset]; //-V519 - chars.c1 = seq_r[seq_ngram_offset + 1]; //-V519 + chars.c0 = seq_r[seq_ngram_offset]; + chars.c1 = seq_r[seq_ngram_offset + 1]; putNGramBase(n, offset); } @@ -317,7 +317,7 @@ namespace VolnitskyTraits { /// ngram for Ul chars.c0 = c0u; - chars.c1 = c1l; //-V1048 + chars.c1 = c1l; putNGramBase(n, offset); } diff --git a/src/Common/ZooKeeper/TestKeeper.cpp b/src/Common/ZooKeeper/TestKeeper.cpp index 4f53a8ac307..fe4cb83c78a 100644 --- a/src/Common/ZooKeeper/TestKeeper.cpp +++ b/src/Common/ZooKeeper/TestKeeper.cpp @@ -212,7 +212,7 @@ std::pair TestKeeperCreateRequest::process(TestKeeper::Contai else { TestKeeper::Node created_node; - created_node.seq_num = 0; //-V1048 + created_node.seq_num = 0; created_node.stat.czxid = zxid; created_node.stat.mzxid = zxid; created_node.stat.ctime = std::chrono::system_clock::now().time_since_epoch() / std::chrono::milliseconds(1); @@ -286,7 +286,7 @@ std::pair TestKeeperRemoveRequest::process(TestKeeper::Contai auto & parent = container.at(parentPath(path)); --parent.stat.numChildren; ++parent.stat.cversion; - response.error = Error::ZOK; //-V1048 + response.error = Error::ZOK; undo = [prev_node, &container, path = path] { @@ -308,7 +308,7 @@ std::pair TestKeeperExistsRequest::process(TestKeeper::Contai if (it != container.end()) { response.stat = it->second.stat; - response.error = Error::ZOK; //-V1048 + response.error = Error::ZOK; } else { @@ -331,7 +331,7 @@ std::pair TestKeeperGetRequest::process(TestKeeper::Container { response.stat = it->second.stat; response.data = it->second.data; - response.error = Error::ZOK; //-V1048 + response.error = Error::ZOK; } return { std::make_shared(response), {} }; @@ -358,7 +358,7 @@ std::pair TestKeeperSetRequest::process(TestKeeper::Container it->second.data = data; ++container.at(parentPath(path)).stat.cversion; response.stat = it->second.stat; - response.error = Error::ZOK; //-V1048 + response.error = Error::ZOK; undo = [prev_node, &container, path = path] { @@ -412,7 +412,7 @@ std::pair TestKeeperListRequest::process(TestKeeper::Containe } response.stat = it->second.stat; - response.error = Error::ZOK; //-V1048 + response.error = Error::ZOK; } return { std::make_shared(response), {} }; @@ -432,7 +432,7 @@ std::pair TestKeeperCheckRequest::process(TestKeeper::Contain } else { - response.error = Error::ZOK; //-V1048 + response.error = Error::ZOK; } return { std::make_shared(response), {} }; @@ -455,7 +455,7 @@ std::pair TestKeeperMultiRequest::process(TestKeeper::Contain try { auto request_it = requests.begin(); - response.error = Error::ZOK; //-V1048 + response.error = Error::ZOK; while (request_it != requests.end()) { const TestKeeperRequest & concrete_request = dynamic_cast(**request_it); diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index 8e72e83dfe4..1fbdd857379 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -826,7 +826,7 @@ void ZooKeeper::receiveEvent() if (length != actual_length) throw Exception(Error::ZMARSHALLINGERROR, "Response length doesn't match. Expected: {}, actual: {}", length, actual_length); - logOperationIfNeeded(request_info.request, response, /* finalize= */ false, elapsed_ms); //-V614 + logOperationIfNeeded(request_info.request, response, /* finalize= */ false, elapsed_ms); } catch (...) { diff --git a/src/Common/malloc.cpp b/src/Common/malloc.cpp index 2de639a5c2b..ff6f2b80686 100644 --- a/src/Common/malloc.cpp +++ b/src/Common/malloc.cpp @@ -32,17 +32,16 @@ static void dummyFunctionForInterposing() __attribute__((used)); static void dummyFunctionForInterposing() { void* dummy; - /// Suppression for PVS-Studio and clang-tidy. - free(nullptr); // -V575 NOLINT - ignore(malloc(0)); // -V575 NOLINT - ignore(calloc(0, 0)); // -V575 NOLINT - ignore(realloc(nullptr, 0)); // -V575 NOLINT - ignore(posix_memalign(&dummy, 0, 0)); // -V575 NOLINT - ignore(aligned_alloc(1, 0)); // -V575 NOLINT - ignore(valloc(0)); // -V575 NOLINT - ignore(memalign(0, 0)); // -V575 NOLINT + free(nullptr); // NOLINT + ignore(malloc(0)); // NOLINT + ignore(calloc(0, 0)); // NOLINT + ignore(realloc(nullptr, 0)); // NOLINT + ignore(posix_memalign(&dummy, 0, 0)); // NOLINT + ignore(aligned_alloc(1, 0)); // NOLINT + ignore(valloc(0)); // NOLINT + ignore(memalign(0, 0)); // NOLINT #if !defined(USE_MUSL) - ignore(pvalloc(0)); // -V575 NOLINT + ignore(pvalloc(0)); // NOLINT #endif } #endif diff --git a/src/Common/tests/gtest_thread_pool_global_full.cpp b/src/Common/tests/gtest_thread_pool_global_full.cpp index c6133f7ca11..583d43be1bb 100644 --- a/src/Common/tests/gtest_thread_pool_global_full.cpp +++ b/src/Common/tests/gtest_thread_pool_global_full.cpp @@ -25,7 +25,7 @@ TEST(ThreadPool, GlobalFull1) std::atomic counter = 0; static constexpr size_t num_jobs = capacity + 1; - auto func = [&] { ++counter; while (counter != num_jobs) {} }; //-V776 + auto func = [&] { ++counter; while (counter != num_jobs) {} }; ThreadPool pool(num_jobs); @@ -63,7 +63,7 @@ TEST(ThreadPool, GlobalFull2) global_pool.wait(); std::atomic counter = 0; - auto func = [&] { ++counter; while (counter != capacity + 1) {} }; //-V776 + auto func = [&] { ++counter; while (counter != capacity + 1) {} }; ThreadPool pool(capacity, 0, capacity); for (size_t i = 0; i < capacity; ++i) diff --git a/src/Compression/tests/gtest_compressionCodec.cpp b/src/Compression/tests/gtest_compressionCodec.cpp index dba6c10a6d6..340b3b6b8f8 100644 --- a/src/Compression/tests/gtest_compressionCodec.cpp +++ b/src/Compression/tests/gtest_compressionCodec.cpp @@ -1130,7 +1130,7 @@ template auto DDCompatibilityTestSequence() { // Generates sequences with double delta in given range. - auto dd_generator = [prev_delta = static_cast(0), prev = static_cast(0)](auto dd) mutable //-V788 + auto dd_generator = [prev_delta = static_cast(0), prev = static_cast(0)](auto dd) mutable { const auto curr = dd + prev + prev_delta; prev = curr; diff --git a/src/Coordination/ZooKeeperDataReader.cpp b/src/Coordination/ZooKeeperDataReader.cpp index 3b803c18dbf..5fa67a60b4b 100644 --- a/src/Coordination/ZooKeeperDataReader.cpp +++ b/src/Coordination/ZooKeeperDataReader.cpp @@ -465,7 +465,7 @@ bool hasErrorsInMultiRequest(Coordination::ZooKeeperRequestPtr request) if (request == nullptr) return true; - for (const auto & subrequest : dynamic_cast(request.get())->requests) // -V522 + for (const auto & subrequest : dynamic_cast(request.get())->requests) if (subrequest == nullptr) return true; return false; diff --git a/src/Core/BaseSettings.h b/src/Core/BaseSettings.h index cae8cecccc9..c49bdcf2739 100644 --- a/src/Core/BaseSettings.h +++ b/src/Core/BaseSettings.h @@ -1018,7 +1018,6 @@ struct DefineAliases \ template class BaseSettings; -//-V:IMPLEMENT_SETTINGS:501 /// NOLINTNEXTLINE #define IMPLEMENT_SETTINGS_TRAITS_(TYPE, NAME, DEFAULT, DESCRIPTION, FLAGS) \ res.field_infos.emplace_back( \ diff --git a/src/Core/Field.h b/src/Core/Field.h index e25692c53fb..95ce43ccd44 100644 --- a/src/Core/Field.h +++ b/src/Core/Field.h @@ -895,7 +895,7 @@ auto & Field::safeGet() template -Field::Field(T && rhs, enable_if_not_field_or_bool_or_stringlike_t) //-V730 +Field::Field(T && rhs, enable_if_not_field_or_bool_or_stringlike_t) { auto && val = castToNearestFieldType(std::forward(rhs)); createConcrete(std::forward(val)); diff --git a/src/Core/MySQL/MySQLReplication.cpp b/src/Core/MySQL/MySQLReplication.cpp index 13272338e5c..1ee027b7185 100644 --- a/src/Core/MySQL/MySQLReplication.cpp +++ b/src/Core/MySQL/MySQLReplication.cpp @@ -581,9 +581,9 @@ namespace MySQLReplication { if (precision <= DecimalUtils::max_precision) return Field(function(precision, scale, Decimal32())); - else if (precision <= DecimalUtils::max_precision) //-V547 + else if (precision <= DecimalUtils::max_precision) return Field(function(precision, scale, Decimal64())); - else if (precision <= DecimalUtils::max_precision) //-V547 + else if (precision <= DecimalUtils::max_precision) return Field(function(precision, scale, Decimal128())); return Field(function(precision, scale, Decimal256())); @@ -649,7 +649,7 @@ namespace MySQLReplication UInt32 val = 0; size_t to_read = compressed_bytes_map[compressed_decimals]; - if (to_read) //-V547 + if (to_read) { readBigEndianStrict(payload, reinterpret_cast(&val), to_read); res *= intExp10OfSize(static_cast(compressed_decimals)); diff --git a/src/Daemon/SentryWriter.cpp b/src/Daemon/SentryWriter.cpp index bb330162818..9f4f18e64d1 100644 --- a/src/Daemon/SentryWriter.cpp +++ b/src/Daemon/SentryWriter.cpp @@ -74,7 +74,7 @@ void SentryWriter::initialize(Poco::Util::LayeredConfiguration & config) if (config.getBool("send_crash_reports.enabled", false)) { - if (debug || (strlen(VERSION_OFFICIAL) > 0)) //-V560 + if (debug || (strlen(VERSION_OFFICIAL) > 0)) enabled = true; } diff --git a/src/DataTypes/Serializations/SerializationLowCardinality.cpp b/src/DataTypes/Serializations/SerializationLowCardinality.cpp index 17b0227a592..90b51c1a6f2 100644 --- a/src/DataTypes/Serializations/SerializationLowCardinality.cpp +++ b/src/DataTypes/Serializations/SerializationLowCardinality.cpp @@ -184,7 +184,7 @@ struct IndexesSerializationType return std::make_shared(); if (type == TUInt32) return std::make_shared(); - if (type == TUInt64) //-V547 + if (type == TUInt64) return std::make_shared(); throw Exception(ErrorCodes::LOGICAL_ERROR, "Can't create DataType from IndexesSerializationType."); diff --git a/src/DataTypes/convertMySQLDataType.cpp b/src/DataTypes/convertMySQLDataType.cpp index 307ff317204..db064a7f06f 100644 --- a/src/DataTypes/convertMySQLDataType.cpp +++ b/src/DataTypes/convertMySQLDataType.cpp @@ -111,9 +111,9 @@ DataTypePtr convertMySQLDataType(MultiEnum type_support, { if (precision <= DecimalUtils::max_precision) res = std::make_shared>(precision, scale); - else if (precision <= DecimalUtils::max_precision) //-V547 + else if (precision <= DecimalUtils::max_precision) res = std::make_shared>(precision, scale); - else if (precision <= DecimalUtils::max_precision) //-V547 + else if (precision <= DecimalUtils::max_precision) res = std::make_shared>(precision, scale); } diff --git a/src/DataTypes/getLeastSupertype.cpp b/src/DataTypes/getLeastSupertype.cpp index cbba822af04..b50aeb76e67 100644 --- a/src/DataTypes/getLeastSupertype.cpp +++ b/src/DataTypes/getLeastSupertype.cpp @@ -129,7 +129,7 @@ DataTypePtr getNumericType(const TypeIndexSet & types) size_t min_bit_width_of_integer = std::max(max_bits_of_signed_integer, max_bits_of_unsigned_integer); /// If unsigned is not covered by signed. - if (max_bits_of_signed_integer && max_bits_of_unsigned_integer >= max_bits_of_signed_integer) //-V1051 + if (max_bits_of_signed_integer && max_bits_of_unsigned_integer >= max_bits_of_signed_integer) { // Because 128 and 256 bit integers are significantly slower, we should not promote to them. // But if we already have wide numbers, promotion is necessary. diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp index ee4ed319abe..34c4fd3d5d8 100644 --- a/src/Databases/DatabaseAtomic.cpp +++ b/src/Databases/DatabaseAtomic.cpp @@ -106,7 +106,7 @@ StoragePtr DatabaseAtomic::detachTable(ContextPtr /* context */, const String & auto table = DatabaseOrdinary::detachTableUnlocked(name); table_name_to_path.erase(name); detached_tables.emplace(table->getStorageID().uuid, table); - not_in_use = cleanupDetachedTables(); //-V1001 + not_in_use = cleanupDetachedTables(); return table; } diff --git a/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp b/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp index eeae110cddf..4d8de325902 100644 --- a/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp +++ b/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp @@ -115,11 +115,11 @@ static DataTypePtr convertPostgreSQLDataType(String & type, Fn auto && r if (precision <= DecimalUtils::max_precision) res = std::make_shared>(precision, scale); - else if (precision <= DecimalUtils::max_precision) //-V547 + else if (precision <= DecimalUtils::max_precision) res = std::make_shared>(precision, scale); - else if (precision <= DecimalUtils::max_precision) //-V547 + else if (precision <= DecimalUtils::max_precision) res = std::make_shared>(precision, scale); - else if (precision <= DecimalUtils::max_precision) //-V547 + else if (precision <= DecimalUtils::max_precision) res = std::make_shared>(precision, scale); else throw Exception(ErrorCodes::BAD_ARGUMENTS, "Precision {} and scale {} are too big and not supported", precision, scale); diff --git a/src/Dictionaries/Embedded/RegionsNames.h b/src/Dictionaries/Embedded/RegionsNames.h index ec06a0b1a33..1e0ea3f0923 100644 --- a/src/Dictionaries/Embedded/RegionsNames.h +++ b/src/Dictionaries/Embedded/RegionsNames.h @@ -84,7 +84,7 @@ public: { size_t language_id = static_cast(language); - if (region_id >= names_refs[language_id].size()) //-V1051 + if (region_id >= names_refs[language_id].size()) return StringRef("", 0); StringRef ref = names_refs[language_id][region_id]; diff --git a/src/Dictionaries/HTTPDictionarySource.cpp b/src/Dictionaries/HTTPDictionarySource.cpp index 7e0056c9cae..d2d3b344df5 100644 --- a/src/Dictionaries/HTTPDictionarySource.cpp +++ b/src/Dictionaries/HTTPDictionarySource.cpp @@ -293,7 +293,7 @@ void registerDictionarySourceHTTP(DictionarySourceFactory & factory) .format = format, .update_field = config.getString(settings_config_prefix + ".update_field", ""), .update_lag = config.getUInt64(settings_config_prefix + ".update_lag", 1), - .header_entries = std::move(header_entries) //-V1030 + .header_entries = std::move(header_entries) }; auto context = copyContextAndApplySettingsFromDictionaryConfig(global_context, config, config_prefix); diff --git a/src/Dictionaries/MySQLDictionarySource.cpp b/src/Dictionaries/MySQLDictionarySource.cpp index 6fb4dffc5e7..c8491d99255 100644 --- a/src/Dictionaries/MySQLDictionarySource.cpp +++ b/src/Dictionaries/MySQLDictionarySource.cpp @@ -233,7 +233,7 @@ bool MySQLDictionarySource::isModified() const if (!configuration.invalidate_query.empty()) { auto response = doInvalidateQuery(configuration.invalidate_query); - if (response == invalidate_query_response) //-V1051 + if (response == invalidate_query_response) return false; invalidate_query_response = response; diff --git a/src/Dictionaries/PostgreSQLDictionarySource.cpp b/src/Dictionaries/PostgreSQLDictionarySource.cpp index 95bfb1a37d8..9f254da0b11 100644 --- a/src/Dictionaries/PostgreSQLDictionarySource.cpp +++ b/src/Dictionaries/PostgreSQLDictionarySource.cpp @@ -117,7 +117,7 @@ bool PostgreSQLDictionarySource::isModified() const if (!configuration.invalidate_query.empty()) { auto response = doInvalidateQuery(configuration.invalidate_query); - if (response == invalidate_query_response) //-V1051 + if (response == invalidate_query_response) return false; invalidate_query_response = response; } diff --git a/src/Dictionaries/XDBCDictionarySource.cpp b/src/Dictionaries/XDBCDictionarySource.cpp index 36dde92699a..23dc7db508d 100644 --- a/src/Dictionaries/XDBCDictionarySource.cpp +++ b/src/Dictionaries/XDBCDictionarySource.cpp @@ -178,7 +178,7 @@ bool XDBCDictionarySource::isModified() const if (!configuration.invalidate_query.empty()) { auto response = doInvalidateQuery(configuration.invalidate_query); - if (invalidate_query_response == response) //-V1051 + if (invalidate_query_response == response) return false; invalidate_query_response = response; } diff --git a/src/Formats/NativeWriter.cpp b/src/Formats/NativeWriter.cpp index c287e9280dd..8100a3868e6 100644 --- a/src/Formats/NativeWriter.cpp +++ b/src/Formats/NativeWriter.cpp @@ -55,7 +55,7 @@ static void writeData(const ISerialization & serialization, const ColumnPtr & co ISerialization::SerializeBinaryBulkSettings settings; settings.getter = [&ostr](ISerialization::SubstreamPath) -> WriteBuffer * { return &ostr; }; settings.position_independent_encoding = false; - settings.low_cardinality_max_dictionary_size = 0; //-V1048 + settings.low_cardinality_max_dictionary_size = 0; ISerialization::SerializeBinaryBulkStatePtr state; serialization.serializeBinaryBulkStatePrefix(*full_column, settings, state); diff --git a/src/Functions/FunctionHelpers.cpp b/src/Functions/FunctionHelpers.cpp index 791b2c1bbdb..c981f666219 100644 --- a/src/Functions/FunctionHelpers.cpp +++ b/src/Functions/FunctionHelpers.cpp @@ -264,7 +264,7 @@ ColumnPtr wrapInNullable(const ColumnPtr & src, const ColumnsWithTypeAndName & a if (const auto * nullable = checkAndGetColumn(*elem.column)) { const ColumnPtr & null_map_column = nullable->getNullMapColumnPtr(); - if (!result_null_map_column) //-V1051 + if (!result_null_map_column) { result_null_map_column = null_map_column; } diff --git a/src/Functions/FunctionsBinaryRepresentation.cpp b/src/Functions/FunctionsBinaryRepresentation.cpp index fccdcf455ce..d44323f8bf3 100644 --- a/src/Functions/FunctionsBinaryRepresentation.cpp +++ b/src/Functions/FunctionsBinaryRepresentation.cpp @@ -51,7 +51,7 @@ struct HexImpl UInt8 byte = x >> offset; /// Skip leading zeros - if (byte == 0 && !was_nonzero && offset && skip_leading_zero) //-V560 + if (byte == 0 && !was_nonzero && offset && skip_leading_zero) continue; was_nonzero = true; @@ -144,7 +144,7 @@ struct BinImpl UInt8 byte = x >> offset; /// Skip leading zeros - if (byte == 0 && !was_nonzero && offset && skip_leading_zero) //-V560 + if (byte == 0 && !was_nonzero && offset && skip_leading_zero) continue; was_nonzero = true; diff --git a/src/Functions/FunctionsMiscellaneous.h b/src/Functions/FunctionsMiscellaneous.h index 4869d366317..75c91a2e964 100644 --- a/src/Functions/FunctionsMiscellaneous.h +++ b/src/Functions/FunctionsMiscellaneous.h @@ -243,7 +243,7 @@ public: capture = std::make_shared(Capture{ .captured_names = captured_names_, - .captured_types = std::move(captured_types), //-V1030 + .captured_types = std::move(captured_types), .lambda_arguments = lambda_arguments_, .return_name = expression_return_name_, .return_type = function_return_type_, diff --git a/src/Functions/ifNull.cpp b/src/Functions/ifNull.cpp index ef301a9662e..1093f3f817f 100644 --- a/src/Functions/ifNull.cpp +++ b/src/Functions/ifNull.cpp @@ -79,7 +79,7 @@ public: arguments[1], }; - auto func_if = FunctionFactory::instance().get("if", context)->build(if_columns); //-V557 + auto func_if = FunctionFactory::instance().get("if", context)->build(if_columns); return func_if->execute(if_columns, result_type, input_rows_count); } diff --git a/src/Functions/isDecimalOverflow.cpp b/src/Functions/isDecimalOverflow.cpp index 504ece7794f..93485b32d03 100644 --- a/src/Functions/isDecimalOverflow.cpp +++ b/src/Functions/isDecimalOverflow.cpp @@ -87,7 +87,7 @@ public: auto result_column = ColumnUInt8::create(); - auto call = [&](const auto & types) -> bool //-V657 + auto call = [&](const auto & types) -> bool { using Types = std::decay_t; using Type = typename Types::RightType; diff --git a/src/Functions/isNaN.cpp b/src/Functions/isNaN.cpp index e6ab425a4d3..1d710294442 100644 --- a/src/Functions/isNaN.cpp +++ b/src/Functions/isNaN.cpp @@ -13,8 +13,7 @@ struct IsNaNImpl template static bool execute(const T t) { - /// Suppression for PVS-Studio. - return t != t; //-V501 + return t != t; } }; diff --git a/src/Functions/isValidUTF8.cpp b/src/Functions/isValidUTF8.cpp index 0871e82adb8..e7aba672356 100644 --- a/src/Functions/isValidUTF8.cpp +++ b/src/Functions/isValidUTF8.cpp @@ -203,7 +203,7 @@ SOFTWARE. len -= 16; }; - while (len >= 16) // NOLINT //-V1044 + while (len >= 16) // NOLINT check_packed(_mm_loadu_si128(reinterpret_cast(data))); /// 0 <= len <= 15 for now. Reading data from data - 1 because of right padding of 15 and left padding diff --git a/src/Functions/runningAccumulate.cpp b/src/Functions/runningAccumulate.cpp index b6fc92a7eb9..b0ba10c4049 100644 --- a/src/Functions/runningAccumulate.cpp +++ b/src/Functions/runningAccumulate.cpp @@ -125,7 +125,7 @@ public: } agg_func.create(place.data()); /// This function can throw. - state_created = true; //-V519 + state_created = true; } agg_func.merge(place.data(), state_to_add, arena.get()); diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index fd547220069..f8931a7f622 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -174,7 +174,7 @@ void assertNotEOF(ReadBuffer & buf); [[noreturn]] void throwAtAssertionFailed(const char * s, ReadBuffer & buf); -inline bool checkChar(char c, ReadBuffer & buf) // -V1071 +inline bool checkChar(char c, ReadBuffer & buf) { char a; if (!buf.peek(a) || a != c) @@ -458,7 +458,7 @@ void readIntText(T & x, ReadBuffer & buf) } template -bool tryReadIntText(T & x, ReadBuffer & buf) // -V1071 +bool tryReadIntText(T & x, ReadBuffer & buf) { return readIntTextImpl(x, buf); } diff --git a/src/IO/WriteHelpers.h b/src/IO/WriteHelpers.h index ebc2a9e34c1..1c0b48c53c3 100644 --- a/src/IO/WriteHelpers.h +++ b/src/IO/WriteHelpers.h @@ -840,7 +840,7 @@ inline void writeDateTimeUnixTimestamp(DateTime64 datetime64, UInt32 scale, Writ auto components = DecimalUtils::split(datetime64, scale); writeIntText(components.whole, buf); - if (scale > 0) //-V547 + if (scale > 0) { buf.write('.'); writeDateTime64FractionalText(components.fractional, scale, buf); diff --git a/src/IO/parseDateTimeBestEffort.cpp b/src/IO/parseDateTimeBestEffort.cpp index e32f50f7450..d4a7d04540f 100644 --- a/src/IO/parseDateTimeBestEffort.cpp +++ b/src/IO/parseDateTimeBestEffort.cpp @@ -287,7 +287,7 @@ ReturnType parseDateTimeBestEffortImpl( UInt8 hour_or_day_of_month_or_month = 0; if (num_digits == 2) readDecimalNumber<2>(hour_or_day_of_month_or_month, digits); - else if (num_digits == 1) //-V547 + else if (num_digits == 1) readDecimalNumber<1>(hour_or_day_of_month_or_month, digits); else return on_error(ErrorCodes::LOGICAL_ERROR, "Cannot read DateTime: logical error, unexpected branch in code"); diff --git a/src/IO/readFloatText.h b/src/IO/readFloatText.h index c4cd46463a3..da4719b8dcb 100644 --- a/src/IO/readFloatText.h +++ b/src/IO/readFloatText.h @@ -145,7 +145,7 @@ template ReturnType readFloatTextPreciseImpl(T & x, ReadBuffer & buf) { static_assert(std::is_same_v || std::is_same_v, "Argument for readFloatTextPreciseImpl must be float or double"); - static_assert('a' > '.' && 'A' > '.' && '\n' < '.' && '\t' < '.' && '\'' < '.' && '"' < '.', "Layout of char is not like ASCII"); //-V590 + static_assert('a' > '.' && 'A' > '.' && '\n' < '.' && '\t' < '.' && '\'' < '.' && '"' < '.', "Layout of char is not like ASCII"); static constexpr bool throw_exception = std::is_same_v; @@ -317,7 +317,7 @@ template ReturnType readFloatTextFastImpl(T & x, ReadBuffer & in) { static_assert(std::is_same_v || std::is_same_v, "Argument for readFloatTextImpl must be float or double"); - static_assert('a' > '.' && 'A' > '.' && '\n' < '.' && '\t' < '.' && '\'' < '.' && '"' < '.', "Layout of char is not like ASCII"); //-V590 + static_assert('a' > '.' && 'A' > '.' && '\n' < '.' && '\t' < '.' && '\'' < '.' && '"' < '.', "Layout of char is not like ASCII"); static constexpr bool throw_exception = std::is_same_v; diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index 2974b5c0ee0..eee8a01d86b 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -275,7 +275,7 @@ static Block createBlockFromAST(const ASTPtr & node, const DataTypes & types, Co assert(tuple || func); - size_t tuple_size = tuple ? tuple->size() : func->arguments->children.size(); //-V1004 + size_t tuple_size = tuple ? tuple->size() : func->arguments->children.size(); if (tuple_size != num_columns) throw Exception(ErrorCodes::INCORRECT_ELEMENT_OF_SET, "Incorrect size of tuple in set: {} instead of {}", tuple_size, num_columns); diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 0feac593acd..33505e41789 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -1650,8 +1650,8 @@ void Context::setCurrentQueryId(const String & query_id) UUID uuid{}; } random; - random.words.a = thread_local_rng(); //-V656 - random.words.b = thread_local_rng(); //-V656 + random.words.a = thread_local_rng(); + random.words.b = thread_local_rng(); String query_id_to_set = query_id; diff --git a/src/Interpreters/DuplicateOrderByVisitor.cpp b/src/Interpreters/DuplicateOrderByVisitor.cpp index 569253ff78d..409e91259fc 100644 --- a/src/Interpreters/DuplicateOrderByVisitor.cpp +++ b/src/Interpreters/DuplicateOrderByVisitor.cpp @@ -99,7 +99,7 @@ void DuplicateOrderByData::visit(ASTSelectQuery & select_query, ASTPtr &) bool is_stateful = false; ASTFunctionStatefulVisitor::Data data{context, is_stateful}; ASTFunctionStatefulVisitor(data).visit(elem); - if (is_stateful) //-V547 + if (is_stateful) return; } } @@ -119,4 +119,3 @@ void DuplicateOrderByData::visit(ASTSelectQuery & select_query, ASTPtr &) } } - diff --git a/src/Interpreters/ExpressionActions.h b/src/Interpreters/ExpressionActions.h index 48f18b3b407..11957997a30 100644 --- a/src/Interpreters/ExpressionActions.h +++ b/src/Interpreters/ExpressionActions.h @@ -254,7 +254,7 @@ struct ExpressionActionsChain : WithContext steps.clear(); } - ActionsDAGPtr getLastActions(bool allow_empty = false) // -V1071 + ActionsDAGPtr getLastActions(bool allow_empty = false) { if (steps.empty()) { diff --git a/src/Interpreters/GroupByFunctionKeysVisitor.h b/src/Interpreters/GroupByFunctionKeysVisitor.h index e21e71d7a72..b1d40320a45 100644 --- a/src/Interpreters/GroupByFunctionKeysVisitor.h +++ b/src/Interpreters/GroupByFunctionKeysVisitor.h @@ -95,7 +95,7 @@ public: KeepFunctionVisitor::Data keep_data{data.key_names_to_keep, keep_key}; KeepFunctionVisitor(keep_data).visit(function_node->arguments); - if (!keep_key) //-V547 + if (!keep_key) (data.key_names_to_keep).erase(function_node->getColumnName()); } diff --git a/src/Interpreters/InJoinSubqueriesPreprocessor.cpp b/src/Interpreters/InJoinSubqueriesPreprocessor.cpp index ca804fe84a3..3858830a43b 100644 --- a/src/Interpreters/InJoinSubqueriesPreprocessor.cpp +++ b/src/Interpreters/InJoinSubqueriesPreprocessor.cpp @@ -179,7 +179,7 @@ private: std::vector renamed; NonGlobalTableVisitor::Data table_data(data.getContext(), data.checker, renamed, &node, nullptr); NonGlobalTableVisitor(table_data).visit(subquery); - if (!renamed.empty()) //-V547 + if (!renamed.empty()) data.renamed_tables.emplace_back(subquery, std::move(renamed)); } } @@ -199,7 +199,7 @@ private: std::vector renamed; NonGlobalTableVisitor::Data table_data(data.getContext(), data.checker, renamed, nullptr, table_join); NonGlobalTableVisitor(table_data).visit(subquery); - if (!renamed.empty()) //-V547 + if (!renamed.empty()) data.renamed_tables.emplace_back(subquery, std::move(renamed)); } else if (table->database_and_table_name) @@ -208,7 +208,7 @@ private: std::vector renamed; NonGlobalTableVisitor::Data table_data{data.getContext(), data.checker, renamed, nullptr, table_join}; NonGlobalTableVisitor(table_data).visit(tb); - if (!renamed.empty()) //-V547 + if (!renamed.empty()) data.renamed_tables.emplace_back(tb, std::move(renamed)); } } diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index cdd4d836c14..559fc3532f5 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -312,7 +312,7 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) { /// We use global context here, because storages lifetime is bigger than query context lifetime - TablesLoader loader{getContext()->getGlobalContext(), {{database_name, database}}, mode}; //-V560 + TablesLoader loader{getContext()->getGlobalContext(), {{database_name, database}}, mode}; loader.loadTables(); loader.startupTables(); } diff --git a/src/Interpreters/InterpreterWatchQuery.cpp b/src/Interpreters/InterpreterWatchQuery.cpp index b2086831e4e..e1af704a358 100644 --- a/src/Interpreters/InterpreterWatchQuery.cpp +++ b/src/Interpreters/InterpreterWatchQuery.cpp @@ -41,7 +41,7 @@ BlockIO InterpreterWatchQuery::execute() const Settings & settings = getContext()->getSettingsRef(); StreamLocalLimits limits; - limits.mode = LimitsMode::LIMITS_CURRENT; //-V1048 + limits.mode = LimitsMode::LIMITS_CURRENT; limits.size_limits.max_rows = settings.max_result_rows; limits.size_limits.max_bytes = settings.max_result_bytes; limits.size_limits.overflow_mode = settings.result_overflow_mode; diff --git a/src/Interpreters/JoinUtils.cpp b/src/Interpreters/JoinUtils.cpp index b8d8dd5df74..a05b58e14a1 100644 --- a/src/Interpreters/JoinUtils.cpp +++ b/src/Interpreters/JoinUtils.cpp @@ -486,7 +486,7 @@ void createMissedColumns(Block & block) for (size_t i = 0; i < block.columns(); ++i) { auto & column = block.getByPosition(i); - if (!column.column) //-V1051 + if (!column.column) column.column = column.type->createColumn(); } } diff --git a/src/Interpreters/PartLog.cpp b/src/Interpreters/PartLog.cpp index 4a1349680fd..249a14710ba 100644 --- a/src/Interpreters/PartLog.cpp +++ b/src/Interpreters/PartLog.cpp @@ -197,7 +197,7 @@ bool PartLog::addNewParts( if (!query_id.empty()) elem.query_id.insert(0, query_id.data(), query_id.size()); - elem.event_type = PartLogElement::NEW_PART; //-V1048 + elem.event_type = PartLogElement::NEW_PART; // construct event_time and event_time_microseconds using the same time point // so that the two times will always be equal up to a precision of a second. diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 3bb46462353..ca7544df4b9 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -691,7 +691,7 @@ static std::tuple executeQueryImpl( if (!interpreter->ignoreLimits()) { - limits.mode = LimitsMode::LIMITS_CURRENT; //-V1048 + limits.mode = LimitsMode::LIMITS_CURRENT; limits.size_limits = SizeLimits(settings.max_result_rows, settings.max_result_bytes, settings.result_overflow_mode); } @@ -795,7 +795,7 @@ static std::tuple executeQueryImpl( { QueryLogElement elem; - elem.type = QueryLogElementType::QUERY_START; //-V1048 + elem.type = QueryLogElementType::QUERY_START; elem.event_time = timeInSeconds(query_start_time); elem.event_time_microseconds = timeInMicroseconds(query_start_time); diff --git a/src/Loggers/Loggers.cpp b/src/Loggers/Loggers.cpp index b3bd8d588cd..645ae5dcc7a 100644 --- a/src/Loggers/Loggers.cpp +++ b/src/Loggers/Loggers.cpp @@ -51,7 +51,7 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log #endif auto current_logger = config.getString("logger", ""); - if (config_logger == current_logger) //-V1051 + if (config_logger == current_logger) return; config_logger = current_logger; diff --git a/src/Parsers/ASTCreateQuery.cpp b/src/Parsers/ASTCreateQuery.cpp index e74be6c66c5..955ce62b0f7 100644 --- a/src/Parsers/ASTCreateQuery.cpp +++ b/src/Parsers/ASTCreateQuery.cpp @@ -365,7 +365,7 @@ void ASTCreateQuery::formatQueryImpl(const FormatSettings & settings, FormatStat FormatStateStacked frame_nested = frame; columns_list->formatImpl(settings, state, frame_nested); settings.ostr << (settings.one_line ? ")" : "\n)"); - frame.expression_list_always_start_on_new_line = false; //-V519 + frame.expression_list_always_start_on_new_line = false; } settings.ostr << (settings.hilite ? hilite_keyword : "") << " AS " << (settings.hilite ? hilite_none : ""); @@ -393,7 +393,7 @@ void ASTCreateQuery::formatQueryImpl(const FormatSettings & settings, FormatStat settings.ostr << (settings.one_line ? ")" : "\n)"); } - frame.expression_list_always_start_on_new_line = false; //-V519 + frame.expression_list_always_start_on_new_line = false; if (inner_storage) { diff --git a/src/Parsers/IParser.h b/src/Parsers/IParser.h index d5fdf6b7eaa..dbd292bcd9f 100644 --- a/src/Parsers/IParser.h +++ b/src/Parsers/IParser.h @@ -95,13 +95,13 @@ public: */ virtual bool parse(Pos & pos, ASTPtr & node, Expected & expected) = 0; - bool ignore(Pos & pos, Expected & expected) // -V1071 + bool ignore(Pos & pos, Expected & expected) { ASTPtr ignore_node; return parse(pos, ignore_node, expected); } - bool ignore(Pos & pos) // -V1071 + bool ignore(Pos & pos) { Expected expected; return ignore(pos, expected); diff --git a/src/Parsers/IParserBase.h b/src/Parsers/IParserBase.h index 6fd195da40d..46f0e672e0d 100644 --- a/src/Parsers/IParserBase.h +++ b/src/Parsers/IParserBase.h @@ -35,7 +35,7 @@ public: return res; } - bool parse(Pos & pos, ASTPtr & node, Expected & expected) override; // -V1071 + bool parse(Pos & pos, ASTPtr & node, Expected & expected) override; protected: virtual bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) = 0; diff --git a/src/Parsers/MySQL/tests/gtest_alter_command_parser.cpp b/src/Parsers/MySQL/tests/gtest_alter_command_parser.cpp index ab838cdbc53..d406cdbd3b9 100644 --- a/src/Parsers/MySQL/tests/gtest_alter_command_parser.cpp +++ b/src/Parsers/MySQL/tests/gtest_alter_command_parser.cpp @@ -9,7 +9,7 @@ using namespace DB; using namespace DB::MySQLParser; -static inline ASTPtr tryParserQuery(IParser & parser, const String & query) // -V1071 +static inline ASTPtr tryParserQuery(IParser & parser, const String & query) { return parseQuery(parser, query.data(), query.data() + query.size(), "", 0, 0); } @@ -261,4 +261,3 @@ TEST(ParserAlterCommand, AlterOptionsCommand) EXPECT_THROW(tryParserQuery(alter_p, "FORCE ALGORITHM DEFAULT"), Exception); EXPECT_THROW(tryParserQuery(alter_p, "ALGORITHM DEFAULT AUTO_INCREMENT 1"), Exception); } - diff --git a/src/Parsers/ParserExplainQuery.cpp b/src/Parsers/ParserExplainQuery.cpp index 7fc997f9548..148b671af71 100644 --- a/src/Parsers/ParserExplainQuery.cpp +++ b/src/Parsers/ParserExplainQuery.cpp @@ -39,9 +39,9 @@ bool ParserExplainQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected else if (s_pipeline.ignore(pos, expected)) kind = ASTExplainQuery::ExplainKind::QueryPipeline; else if (s_plan.ignore(pos, expected)) - kind = ASTExplainQuery::ExplainKind::QueryPlan; //-V1048 + kind = ASTExplainQuery::ExplainKind::QueryPlan; else if (s_estimates.ignore(pos, expected)) - kind = ASTExplainQuery::ExplainKind::QueryEstimates; //-V1048 + kind = ASTExplainQuery::ExplainKind::QueryEstimates; else if (s_table_override.ignore(pos, expected)) kind = ASTExplainQuery::ExplainKind::TableOverride; else if (s_current_transaction.ignore(pos, expected)) diff --git a/src/Processors/Formats/IRowOutputFormat.cpp b/src/Processors/Formats/IRowOutputFormat.cpp index 3c0692945d4..51e1ca89fce 100644 --- a/src/Processors/Formats/IRowOutputFormat.cpp +++ b/src/Processors/Formats/IRowOutputFormat.cpp @@ -86,7 +86,7 @@ void IRowOutputFormat::writeMinExtreme(const DB::Columns & columns, size_t row_n write(columns, row_num); } -void IRowOutputFormat::writeMaxExtreme(const DB::Columns & columns, size_t row_num) //-V524 +void IRowOutputFormat::writeMaxExtreme(const DB::Columns & columns, size_t row_num) { write(columns, row_num); } diff --git a/src/Processors/QueryPlan/QueryPlan.cpp b/src/Processors/QueryPlan/QueryPlan.cpp index 48a9fbd7a34..8b666bba7da 100644 --- a/src/Processors/QueryPlan/QueryPlan.cpp +++ b/src/Processors/QueryPlan/QueryPlan.cpp @@ -177,7 +177,7 @@ QueryPipelineBuilderPtr QueryPlan::buildQueryPipeline( if (last_pipeline) { frame.pipelines.emplace_back(std::move(last_pipeline)); - last_pipeline = nullptr; //-V1048 + last_pipeline = nullptr; } size_t next_child = frame.pipelines.size(); diff --git a/src/Processors/QueryPlan/SortingStep.cpp b/src/Processors/QueryPlan/SortingStep.cpp index 97f5f573a0e..0308e320e3a 100644 --- a/src/Processors/QueryPlan/SortingStep.cpp +++ b/src/Processors/QueryPlan/SortingStep.cpp @@ -238,7 +238,7 @@ void SortingStep::fullSortStreams( }); StreamLocalLimits limits; - limits.mode = LimitsMode::LIMITS_CURRENT; //-V1048 + limits.mode = LimitsMode::LIMITS_CURRENT; limits.size_limits = sort_settings.size_limits; pipeline.addSimpleTransform( diff --git a/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp b/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp index 9771c24f256..a50bdff0de9 100644 --- a/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp +++ b/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp @@ -224,12 +224,12 @@ IProcessor::Status GroupingAggregatedTransform::prepare() /// Sanity check. If new bucket was read, we should be able to push it. /// This is always false, but we still keep this condition in case the code will be changed. - if (!all_inputs_finished) // -V547 + if (!all_inputs_finished) throw Exception(ErrorCodes::LOGICAL_ERROR, "GroupingAggregatedTransform has read new two-level bucket, but couldn't push it."); } else { - if (!all_inputs_finished) // -V547 + if (!all_inputs_finished) throw Exception(ErrorCodes::LOGICAL_ERROR, "GroupingAggregatedTransform should have read all chunks for single level aggregation, " "but not all of the inputs are finished."); diff --git a/src/Processors/Transforms/buildPushingToViewsChain.cpp b/src/Processors/Transforms/buildPushingToViewsChain.cpp index 33d6a1f2403..a58d70a8428 100644 --- a/src/Processors/Transforms/buildPushingToViewsChain.cpp +++ b/src/Processors/Transforms/buildPushingToViewsChain.cpp @@ -368,7 +368,7 @@ Chain buildPushingToViewsChain( out = buildPushingToViewsChain( view, view_metadata_snapshot, insert_context, ASTPtr(), false, thread_status_holder, view_counter_ms); - views_data->views.emplace_back(ViewRuntimeData{ //-V614 + views_data->views.emplace_back(ViewRuntimeData{ std::move(query), out.getInputHeader(), view_id, diff --git a/src/QueryPipeline/QueryPipelineBuilder.h b/src/QueryPipeline/QueryPipelineBuilder.h index 0a102d186ca..4f984680c75 100644 --- a/src/QueryPipeline/QueryPipelineBuilder.h +++ b/src/QueryPipeline/QueryPipelineBuilder.h @@ -162,7 +162,7 @@ public: { auto num_threads = pipe.maxParallelStreams(); - if (max_threads) //-V1051 + if (max_threads) num_threads = std::min(num_threads, max_threads); return std::max(1, num_threads); diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index f76c342fa9a..870a34fa665 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -895,7 +895,7 @@ void TCPHandler::processTablesStatusRequest() status.absolute_delay = static_cast(replicated_table->getAbsoluteDelay()); } else - status.is_replicated = false; //-V1048 + status.is_replicated = false; response.table_states_by_id.emplace(table_name, std::move(status)); } diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index a479f3ccb22..426d64f41ee 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -98,7 +98,7 @@ public: /// Storage metadata can be set separately in setInMemoryMetadata method explicit IStorage(StorageID storage_id_) : storage_id(std::move(storage_id_)) - , metadata(std::make_unique()) {} //-V730 + , metadata(std::make_unique()) {} IStorage(const IStorage &) = delete; IStorage & operator=(const IStorage &) = delete; diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index cbbdb911974..b671106f46a 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -814,7 +814,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDisk( } /// We will remove directory if it's already exists. Make precautions. - if (tmp_prefix.empty() //-V560 + if (tmp_prefix.empty() || part_name.empty() || std::string::npos != tmp_prefix.find_first_of("/.") || std::string::npos != part_name.find_first_of("/.")) diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 5de13020a1d..06b4cb07f1f 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -750,7 +750,7 @@ void IMergeTreeDataPart::loadIndex() for (size_t j = 0; j < key_size; ++j) key_serializations[j] = primary_key.data_types[j]->getDefaultSerialization(); - for (size_t i = 0; i < marks_count; ++i) //-V756 + for (size_t i = 0; i < marks_count; ++i) for (size_t j = 0; j < key_size; ++j) key_serializations[j]->deserializeBinary(*loaded_index[j], *index_file, {}); diff --git a/src/Storages/MergeTree/MergeTask.h b/src/Storages/MergeTree/MergeTask.h index ad035ed4ecf..9b777345c1d 100644 --- a/src/Storages/MergeTree/MergeTask.h +++ b/src/Storages/MergeTree/MergeTask.h @@ -124,7 +124,7 @@ private: /// By default this context is uninitialed, but some variables has to be set after construction, /// some variables are used in a process of execution /// Proper initialization is responsibility of the author - struct GlobalRuntimeContext : public IStageRuntimeContext //-V730 + struct GlobalRuntimeContext : public IStageRuntimeContext { MergeList::Entry * merge_entry{nullptr}; /// If not null, use this instead of the global MergeList::Entry. This is for merging projections. @@ -186,7 +186,7 @@ private: /// By default this context is uninitialed, but some variables has to be set after construction, /// some variables are used in a process of execution /// Proper initialization is responsibility of the author - struct ExecuteAndFinalizeHorizontalPartRuntimeContext : public IStageRuntimeContext //-V730 + struct ExecuteAndFinalizeHorizontalPartRuntimeContext : public IStageRuntimeContext { /// Dependencies String suffix; @@ -259,7 +259,7 @@ private: /// By default this context is uninitialed, but some variables has to be set after construction, /// some variables are used in a process of execution /// Proper initialization is responsibility of the author - struct VerticalMergeRuntimeContext : public IStageRuntimeContext //-V730 + struct VerticalMergeRuntimeContext : public IStageRuntimeContext { /// Begin dependencies from previous stage std::unique_ptr rows_sources_file; @@ -331,7 +331,7 @@ private: /// By default this context is uninitialed, but some variables has to be set after construction, /// some variables are used in a process of execution /// Proper initialization is responsibility of the author - struct MergeProjectionsRuntimeContext : public IStageRuntimeContext //-V730 + struct MergeProjectionsRuntimeContext : public IStageRuntimeContext { /// Only one dependency bool need_sync{false}; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 7961d85b867..b7c48a7ced1 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -2175,7 +2175,7 @@ void MergeTreeData::removePartsFinally(const MergeTreeData::DataPartsVector & pa part_log_elem.event_time = timeInSeconds(time_now); part_log_elem.event_time_microseconds = timeInMicroseconds(time_now); - part_log_elem.duration_ms = 0; //-V1048 + part_log_elem.duration_ms = 0; part_log_elem.database_name = table_id.database_name; part_log_elem.table_name = table_id.table_name; @@ -3808,7 +3808,7 @@ MergeTreeData::PartsToRemoveFromZooKeeper MergeTreeData::removePartsInRangeFromW void MergeTreeData::restoreAndActivatePart(const DataPartPtr & part, DataPartsLock * acquired_lock) { - auto lock = (acquired_lock) ? DataPartsLock() : lockParts(); //-V1018 + auto lock = (acquired_lock) ? DataPartsLock() : lockParts(); if (part->getState() == DataPartState::Active) return; addPartContributionToColumnAndSecondaryIndexSizes(part); @@ -5294,7 +5294,7 @@ MergeTreeData::DataPartsVector MergeTreeData::getDataPartsVectorForInternalUsage auto range = getDataPartsStateRange(state); std::swap(buf, res); res.clear(); - std::merge(range.begin(), range.end(), buf.begin(), buf.end(), std::back_inserter(res), LessDataPart()); //-V783 + std::merge(range.begin(), range.end(), buf.begin(), buf.end(), std::back_inserter(res), LessDataPart()); } if (out_states != nullptr) diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index 94c3651f1f9..0b650eb9f16 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -139,8 +139,8 @@ void writeColumnSingleGranule( ISerialization::SerializeBinaryBulkSettings serialize_settings; serialize_settings.getter = stream_getter; - serialize_settings.position_independent_encoding = true; //-V1048 - serialize_settings.low_cardinality_max_dictionary_size = 0; //-V1048 + serialize_settings.position_independent_encoding = true; + serialize_settings.low_cardinality_max_dictionary_size = 0; serialization->serializeBinaryBulkStatePrefix(*column.column, serialize_settings, state); serialization->serializeBinaryBulkWithMultipleStreams(*column.column, from_row, number_of_rows, serialize_settings, state); @@ -236,7 +236,7 @@ void MergeTreeDataPartWriterCompact::writeDataBlock(const Block & block, const G stream_getter, granule.start_row, granule.rows_to_write); /// Each type always have at least one substream - prev_stream->hashing_buf.next(); //-V522 + prev_stream->hashing_buf.next(); } writeIntBinary(granule.rows_to_write, marks_out); diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index a108ee18931..4fe470efcce 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -1169,7 +1169,7 @@ RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipInd index_stats.emplace_back(ReadFromMergeTree::IndexStat{ .type = ReadFromMergeTree::IndexType::Skip, .name = index_name, - .description = std::move(description), //-V1030 + .description = std::move(description), .num_parts_after = index_and_condition.stat.total_parts - index_and_condition.stat.parts_dropped, .num_granules_after = index_and_condition.stat.total_granules - index_and_condition.stat.granules_dropped}); } @@ -1186,7 +1186,7 @@ RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipInd index_stats.emplace_back(ReadFromMergeTree::IndexStat{ .type = ReadFromMergeTree::IndexType::Skip, .name = index_name, - .description = std::move(description), //-V1030 + .description = std::move(description), .num_parts_after = index_and_condition.stat.total_parts - index_and_condition.stat.parts_dropped, .num_granules_after = index_and_condition.stat.total_granules - index_and_condition.stat.granules_dropped}); } diff --git a/src/Storages/MergeTree/MergeTreeIndexSet.cpp b/src/Storages/MergeTree/MergeTreeIndexSet.cpp index d28272b6d73..901636a2de9 100644 --- a/src/Storages/MergeTree/MergeTreeIndexSet.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexSet.cpp @@ -73,7 +73,7 @@ void MergeTreeIndexGranuleSet::serializeBinary(WriteBuffer & ostr) const ISerialization::SerializeBinaryBulkSettings settings; settings.getter = [&ostr](ISerialization::SubstreamPath) -> WriteBuffer * { return &ostr; }; settings.position_independent_encoding = false; - settings.low_cardinality_max_dictionary_size = 0; //-V1048 + settings.low_cardinality_max_dictionary_size = 0; auto serialization = type->getDefaultSerialization(); ISerialization::SerializeBinaryBulkStatePtr state; diff --git a/src/Storages/MergeTree/MergeTreePartInfo.h b/src/Storages/MergeTree/MergeTreePartInfo.h index b91bc1e595b..a869701ae20 100644 --- a/src/Storages/MergeTree/MergeTreePartInfo.h +++ b/src/Storages/MergeTree/MergeTreePartInfo.h @@ -121,7 +121,7 @@ struct MergeTreePartInfo /// Simple sanity check for partition ID. Checking that it's not too long or too short, doesn't contain a lot of '_'. static void validatePartitionID(const String & partition_id, MergeTreeDataFormatVersion format_version); - static MergeTreePartInfo fromPartName(const String & part_name, MergeTreeDataFormatVersion format_version); // -V1071 + static MergeTreePartInfo fromPartName(const String & part_name, MergeTreeDataFormatVersion format_version); static std::optional tryParsePartName( std::string_view part_name, MergeTreeDataFormatVersion format_version); diff --git a/src/Storages/MergeTree/SimpleMergeSelector.cpp b/src/Storages/MergeTree/SimpleMergeSelector.cpp index f9ed6aedc60..15291622a2a 100644 --- a/src/Storages/MergeTree/SimpleMergeSelector.cpp +++ b/src/Storages/MergeTree/SimpleMergeSelector.cpp @@ -173,7 +173,7 @@ void selectWithinPartition( for (size_t end = begin + 2; end <= parts_count; ++end) { assert(end > begin); - if (settings.max_parts_to_merge_at_once && end - begin > settings.max_parts_to_merge_at_once) //-V658 + if (settings.max_parts_to_merge_at_once && end - begin > settings.max_parts_to_merge_at_once) break; if (!parts[end - 1].shall_participate_in_merges) diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index d62c568c8c7..a69f26203e9 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -879,7 +879,7 @@ bool StorageBuffer::flushBuffer(Buffer & buffer, bool check_thresholds, bool loc buffer.data.swap(block_to_write); - if (!buffer.first_write_time) // -V547 + if (!buffer.first_write_time) buffer.first_write_time = current_time; /// After a while, the next write attempt will happen. diff --git a/src/Storages/StorageFactory.cpp b/src/Storages/StorageFactory.cpp index ff141bf108f..f9bc25ef72c 100644 --- a/src/Storages/StorageFactory.cpp +++ b/src/Storages/StorageFactory.cpp @@ -223,7 +223,7 @@ StoragePtr StorageFactory::get( assert(arguments.getContext() == arguments.getContext()->getGlobalContext()); auto res = storages.at(name).creator_fn(arguments); - if (!empty_engine_args.empty()) //-V547 + if (!empty_engine_args.empty()) { /// Storage creator modified empty arguments list, so we should modify the query assert(storage_def && storage_def->engine && !storage_def->engine->arguments); diff --git a/src/Storages/StorageJoin.cpp b/src/Storages/StorageJoin.cpp index b57e717c272..ab1406f9bd6 100644 --- a/src/Storages/StorageJoin.cpp +++ b/src/Storages/StorageJoin.cpp @@ -552,7 +552,7 @@ private: if (!position) position = decltype(position)( - static_cast(new typename Map::const_iterator(map.begin())), //-V572 + static_cast(new typename Map::const_iterator(map.begin())), [](void * ptr) { delete reinterpret_cast(ptr); }); auto & it = *reinterpret_cast(position.get()); diff --git a/src/Storages/StorageLog.cpp b/src/Storages/StorageLog.cpp index f2eca42ed0b..338fb54c7e5 100644 --- a/src/Storages/StorageLog.cpp +++ b/src/Storages/StorageLog.cpp @@ -204,7 +204,7 @@ void LogSource::readData(const NameAndTypePair & name_and_type, ColumnPtr & colu auto create_stream_getter = [&](bool stream_for_prefix) { - return [&, stream_for_prefix] (const ISerialization::SubstreamPath & path) -> ReadBuffer * //-V1047 + return [&, stream_for_prefix] (const ISerialization::SubstreamPath & path) -> ReadBuffer * { if (cache.contains(ISerialization::getSubcolumnNameForStream(path))) return nullptr; diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index e42ad123551..065e88c9399 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -1191,7 +1191,7 @@ UInt32 StorageMergeTree::getMaxLevelInBetween(const DataPartPtr & left, const Da return level; } -bool StorageMergeTree::scheduleDataProcessingJob(BackgroundJobsAssignee & assignee) //-V657 +bool StorageMergeTree::scheduleDataProcessingJob(BackgroundJobsAssignee & assignee) { if (shutdown_called) return false; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index db99007cb4f..976bec7e29d 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -6117,7 +6117,7 @@ void StorageReplicatedMergeTree::fetchPartition( ContextPtr query_context) { Macros::MacroExpansionInfo info; - info.expand_special_macros_only = false; //-V1048 + info.expand_special_macros_only = false; info.table_id = getStorageID(); info.table_id.uuid = UUIDHelpers::Nil; auto expand_from = query_context->getMacros()->expand(from_, info); @@ -7227,7 +7227,7 @@ void StorageReplicatedMergeTree::movePartitionToTable(const StoragePtr & dest_ta entry_delete.type = LogEntry::DROP_RANGE; entry_delete.source_replica = replica_name; entry_delete.new_part_name = drop_range_fake_part_name; - entry_delete.detach = false; //-V1048 + entry_delete.detach = false; entry_delete.create_time = time(nullptr); } From 28c020692acad462fcd87cf1e010b13b739a8225 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 19 Feb 2023 23:42:00 +0100 Subject: [PATCH 423/717] Inhibit settings randomization in `01304_direct_io_long.sh` --- tests/queries/0_stateless/01304_direct_io_long.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01304_direct_io_long.sh b/tests/queries/0_stateless/01304_direct_io_long.sh index 57638a7d0a6..bd503f40396 100755 --- a/tests/queries/0_stateless/01304_direct_io_long.sh +++ b/tests/queries/0_stateless/01304_direct_io_long.sh @@ -7,7 +7,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) $CLICKHOUSE_CLIENT --multiquery --query " DROP TABLE IF EXISTS bug; - CREATE TABLE bug (UserID UInt64, Date Date) ENGINE = MergeTree ORDER BY Date; + CREATE TABLE bug (UserID UInt64, Date Date) ENGINE = MergeTree ORDER BY Date + SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi', merge_max_block_size = 8192; INSERT INTO bug SELECT rand64(), '2020-06-07' FROM numbers(50000000); OPTIMIZE TABLE bug FINAL;" LOG="$CLICKHOUSE_TMP/err-$CLICKHOUSE_DATABASE" From 66984088d6931a1023354f37a9d1b5931758c756 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 19 Feb 2023 23:53:50 +0100 Subject: [PATCH 424/717] Fix double whitespace in comment in test --- .../0_stateless/01081_PartialSortingTransform_full_column.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01081_PartialSortingTransform_full_column.sql b/tests/queries/0_stateless/01081_PartialSortingTransform_full_column.sql index eaf1278d9ba..59ab5595577 100644 --- a/tests/queries/0_stateless/01081_PartialSortingTransform_full_column.sql +++ b/tests/queries/0_stateless/01081_PartialSortingTransform_full_column.sql @@ -6,7 +6,7 @@ select 1 from remote('127.{1,2}', currentDatabase(), test_01081) lhs join system -- With multiple blocks triggers: -- --- Code: 171. DB::Exception: Received from localhost:9000. DB::Exception: Received from 127.2:9000. DB::Exception: Block structure mismatch in function connect between PartialSortingTransform and LazyOutputFormat stream: different columns: +-- Code: 171. DB::Exception: Received from localhost:9000. DB::Exception: Received from 127.2:9000. DB::Exception: Block structure mismatch in function connect between PartialSortingTransform and LazyOutputFormat stream: different columns: -- _dummy Int Int32(size = 0), 1 UInt8 UInt8(size = 0) -- _dummy Int Int32(size = 0), 1 UInt8 Const(size = 0, UInt8(size = 1)). From b5cbcea6778d01090a0ad01fa0a313f84274aa96 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 20 Feb 2023 00:13:53 +0100 Subject: [PATCH 425/717] Rename test --- ...al_desc.reference => 02666_read_in_order_final_desc.reference} | 0 ...in_order_final_desc.sql => 02666_read_in_order_final_desc.sql} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/queries/0_stateless/{25336_read_in_order_final_desc.reference => 02666_read_in_order_final_desc.reference} (100%) rename tests/queries/0_stateless/{25336_read_in_order_final_desc.sql => 02666_read_in_order_final_desc.sql} (100%) diff --git a/tests/queries/0_stateless/25336_read_in_order_final_desc.reference b/tests/queries/0_stateless/02666_read_in_order_final_desc.reference similarity index 100% rename from tests/queries/0_stateless/25336_read_in_order_final_desc.reference rename to tests/queries/0_stateless/02666_read_in_order_final_desc.reference diff --git a/tests/queries/0_stateless/25336_read_in_order_final_desc.sql b/tests/queries/0_stateless/02666_read_in_order_final_desc.sql similarity index 100% rename from tests/queries/0_stateless/25336_read_in_order_final_desc.sql rename to tests/queries/0_stateless/02666_read_in_order_final_desc.sql From b6394ae55931fe48077a46726ac9dab0874765b1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 20 Feb 2023 00:17:53 +0100 Subject: [PATCH 426/717] Add bug --- .../0_stateless/02667_wrong_structure.reference | 0 .../queries/0_stateless/02667_wrong_structure.sql | 15 +++++++++++++++ 2 files changed, 15 insertions(+) create mode 100644 tests/queries/0_stateless/02667_wrong_structure.reference create mode 100644 tests/queries/0_stateless/02667_wrong_structure.sql diff --git a/tests/queries/0_stateless/02667_wrong_structure.reference b/tests/queries/0_stateless/02667_wrong_structure.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02667_wrong_structure.sql b/tests/queries/0_stateless/02667_wrong_structure.sql new file mode 100644 index 00000000000..e8dc20a1aad --- /dev/null +++ b/tests/queries/0_stateless/02667_wrong_structure.sql @@ -0,0 +1,15 @@ +DROP TABLE IF EXISTS test_table__fuzz_0; +DROP TABLE IF EXISTS test_table__fuzz_2; + +SET allow_suspicious_low_cardinality_types = 1; + +CREATE TABLE test_table__fuzz_0 (`id` UInt64, `value` String) ENGINE = TinyLog; +CREATE TABLE test_table__fuzz_2 (`id` LowCardinality(UInt64), `value` LowCardinality(String)) ENGINE = TinyLog; + +INSERT INTO test_table__fuzz_0 VALUES (1911323367950415347,'@]~2|%N'),(15844130449337515313,'),Z?yV'),(10948652031493940763,'RC'),(10996795342436375388,''),(6019500279534407119,''),(3133348157913025617,'{fW!'),(8081599482719673616,'s'),(11115208170338871149,'sg`:}'),(9740456510723628701,'P'),(16586086969122622868,'Bn'); +INSERT INTO test_table__fuzz_2 VALUES (9806329011943062144,'wS6*'),(15996368926351777736,'pg.eA77G+P'),(5001315181732963827,'Qt,1)+pxB'),(10351886378573180883,'h--;'),(12739541939251729647,'>7sm*'),(6903016743298037689,'aVF8.ZU'),(9608529974121700083,',ZT'),(12616722808145461678,'#9Qp]'),(13678566439661733359,' J8'),(13909219891163965895,'.'); + +SELECT arrayMap(x -> (id + (SELECT id FROM test_table__fuzz_0 WHERE arrayMap(x -> (id + 9806329011943062144), [10])[NULL])), [1048575]) FROM test_table__fuzz_2; + +DROP TABLE test_table__fuzz_0; +DROP TABLE test_table__fuzz_2; From 1bd9629ed0c3e7484112e46dbc5f03ae85d430f9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 20 Feb 2023 00:21:14 +0100 Subject: [PATCH 427/717] Simpler --- tests/queries/0_stateless/02667_wrong_structure.sql | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/queries/0_stateless/02667_wrong_structure.sql b/tests/queries/0_stateless/02667_wrong_structure.sql index e8dc20a1aad..16e6eff82b7 100644 --- a/tests/queries/0_stateless/02667_wrong_structure.sql +++ b/tests/queries/0_stateless/02667_wrong_structure.sql @@ -1,15 +1,11 @@ -DROP TABLE IF EXISTS test_table__fuzz_0; DROP TABLE IF EXISTS test_table__fuzz_2; SET allow_suspicious_low_cardinality_types = 1; -CREATE TABLE test_table__fuzz_0 (`id` UInt64, `value` String) ENGINE = TinyLog; CREATE TABLE test_table__fuzz_2 (`id` LowCardinality(UInt64), `value` LowCardinality(String)) ENGINE = TinyLog; -INSERT INTO test_table__fuzz_0 VALUES (1911323367950415347,'@]~2|%N'),(15844130449337515313,'),Z?yV'),(10948652031493940763,'RC'),(10996795342436375388,''),(6019500279534407119,''),(3133348157913025617,'{fW!'),(8081599482719673616,'s'),(11115208170338871149,'sg`:}'),(9740456510723628701,'P'),(16586086969122622868,'Bn'); -INSERT INTO test_table__fuzz_2 VALUES (9806329011943062144,'wS6*'),(15996368926351777736,'pg.eA77G+P'),(5001315181732963827,'Qt,1)+pxB'),(10351886378573180883,'h--;'),(12739541939251729647,'>7sm*'),(6903016743298037689,'aVF8.ZU'),(9608529974121700083,',ZT'),(12616722808145461678,'#9Qp]'),(13678566439661733359,' J8'),(13909219891163965895,'.'); +INSERT INTO test_table__fuzz_2 VALUES (9806329011943062144,'wS6*'); -SELECT arrayMap(x -> (id + (SELECT id FROM test_table__fuzz_0 WHERE arrayMap(x -> (id + 9806329011943062144), [10])[NULL])), [1048575]) FROM test_table__fuzz_2; +SELECT arrayMap(x -> (id + (SELECT c1 AS id FROM VALUES((1911323367950415347, '@]~2|%N')) WHERE arrayMap(x -> (id + 9806329011943062144), [10])[NULL])), [1048575]) FROM test_table__fuzz_2; -DROP TABLE test_table__fuzz_0; DROP TABLE test_table__fuzz_2; From f63ed43f216dc34544ab62fc580574d8352ef99a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 20 Feb 2023 00:21:51 +0100 Subject: [PATCH 428/717] Simpler --- tests/queries/0_stateless/02667_wrong_structure.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02667_wrong_structure.sql b/tests/queries/0_stateless/02667_wrong_structure.sql index 16e6eff82b7..384a7b70a43 100644 --- a/tests/queries/0_stateless/02667_wrong_structure.sql +++ b/tests/queries/0_stateless/02667_wrong_structure.sql @@ -6,6 +6,6 @@ CREATE TABLE test_table__fuzz_2 (`id` LowCardinality(UInt64), `value` LowCardina INSERT INTO test_table__fuzz_2 VALUES (9806329011943062144,'wS6*'); -SELECT arrayMap(x -> (id + (SELECT c1 AS id FROM VALUES((1911323367950415347, '@]~2|%N')) WHERE arrayMap(x -> (id + 9806329011943062144), [10])[NULL])), [1048575]) FROM test_table__fuzz_2; +SELECT arrayMap(x -> (id + (SELECT 1911323367950415347 AS id WHERE arrayMap(x -> (id + 9806329011943062144), [10])[NULL])), [1048575]) FROM test_table__fuzz_2; DROP TABLE test_table__fuzz_2; From 77373a9381a8cff730cfd88b712c1763dc6c2a0f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 20 Feb 2023 00:23:30 +0100 Subject: [PATCH 429/717] Simpler --- tests/queries/0_stateless/02667_wrong_structure.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02667_wrong_structure.sql b/tests/queries/0_stateless/02667_wrong_structure.sql index 384a7b70a43..70d84cd9242 100644 --- a/tests/queries/0_stateless/02667_wrong_structure.sql +++ b/tests/queries/0_stateless/02667_wrong_structure.sql @@ -6,6 +6,6 @@ CREATE TABLE test_table__fuzz_2 (`id` LowCardinality(UInt64), `value` LowCardina INSERT INTO test_table__fuzz_2 VALUES (9806329011943062144,'wS6*'); -SELECT arrayMap(x -> (id + (SELECT 1911323367950415347 AS id WHERE arrayMap(x -> (id + 9806329011943062144), [10])[NULL])), [1048575]) FROM test_table__fuzz_2; +SELECT arrayMap(x -> (id + (SELECT 1 AS id WHERE arrayMap(x -> (id + 1), [10])[NULL])), [1048575]) FROM test_table__fuzz_2; DROP TABLE test_table__fuzz_2; From d9c675e1ca3fa84cf81e367906373992ab7662a9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 20 Feb 2023 00:24:41 +0100 Subject: [PATCH 430/717] Simpler --- tests/queries/0_stateless/02667_wrong_structure.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02667_wrong_structure.sql b/tests/queries/0_stateless/02667_wrong_structure.sql index 70d84cd9242..9997143cfdf 100644 --- a/tests/queries/0_stateless/02667_wrong_structure.sql +++ b/tests/queries/0_stateless/02667_wrong_structure.sql @@ -6,6 +6,6 @@ CREATE TABLE test_table__fuzz_2 (`id` LowCardinality(UInt64), `value` LowCardina INSERT INTO test_table__fuzz_2 VALUES (9806329011943062144,'wS6*'); -SELECT arrayMap(x -> (id + (SELECT 1 AS id WHERE arrayMap(x -> (id + 1), [10])[NULL])), [1048575]) FROM test_table__fuzz_2; +SELECT arrayMap(x -> (id + (SELECT 1 AS id WHERE [10][NULL])), [1048575]) FROM test_table__fuzz_2; DROP TABLE test_table__fuzz_2; From f9dac6d4cff13f17e1534a59792a8ed6deba805d Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky <43110995+evillique@users.noreply.github.com> Date: Mon, 20 Feb 2023 00:26:14 +0100 Subject: [PATCH 431/717] Fix ASTAsterisk::clone() --- src/Parsers/ASTAsterisk.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Parsers/ASTAsterisk.cpp b/src/Parsers/ASTAsterisk.cpp index 1ffbb85da7c..7e872e944bd 100644 --- a/src/Parsers/ASTAsterisk.cpp +++ b/src/Parsers/ASTAsterisk.cpp @@ -8,6 +8,7 @@ namespace DB ASTPtr ASTAsterisk::clone() const { auto clone = std::make_shared(*this); + clone->children.clear(); if (expression) { clone->expression = expression->clone(); clone->children.push_back(clone->expression); } if (transformers) { clone->transformers = transformers->clone(); clone->children.push_back(clone->transformers); } From 5427368fda5a5cfba9fbdadf56afefbd7ebdfbe3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 20 Feb 2023 00:26:46 +0100 Subject: [PATCH 432/717] Simpler --- tests/queries/0_stateless/02667_wrong_structure.sql | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/tests/queries/0_stateless/02667_wrong_structure.sql b/tests/queries/0_stateless/02667_wrong_structure.sql index 9997143cfdf..04370073c98 100644 --- a/tests/queries/0_stateless/02667_wrong_structure.sql +++ b/tests/queries/0_stateless/02667_wrong_structure.sql @@ -1,11 +1 @@ -DROP TABLE IF EXISTS test_table__fuzz_2; - -SET allow_suspicious_low_cardinality_types = 1; - -CREATE TABLE test_table__fuzz_2 (`id` LowCardinality(UInt64), `value` LowCardinality(String)) ENGINE = TinyLog; - -INSERT INTO test_table__fuzz_2 VALUES (9806329011943062144,'wS6*'); - -SELECT arrayMap(x -> (id + (SELECT 1 AS id WHERE [10][NULL])), [1048575]) FROM test_table__fuzz_2; - -DROP TABLE test_table__fuzz_2; +SELECT arrayMap(x -> (toLowCardinality(1) + (SELECT 1 WHERE 0)), [1]); From e8b4869ec02674ae43601035060cbb7d2d71c157 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 20 Feb 2023 00:28:29 +0100 Subject: [PATCH 433/717] Simpler --- tests/queries/0_stateless/02667_wrong_structure.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02667_wrong_structure.sql b/tests/queries/0_stateless/02667_wrong_structure.sql index 04370073c98..ed3fca5e0f2 100644 --- a/tests/queries/0_stateless/02667_wrong_structure.sql +++ b/tests/queries/0_stateless/02667_wrong_structure.sql @@ -1 +1 @@ -SELECT arrayMap(x -> (toLowCardinality(1) + (SELECT 1 WHERE 0)), [1]); +SELECT arrayMap(x -> (toLowCardinality(1) + 1::Nullable(UInt8)), [1]); From d48ec14cabc454fc54001921d26df3ad1af1795d Mon Sep 17 00:00:00 2001 From: Jus <40656180+jus1096@users.noreply.github.com> Date: Mon, 20 Feb 2023 10:59:57 +0400 Subject: [PATCH 434/717] Add description function Add description function. Fix bugs --- .../functions/other-functions.md | 120 +++++++++++++++++- 1 file changed, 119 insertions(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/functions/other-functions.md b/docs/ru/sql-reference/functions/other-functions.md index af21ccd6bed..6e0c4ba146d 100644 --- a/docs/ru/sql-reference/functions/other-functions.md +++ b/docs/ru/sql-reference/functions/other-functions.md @@ -295,6 +295,10 @@ SELECT byteSize(NULL, 1, 0.3, ''); Спит seconds секунд на каждый блок данных. Можно указать как целое число, так и число с плавающей запятой. +## sleepEachRow(seconds) {# sleepeachrowseconds} + +Спит seconds секунд на каждую строку. Можно указать как целое число, так и число с плавающей запятой. + ## currentDatabase() {#currentdatabase} Возвращает имя текущей базы данных. @@ -590,6 +594,27 @@ LIMIT 10 └────────────────┴─────────┘ ``` +## formatReadableDecimalSize(x) + +Принимает размер (число байт). Возвращает округленный размер с суффиксом (KiB, MiB и т.д.) в виде строки. + +Пример: + +``` sql +SELECT + arrayJoin([1, 1024, 1024*1024, 192851925]) AS filesize_bytes, + formatReadableDecimalSize(filesize_bytes) AS filesize +``` + +``` text +┌─filesize_bytes─┬─filesize───┐ +│ 1 │ 1.00 B │ +│ 1024 │ 1.02 KB │ +│ 1048576 │ 1.05 MB │ +│ 192851925 │ 192.85 MB │ +└────────────────┴────────────┘ +``` + ## formatReadableSize(x) {#formatreadablesizex} Принимает размер (число байт). Возвращает округленный размер с суффиксом (KiB, MiB и т.д.) в виде строки. @@ -634,6 +659,92 @@ SELECT └────────────────┴───────────────────┘ ``` +## formatReadableTimeDelta {#formatreadabletimedelta} + +Принимает дельту времени в секундах. Возвращает дельту времени с (год, месяц, день, час, минута, секунда) в виде строки. + +**Синтаксис** + +``` sql +formatReadableTimeDelta(column[, maximum_unit]) +``` + +**Аргументы** + +- `column` — Столбец с числовой дельтой времени. +- `maximum_unit` — Опицонально. Максимальная единица измерения для отображения. Допустимые значения: секунды, минуты, часы, дни, месяцы, годы. + +Пример: + +``` sql +SELECT + arrayJoin([100, 12345, 432546534]) AS elapsed, + formatReadableTimeDelta(elapsed) AS time_delta +``` + +``` text +┌────elapsed─┬─time_delta ─────────────────────────────────────────────────────┐ +│ 100 │ 1 minute and 40 seconds │ +│ 12345 │ 3 hours, 25 minutes and 45 seconds │ +│ 432546534 │ 13 years, 8 months, 17 days, 7 hours, 48 minutes and 54 seconds │ +└────────────┴─────────────────────────────────────────────────────────────────┘ +``` + +``` sql +SELECT + arrayJoin([100, 12345, 432546534]) AS elapsed, + formatReadableTimeDelta(elapsed, 'minutes') AS time_delta +``` + +``` text +┌────elapsed─┬─time_delta ─────────────────────────────────────────────────────┐ +│ 100 │ 1 minute and 40 seconds │ +│ 12345 │ 205 minutes and 45 seconds │ +│ 432546534 │ 7209108 minutes and 54 seconds │ +└────────────┴─────────────────────────────────────────────────────────────────┘ +``` + +## parseTimeDelta {#parsetimedelta} + +Преобразует последовательность символов, которая напоминает нечто похожее на единицу времени. + +**Синтаксис** + +```sql +parseTimeDelta(timestr) +``` + +**Аргументы** + +- `timestr` — Последовательность символов, которая напоминает нечто похожее на единицу времени. + + +**Возвращаемое значение** + +- Число с плавающей точкой, содержащее количество секунд. + +**Пример** + +```sql +SELECT parseTimeDelta('11s+22min') +``` + +```text +┌─parseTimeDelta('11s+22min')─┐ +│ 1331 │ +└─────────────────────────────┘ +``` + +```sql +SELECT parseTimeDelta('1yr2mo') +``` + +```text +┌─parseTimeDelta('1yr2mo')─┐ +│ 36806400 │ +└──────────────────────────┘ +``` + ## least(a, b) {#leasta-b} Возвращает наименьшее значение из a и b. @@ -657,6 +768,10 @@ SELECT Возвращает ID сборки, сгенерированный компилятором для данного сервера ClickHouse. Если функция вызывается в контексте распределенной таблицы, то она генерирует обычный столбец со значениями, актуальными для каждого шарда. Иначе возвращается константа. +## blockNumber {#function-blocknumber} + +Возвращает порядковый номер блока данных, в котором находится строка. + ## rowNumberInBlock {#function-rownumberinblock} Возвращает порядковый номер строки в блоке данных. Для каждого блока данных нумерация начинается с 0. @@ -679,6 +794,7 @@ neighbor(column, offset[, default_value]) :::danger "Предупреждение" Функция может получить доступ к значению в столбце соседней строки только внутри обрабатываемого в данный момент блока данных. +::: Порядок строк, используемый при вычислении функции `neighbor`, может отличаться от порядка строк, возвращаемых пользователю. Чтобы этого не случилось, вы можете сделать подзапрос с [ORDER BY](../../sql-reference/statements/select/order-by.md) и вызвать функцию извне подзапроса. @@ -788,6 +904,7 @@ FROM numbers(16) :::danger "Предупреждение" Функция может взять значение предыдущей строки только внутри текущего обработанного блока данных. +::: Результат функции зависит от затронутых блоков данных и порядка данных в блоке. @@ -869,7 +986,7 @@ WHERE diff != 1 :::danger "Предупреждение" События должны быть отсортированы по возрастанию времени начала. Если это требование нарушено, то функция вызывает исключение. Каждый блок данных обрабатывается независимо. Если события из разных блоков данных накладываются по времени, они не могут быть корректно обработаны. - +::: **Синтаксис** ``` sql @@ -1560,6 +1677,7 @@ FROM numbers(10); :::danger "Warning" Функция обнуляет состояние для каждого нового блока. +::: **Синтаксис** From 5557e667668aba87401d8b3d6a055a57c8269663 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Mon, 20 Feb 2023 07:58:37 +0000 Subject: [PATCH 435/717] Small update of sparkbar docs - parameter name "width" was misleading --- .../sql-reference/aggregate-functions/reference/sparkbar.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/aggregate-functions/reference/sparkbar.md b/docs/en/sql-reference/aggregate-functions/reference/sparkbar.md index 12da9be2847..05ea373d4da 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/sparkbar.md +++ b/docs/en/sql-reference/aggregate-functions/reference/sparkbar.md @@ -16,12 +16,12 @@ Otherwise, values outside the interval are ignored. **Syntax** ``` sql -sparkbar(width[, min_x, max_x])(x, y) +sparkbar(buckets[, min_x, max_x])(x, y) ``` **Parameters** -- `width` — The number of segments. Type: [Integer](../../../sql-reference/data-types/int-uint.md). +- `buckets` — The number of segments. Type: [Integer](../../../sql-reference/data-types/int-uint.md). - `min_x` — The interval start. Optional parameter. - `max_x` — The interval end. Optional parameter. From e5e102763710b30f739055b9120b452e1679c6d7 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Mon, 20 Feb 2023 09:48:55 +0100 Subject: [PATCH 436/717] Support for IN clause in parameterized views Implementation: * In case of parameterized views, the IN clause cannot be evaluated as constant expression during CREATE VIEW, added a check to ignore this step in case of parameterized view. * If parmeters are not in IN clause, we continue to evaluate it as constant expression. --- src/Interpreters/ActionsVisitor.cpp | 3 ++- .../0_stateless/02428_parameterized_view.reference | 3 +++ tests/queries/0_stateless/02428_parameterized_view.sh | 10 ++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index 2974b5c0ee0..a2ebc200613 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -957,7 +957,8 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & /// Let's find the type of the first argument (then getActionsImpl will be called again and will not affect anything). visit(node.arguments->children.at(0), data); - if (!data.no_makeset && (prepared_set = makeSet(node, data, data.no_subqueries))) + if (!data.no_makeset && !(data.is_create_parameterized_view && !analyzeReceiveQueryParams(ast).empty()) + && (prepared_set = makeSet(node, data, data.no_subqueries))) { /// Transform tuple or subquery into a set. } diff --git a/tests/queries/0_stateless/02428_parameterized_view.reference b/tests/queries/0_stateless/02428_parameterized_view.reference index db3ffd0b01e..52a31f53cc1 100644 --- a/tests/queries/0_stateless/02428_parameterized_view.reference +++ b/tests/queries/0_stateless/02428_parameterized_view.reference @@ -32,3 +32,6 @@ ERROR 3 3 5 +10 +20 +10 diff --git a/tests/queries/0_stateless/02428_parameterized_view.sh b/tests/queries/0_stateless/02428_parameterized_view.sh index 44c1976a654..6118013b665 100755 --- a/tests/queries/0_stateless/02428_parameterized_view.sh +++ b/tests/queries/0_stateless/02428_parameterized_view.sh @@ -13,6 +13,8 @@ $CLICKHOUSE_CLIENT -q "DROP VIEW IF EXISTS test_02428_pv4" $CLICKHOUSE_CLIENT -q "DROP VIEW IF EXISTS test_02428_pv5" $CLICKHOUSE_CLIENT -q "DROP VIEW IF EXISTS test_02428_pv6" $CLICKHOUSE_CLIENT -q "DROP VIEW IF EXISTS test_02428_pv7" +$CLICKHOUSE_CLIENT -q "DROP VIEW IF EXISTS test_02428_pv8" +$CLICKHOUSE_CLIENT -q "DROP VIEW IF EXISTS test_02428_pv9" $CLICKHOUSE_CLIENT -q "DROP VIEW IF EXISTS test_02428_v1" $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS test_02428_Catalog" $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS ${CLICKHOUSE_TEST_UNIQUE_NAME}.pv1" @@ -75,12 +77,20 @@ $CLICKHOUSE_CLIENT -q "SELECT * FROM test_02428_pv6(price=10)" $CLICKHOUSE_CLIENT -q "CREATE VIEW test_02428_pv7 AS SELECT Price/{price:UInt64} FROM test_02428_Catalog ORDER BY Price" $CLICKHOUSE_CLIENT -q "SELECT * FROM test_02428_pv7(price=10)" +$CLICKHOUSE_CLIENT -q "CREATE VIEW test_02428_pv8 AS SELECT Price FROM test_02428_Catalog WHERE Price IN ({prices:Array(UInt64)}) ORDER BY Price" +$CLICKHOUSE_CLIENT -q "SELECT * FROM test_02428_pv8(prices=[10,20])" + +$CLICKHOUSE_CLIENT -q "CREATE VIEW test_02428_pv9 AS SELECT Price FROM test_02428_Catalog WHERE Price IN (10,20) AND Quantity={quantity:UInt64} ORDER BY Price" +$CLICKHOUSE_CLIENT -q "SELECT * FROM test_02428_pv9(quantity=3)" + $CLICKHOUSE_CLIENT -q "DROP VIEW test_02428_pv1" $CLICKHOUSE_CLIENT -q "DROP VIEW test_02428_pv2" $CLICKHOUSE_CLIENT -q "DROP VIEW test_02428_pv3" $CLICKHOUSE_CLIENT -q "DROP VIEW test_02428_pv5" $CLICKHOUSE_CLIENT -q "DROP VIEW test_02428_pv6" $CLICKHOUSE_CLIENT -q "DROP VIEW test_02428_pv7" +$CLICKHOUSE_CLIENT -q "DROP VIEW test_02428_pv8" +$CLICKHOUSE_CLIENT -q "DROP VIEW test_02428_pv9" $CLICKHOUSE_CLIENT -q "DROP VIEW test_02428_v1" $CLICKHOUSE_CLIENT -q "DROP TABLE test_02428_Catalog" $CLICKHOUSE_CLIENT -q "DROP TABLE ${CLICKHOUSE_TEST_UNIQUE_NAME}.pv1" From 238d44783b10e89584432091a1cdd4ff843b3ba5 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 20 Feb 2023 09:05:48 +0100 Subject: [PATCH 437/717] Fix flakiness of expect tests for clickhouse-client by avoiding history overlap Yes, all writes to the history file is done under flock, *but*, before writing the history file there is sort(), and so if you will run the following tests the 01300_client_save_history_when_terminated_long will fail: $ /src/tests/clickhouse-test --print-time -j2 01180_client_syntax_errors 01300_client_save_history_when_terminated_long 0001_select And it has nothing todo with timeouts: expect: does "" (spawn_id exp8) match glob pattern "for the history"? no f8f1dbfdaaca :) select (1, 2 expect: does "\u001b[1Gf8f1dbfdaaca :) select \u001b[0;22;33m(\u001b[0;22;32m1\u001b[0;1m,\u001b[0m \u001b[0;22;32m2\u001b[0m\u001b[J" (spawn_id exp8) match glob pattern "for the history"? no expect: does "\u001b[1Gf8f1dbfdaaca :) select \u001b[0;22;33m(\u001b[0;22;32m1\u001b[0;1m,\u001b[0m \u001b[0;22;32m2\u001b[0m\u001b[J\u001b[29G" (spawn_id exp8) match glob pattern "for the history"? no expect: timed out The "select (1, 2" is from 01180_client_syntax_errors And use real file only when the history should be preserved across runs (i.e. there are multiple invocations of clickhouse-client) CI: https://s3.amazonaws.com/clickhouse-test-reports/0/1adfbac19fed9813725d8b1df14e617b58a45d20/stateless_tests__asan__[2/2].html Signed-off-by: Azat Khuzhin --- .gitignore | 1 + .../01179_insert_values_semicolon.expect | 2 +- .../01180_client_syntax_errors.expect | 2 +- ...client_interactive_vertical_multiline.expect | 5 +++-- ...lient_interactive_vertical_singleline.expect | 2 +- ...ent_save_history_when_terminated_long.expect | 5 +++-- ...nt_autocomplete_word_break_characters.expect | 2 +- .../01520_client_print_query_id.expect | 2 +- .../01565_reconnect_after_client_error.expect | 2 +- ...ghlight_multi_line_comment_regression.expect | 2 +- .../01933_client_replxx_convert_history.expect | 17 ++++++++--------- .../0_stateless/01945_show_debug_warning.expect | 8 ++++---- .../02003_memory_limit_in_client.expect | 6 +++--- .../0_stateless/02047_client_exception.expect | 2 +- .../02105_backslash_letter_commands.expect | 4 ++-- .../0_stateless/02116_interactive_hello.expect | 2 +- .../02132_client_history_navigation.expect | 4 ++-- ...02160_client_autocomplete_parse_query.expect | 2 +- .../02417_repeat_input_commands.expect | 4 ++-- 19 files changed, 38 insertions(+), 36 deletions(-) diff --git a/.gitignore b/.gitignore index 7a513ec1a09..14b860244c2 100644 --- a/.gitignore +++ b/.gitignore @@ -161,6 +161,7 @@ website/package-lock.json tests/queries/0_stateless/test_* tests/queries/0_stateless/*.binary tests/queries/0_stateless/*.generated-expect +tests/queries/0_stateless/*.expect.history # rust /rust/**/target diff --git a/tests/queries/0_stateless/01179_insert_values_semicolon.expect b/tests/queries/0_stateless/01179_insert_values_semicolon.expect index 9d35941ae40..d66a84769f2 100755 --- a/tests/queries/0_stateless/01179_insert_values_semicolon.expect +++ b/tests/queries/0_stateless/01179_insert_values_semicolon.expect @@ -15,7 +15,7 @@ expect_after { timeout { exit 1 } } -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --history_file=/dev/null" expect ":) " send -- "DROP TABLE IF EXISTS test_01179\r" diff --git a/tests/queries/0_stateless/01180_client_syntax_errors.expect b/tests/queries/0_stateless/01180_client_syntax_errors.expect index da3dfbec6df..a031333313a 100755 --- a/tests/queries/0_stateless/01180_client_syntax_errors.expect +++ b/tests/queries/0_stateless/01180_client_syntax_errors.expect @@ -14,7 +14,7 @@ expect_after { timeout { exit 1 } } -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --history_file=/dev/null" expect ":) " # Make a query with syntax error diff --git a/tests/queries/0_stateless/01293_client_interactive_vertical_multiline.expect b/tests/queries/0_stateless/01293_client_interactive_vertical_multiline.expect index bab1dd224cf..629698b4565 100755 --- a/tests/queries/0_stateless/01293_client_interactive_vertical_multiline.expect +++ b/tests/queries/0_stateless/01293_client_interactive_vertical_multiline.expect @@ -3,6 +3,7 @@ set basedir [file dirname $argv0] set basename [file tail $argv0] exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 +set history_file $env(CLICKHOUSE_TMP)/$basename.history log_user 0 set timeout 10 @@ -18,7 +19,7 @@ expect_after { # useful debugging configuration # exp_internal 1 -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --history_file=$history_file" expect ":) " send -- "SELECT 1\r" @@ -60,7 +61,7 @@ expect ":) " send -- "" expect eof -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --highlight 0 --multiline" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --highlight 0 --multiline --history_file=$history_file" expect ":) " send -- "SELECT 1;\r" diff --git a/tests/queries/0_stateless/01293_client_interactive_vertical_singleline.expect b/tests/queries/0_stateless/01293_client_interactive_vertical_singleline.expect index 83eced841ce..e64d71c6319 100755 --- a/tests/queries/0_stateless/01293_client_interactive_vertical_singleline.expect +++ b/tests/queries/0_stateless/01293_client_interactive_vertical_singleline.expect @@ -14,7 +14,7 @@ expect_after { timeout { exit 1 } } -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --history_file=/dev/null" expect ":) " send -- "SELECT 1\r" diff --git a/tests/queries/0_stateless/01300_client_save_history_when_terminated_long.expect b/tests/queries/0_stateless/01300_client_save_history_when_terminated_long.expect index 06a60ed95a2..de485383024 100755 --- a/tests/queries/0_stateless/01300_client_save_history_when_terminated_long.expect +++ b/tests/queries/0_stateless/01300_client_save_history_when_terminated_long.expect @@ -4,6 +4,7 @@ set basedir [file dirname $argv0] set basename [file tail $argv0] exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 +set history_file $env(CLICKHOUSE_TMP)/$basename.history log_user 0 set timeout 60 @@ -15,7 +16,7 @@ expect_after { timeout { exit 1 } } -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --history_file=$history_file" expect ":) " # Make a query @@ -28,7 +29,7 @@ exec kill -9 [exp_pid] close # Run client one more time and press "up" to see the last recorded query -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --history_file=$history_file" expect ":) " send -- "\[A" expect "for the history" diff --git a/tests/queries/0_stateless/01370_client_autocomplete_word_break_characters.expect b/tests/queries/0_stateless/01370_client_autocomplete_word_break_characters.expect index fff0dd015e1..bbd0fdd223b 100755 --- a/tests/queries/0_stateless/01370_client_autocomplete_word_break_characters.expect +++ b/tests/queries/0_stateless/01370_client_autocomplete_word_break_characters.expect @@ -14,7 +14,7 @@ expect_after { timeout { exit 1 } } -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --history_file=/dev/null" expect ":) " # Make a query diff --git a/tests/queries/0_stateless/01520_client_print_query_id.expect b/tests/queries/0_stateless/01520_client_print_query_id.expect index 0e8f660041d..b16cd6d499b 100755 --- a/tests/queries/0_stateless/01520_client_print_query_id.expect +++ b/tests/queries/0_stateless/01520_client_print_query_id.expect @@ -14,7 +14,7 @@ expect_after { timeout { exit 1 } } -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --history_file=/dev/null" expect ":) " # Make a query diff --git a/tests/queries/0_stateless/01565_reconnect_after_client_error.expect b/tests/queries/0_stateless/01565_reconnect_after_client_error.expect index 035698f524b..076b91390bd 100755 --- a/tests/queries/0_stateless/01565_reconnect_after_client_error.expect +++ b/tests/queries/0_stateless/01565_reconnect_after_client_error.expect @@ -19,7 +19,7 @@ expect_after { timeout { exit 1 } } -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion -mn" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion -mn --history_file=/dev/null" expect "\n:) " send -- "DROP TABLE IF EXISTS t01565;\n" diff --git a/tests/queries/0_stateless/01755_client_highlight_multi_line_comment_regression.expect b/tests/queries/0_stateless/01755_client_highlight_multi_line_comment_regression.expect index 3d9c633eb44..293fd2e13f4 100755 --- a/tests/queries/0_stateless/01755_client_highlight_multi_line_comment_regression.expect +++ b/tests/queries/0_stateless/01755_client_highlight_multi_line_comment_regression.expect @@ -14,7 +14,7 @@ expect_after { timeout { exit 1 } } -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --history_file=/dev/null" expect ":) " # regression for heap-buffer-overflow issue (under ASAN) diff --git a/tests/queries/0_stateless/01933_client_replxx_convert_history.expect b/tests/queries/0_stateless/01933_client_replxx_convert_history.expect index 111389e49b2..0c95b630742 100755 --- a/tests/queries/0_stateless/01933_client_replxx_convert_history.expect +++ b/tests/queries/0_stateless/01933_client_replxx_convert_history.expect @@ -1,10 +1,9 @@ #!/usr/bin/expect -f -# Tags: no-parallel -# Tag no-parallel: Uses non unique history file set basedir [file dirname $argv0] set basename [file tail $argv0] exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 +set history_file $env(CLICKHOUSE_TMP)/$basename.history log_user 0 set timeout 60 @@ -16,17 +15,17 @@ expect_after { timeout { exit 1 } } -exec bash -c "echo select 1 > $argv0.txt" -exec bash -c "echo select 1 >> $argv0.txt" -exec bash -c "echo select 1 >> $argv0.txt" +exec bash -c "echo select 1 > $history_file.txt" +exec bash -c "echo select 1 >> $history_file.txt" +exec bash -c "echo select 1 >> $history_file.txt" -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --history_file=$argv0.txt" -expect "The history file ($argv0.txt) is in old format. 3 lines, 1 unique lines." +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --history_file=$history_file.txt" +expect "The history file ($history_file.txt) is in old format. 3 lines, 1 unique lines." expect ":) " send -- "\4" expect eof -spawn bash -c "wc -l $argv0.txt" +spawn bash -c "wc -l $history_file.txt" # The following lines are expected: # # ### YYYY-MM-DD HH:MM:SS.SSS @@ -35,4 +34,4 @@ spawn bash -c "wc -l $argv0.txt" expect "2" expect eof -exec bash -c "rm $argv0.txt" +exec bash -c "rm $history_file.txt" diff --git a/tests/queries/0_stateless/01945_show_debug_warning.expect b/tests/queries/0_stateless/01945_show_debug_warning.expect index ca423ee106c..4a0abb358d5 100755 --- a/tests/queries/0_stateless/01945_show_debug_warning.expect +++ b/tests/queries/0_stateless/01945_show_debug_warning.expect @@ -20,14 +20,14 @@ expect_after { set Debug_type 0 -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --history_file=/dev/null" expect ":) " # Check debug type send -- "SELECT value FROM system.build_options WHERE name='BUILD_TYPE'\r" expect { "Debug" { - set Debug_type 1 + set Debug_type 1 expect ":) " } "RelWithDebInfo" @@ -38,7 +38,7 @@ expect eof if { $Debug_type > 0} { -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --history_file=/dev/null" expect "Warnings:" expect " * Server was built in debug mode. It will work slowly." expect ":) " @@ -52,7 +52,7 @@ send -- "q\r" expect eof } -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --max_memory_usage_for_all_queries=123" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --max_memory_usage_for_all_queries=123 --history_file=/dev/null" expect "Warnings:" expect " * Some obsolete setting is changed." expect ":) " diff --git a/tests/queries/0_stateless/02003_memory_limit_in_client.expect b/tests/queries/0_stateless/02003_memory_limit_in_client.expect index a8e8c1d5786..e03fb9308f4 100755 --- a/tests/queries/0_stateless/02003_memory_limit_in_client.expect +++ b/tests/queries/0_stateless/02003_memory_limit_in_client.expect @@ -22,7 +22,7 @@ expect_after { # # Check that the query will fail in clickhouse-client # -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --max_memory_usage_in_client=1" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --max_memory_usage_in_client=1 --history_file=/dev/null" expect ":) " send -- "SELECT arrayMap(x -> range(x), range(number)) FROM numbers(1000)\r" @@ -37,7 +37,7 @@ expect eof # # Check that the query will fail in clickhouse-client # -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --max_memory_usage_in_client=1" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --max_memory_usage_in_client=1 --history_file=/dev/null" expect ":) " send -- "SELECT arrayMap(x -> range(x), range(number)) FROM numbers(1000)\r" @@ -52,7 +52,7 @@ expect eof # # Check that the query will not fail (due to max_untracked_memory) # -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --max_memory_usage_in_client=1" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --max_memory_usage_in_client=1 --history_file=/dev/null" expect ":) " send -- "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000\r" diff --git a/tests/queries/0_stateless/02047_client_exception.expect b/tests/queries/0_stateless/02047_client_exception.expect index 50ed09d03c5..9ae8495cfcb 100755 --- a/tests/queries/0_stateless/02047_client_exception.expect +++ b/tests/queries/0_stateless/02047_client_exception.expect @@ -15,7 +15,7 @@ expect_after { timeout { exit 1 } } -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --history_file=/dev/null" expect ":) " send -- "DROP TABLE IF EXISTS test_02047\r" diff --git a/tests/queries/0_stateless/02105_backslash_letter_commands.expect b/tests/queries/0_stateless/02105_backslash_letter_commands.expect index 707a544f6bb..a7340020611 100755 --- a/tests/queries/0_stateless/02105_backslash_letter_commands.expect +++ b/tests/queries/0_stateless/02105_backslash_letter_commands.expect @@ -5,7 +5,7 @@ set basename [file tail $argv0] exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 log_user 0 -set timeout 02 +set timeout 60 match_max 100000 expect_after { # Do not ignore eof from expect @@ -14,7 +14,7 @@ expect_after { timeout { exit 1 } } -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --history_file=/dev/null" expect ":) " # Send a command diff --git a/tests/queries/0_stateless/02116_interactive_hello.expect b/tests/queries/0_stateless/02116_interactive_hello.expect index 5fa31d33e87..443f835b76d 100755 --- a/tests/queries/0_stateless/02116_interactive_hello.expect +++ b/tests/queries/0_stateless/02116_interactive_hello.expect @@ -16,7 +16,7 @@ expect_after { timeout { exit 1 } } -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --history_file=/dev/null" expect -re "ClickHouse client version \[\\d\]{2}.\[\\d\]{1,2}.\[\\d\]{1,2}.\[\\d\]{1,2}.\r" expect -re "Connecting to database .* at localhost:9000 as user default.\r" diff --git a/tests/queries/0_stateless/02132_client_history_navigation.expect b/tests/queries/0_stateless/02132_client_history_navigation.expect index 10167fb2e97..82e994f4fc1 100755 --- a/tests/queries/0_stateless/02132_client_history_navigation.expect +++ b/tests/queries/0_stateless/02132_client_history_navigation.expect @@ -5,7 +5,7 @@ set basename [file tail $argv0] exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 log_user 0 -set timeout 3 +set timeout 60 match_max 100000 expect_after { @@ -18,7 +18,7 @@ expect_after { # useful debugging configuration # exp_internal 1 -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --highlight 0" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --highlight 0 --history_file=/dev/null" expect ":) " # Make a query diff --git a/tests/queries/0_stateless/02160_client_autocomplete_parse_query.expect b/tests/queries/0_stateless/02160_client_autocomplete_parse_query.expect index b95f85403e3..12bfcc7c041 100755 --- a/tests/queries/0_stateless/02160_client_autocomplete_parse_query.expect +++ b/tests/queries/0_stateless/02160_client_autocomplete_parse_query.expect @@ -15,7 +15,7 @@ expect_after { timeout { exit 1 } } -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --history_file=/dev/null" expect ":) " # Make a query diff --git a/tests/queries/0_stateless/02417_repeat_input_commands.expect b/tests/queries/0_stateless/02417_repeat_input_commands.expect index 119aac68645..85be0a02881 100755 --- a/tests/queries/0_stateless/02417_repeat_input_commands.expect +++ b/tests/queries/0_stateless/02417_repeat_input_commands.expect @@ -5,7 +5,7 @@ set basename [file tail $argv0] exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 log_user 0 -set timeout 10 +set timeout 60 match_max 100000 expect_after { @@ -15,7 +15,7 @@ expect_after { timeout { exit 1 } } -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --history_file=/dev/null" expect ":) " # ----------------------------------------- From 2018559fa5f93c31af8f524c271504239d3ed56b Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 20 Feb 2023 09:10:28 +0100 Subject: [PATCH 438/717] Add style check for using --history_file in expect tests Signed-off-by: Azat Khuzhin --- utils/check-style/check-style | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/utils/check-style/check-style b/utils/check-style/check-style index 1ae003174d7..8436d3378d9 100755 --- a/utils/check-style/check-style +++ b/utils/check-style/check-style @@ -333,6 +333,11 @@ expect_tests=( $(find $ROOT_PATH/tests/queries -name '*.expect') ) for test_case in "${expect_tests[@]}"; do pattern="^exp_internal -f \$env(CLICKHOUSE_TMP)/\$basename.debuglog 0$" grep -q "$pattern" "$test_case" || echo "Missing '$pattern' in '$test_case'" + + if grep -q "^spawn.*CLICKHOUSE_CLIENT_BINARY$" "$test_case"; then + pattern="^spawn.*CLICKHOUSE_CLIENT_BINARY.*--history_file$" + grep -q "$pattern" "$test_case" || echo "Missing '$pattern' in '$test_case'" + fi done # Conflict markers From 6e1f284edac731d902b38c8062f895d92b47e9e3 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 20 Feb 2023 09:25:28 +0100 Subject: [PATCH 439/717] Improve detection of debug builds in 01945_show_debug_warning Signed-off-by: Azat Khuzhin --- .../0_stateless/01945_show_debug_warning.expect | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/queries/0_stateless/01945_show_debug_warning.expect b/tests/queries/0_stateless/01945_show_debug_warning.expect index 4a0abb358d5..6714901ab20 100755 --- a/tests/queries/0_stateless/01945_show_debug_warning.expect +++ b/tests/queries/0_stateless/01945_show_debug_warning.expect @@ -24,13 +24,13 @@ spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \ expect ":) " # Check debug type -send -- "SELECT value FROM system.build_options WHERE name='BUILD_TYPE'\r" +send -- "SELECT lower(value) FROM system.build_options WHERE name='BUILD_TYPE'\r" expect { -"Debug" { - set Debug_type 1 - expect ":) " + "debug" { + set Debug_type 1 + expect ":) " } -"RelWithDebInfo" + "relwithdebinfo" } send -- "q\r" From ccf87a6afd1fa93ea6ad8fe454ee27edd75c73ef Mon Sep 17 00:00:00 2001 From: lzydmxy <13126752315@163.com> Date: Mon, 20 Feb 2023 16:52:55 +0800 Subject: [PATCH 440/717] add integration test for move partition to disk on cluster --- .../configs/config.d/cluster.xml | 17 ++++ .../config.d/storage_configuration.xml | 28 ++++++ .../test.py | 94 +++++++++++++++++++ 3 files changed, 139 insertions(+) create mode 100644 tests/integration/test_move_partition_to_disk_on_cluster/configs/config.d/cluster.xml create mode 100644 tests/integration/test_move_partition_to_disk_on_cluster/configs/config.d/storage_configuration.xml create mode 100644 tests/integration/test_move_partition_to_disk_on_cluster/test.py diff --git a/tests/integration/test_move_partition_to_disk_on_cluster/configs/config.d/cluster.xml b/tests/integration/test_move_partition_to_disk_on_cluster/configs/config.d/cluster.xml new file mode 100644 index 00000000000..2316050b629 --- /dev/null +++ b/tests/integration/test_move_partition_to_disk_on_cluster/configs/config.d/cluster.xml @@ -0,0 +1,17 @@ + + + + + true + + node1 + 9000 + + + node2 + 9000 + + + + + \ No newline at end of file diff --git a/tests/integration/test_move_partition_to_disk_on_cluster/configs/config.d/storage_configuration.xml b/tests/integration/test_move_partition_to_disk_on_cluster/configs/config.d/storage_configuration.xml new file mode 100644 index 00000000000..3289186c175 --- /dev/null +++ b/tests/integration/test_move_partition_to_disk_on_cluster/configs/config.d/storage_configuration.xml @@ -0,0 +1,28 @@ + + + + + + /jbod1/ + + + /external/ + + + + + + +
+ jbod1 +
+ + external + + + + + + + + diff --git a/tests/integration/test_move_partition_to_disk_on_cluster/test.py b/tests/integration/test_move_partition_to_disk_on_cluster/test.py new file mode 100644 index 00000000000..fe8606bd549 --- /dev/null +++ b/tests/integration/test_move_partition_to_disk_on_cluster/test.py @@ -0,0 +1,94 @@ +import pytest +from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +node1 = cluster.add_instance( + "node1", + main_configs=[ + "configs/config.d/storage_configuration.xml", + "configs/config.d/cluster.xml", + ], + with_zookeeper=True, + stay_alive=True, + tmpfs=["/jbod1:size=10M", "/external:size=10M"], + macros={"shard": 0, "replica": 1}, +) + +node2 = cluster.add_instance( + "node2", + main_configs=[ + "configs/config.d/storage_configuration.xml", + "configs/config.d/cluster.xml", + ], + with_zookeeper=True, + stay_alive=True, + tmpfs=["/jbod1:size=10M", "/external:size=10M"], + macros={"shard": 0, "replica": 2}, +) + + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +def test_move_partition_to_disk_on_cluster(start_cluster): + for node in [node1, node2]: + node.query( + sql="CREATE TABLE test_local_table" + "(x UInt64) " + "ENGINE=ReplicatedMergeTree('/clickhouse/tables/test_local_table', '{replica}') " + "ORDER BY tuple()" + "SETTINGS storage_policy = 'jbod_with_external';", + ) + + node1.query("INSERT INTO test_local_table VALUES (0)") + node1.query("SYSTEM SYNC REPLICA test_local_table", timeout=30) + + try: + node1.query( + sql="ALTER TABLE test_local_table ON CLUSTER 'test_cluster' MOVE PARTITION tuple() TO DISK 'jbod1';", + ) + except QueryRuntimeException: + pass + + for node in [node1, node2]: + assert ( + node.query( + "SELECT partition_id, disk_name FROM system.parts WHERE table = 'test_local_table' FORMAT Values" + ) + == "('all','jbod1')" + ) + + node1.query( + sql="ALTER TABLE test_local_table ON CLUSTER 'test_cluster' MOVE PARTITION tuple() TO DISK 'external';", + ) + + for node in [node1, node2]: + assert ( + node.query( + "SELECT partition_id, disk_name FROM system.parts WHERE table = 'test_local_table' FORMAT Values" + ) + == "('all','external')" + ) + + node1.query( + sql="ALTER TABLE test_local_table ON CLUSTER 'test_cluster' MOVE PARTITION tuple() TO VOLUME 'main';", + ) + + for node in [node1, node2]: + assert ( + node.query( + "SELECT partition_id, disk_name FROM system.parts WHERE table = 'test_local_table' FORMAT Values" + ) + == "('all','jbod1')" + ) + + From d39cffb3f2832188fcba1ea5f9b6d414152ea145 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Fri, 17 Feb 2023 20:16:42 +0100 Subject: [PATCH 441/717] Add rollback commands for the case of broad error --- tests/ci/release.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/tests/ci/release.py b/tests/ci/release.py index 52723e46693..d5537f106c5 100755 --- a/tests/ci/release.py +++ b/tests/ci/release.py @@ -277,8 +277,12 @@ class Release: dry_run=self.dry_run, ) + @property + def has_rollback(self) -> bool: + return bool(self._rollback_stack) + def log_rollback(self): - if self._rollback_stack: + if self.has_rollback: rollback = self._rollback_stack.copy() rollback.reverse() logging.info( @@ -635,7 +639,20 @@ def main(): repo, args.commit, args.release_type, args.dry_run, args.with_stderr ) - release.do(args.check_dirty, args.check_branch) + try: + release.do(args.check_dirty, args.check_branch) + except: + if release.has_rollback: + logging.error( + "!!The release process finished with error, read the output carefully!!" + ) + logging.error( + "Probably, rollback finished with error. " + "If you don't see any of the following commands in the output, " + "execute them manually:" + ) + release.log_rollback() + raise if __name__ == "__main__": From 9c7fc297787e01ca53d3feb8fbece60ddbf31f87 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Fri, 17 Feb 2023 20:39:38 +0100 Subject: [PATCH 442/717] Add dedicated key for checking launching release.py from master --- tests/ci/release.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tests/ci/release.py b/tests/ci/release.py index d5537f106c5..a4fe4046572 100755 --- a/tests/ci/release.py +++ b/tests/ci/release.py @@ -173,7 +173,9 @@ class Release: self.check_commit_release_ready() - def do(self, check_dirty: bool, check_branch: bool) -> None: + def do( + self, check_dirty: bool, check_run_from_master: bool, check_branch: bool + ) -> None: self.check_prerequisites() if check_dirty: @@ -183,8 +185,9 @@ class Release: except subprocess.CalledProcessError: logging.fatal("Repo contains uncommitted changes") raise - if self._git.branch != "master": - raise Exception("the script must be launched only from master") + + if check_run_from_master and self._git.branch != "master": + raise Exception("the script must be launched only from master") self.set_release_info() @@ -606,6 +609,14 @@ def parse_args() -> argparse.Namespace: default=argparse.SUPPRESS, help="(dangerous) if set, skip check repository for uncommited changes", ) + parser.add_argument("--check-run-from-master", default=True, help=argparse.SUPPRESS) + parser.add_argument( + "--no-run-from-master", + dest="check_run_from_master", + action="store_false", + default=argparse.SUPPRESS, + help="(for development) if set, the script could run from non-master branch", + ) parser.add_argument("--check-branch", default=True, help=argparse.SUPPRESS) parser.add_argument( "--no-check-branch", @@ -640,7 +651,7 @@ def main(): ) try: - release.do(args.check_dirty, args.check_branch) + release.do(args.check_dirty, args.check_run_from_master, args.check_branch) except: if release.has_rollback: logging.error( From dbb4bdee1d0f1fdb5f8fbb6c7119cda6036ac81b Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 20 Feb 2023 11:40:04 +0100 Subject: [PATCH 443/717] Allow parallel execution of 02003_memory_limit_in_client Signed-off-by: Azat Khuzhin --- tests/queries/0_stateless/02003_memory_limit_in_client.expect | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02003_memory_limit_in_client.expect b/tests/queries/0_stateless/02003_memory_limit_in_client.expect index e03fb9308f4..2b59b54b4dc 100755 --- a/tests/queries/0_stateless/02003_memory_limit_in_client.expect +++ b/tests/queries/0_stateless/02003_memory_limit_in_client.expect @@ -1,5 +1,5 @@ #!/usr/bin/expect -f -# Tags: no-parallel, no-fasttest +# Tags: no-fasttest # This is a test for system.warnings. Testing in interactive mode is necessary, # as we want to see certain warnings from client From 9bdb3220b920c50c12a76050840ff6a2240c88c5 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 20 Feb 2023 12:28:09 +0100 Subject: [PATCH 444/717] tests: add a note about alacritty for 02132_client_history_navigation Signed-off-by: Azat Khuzhin --- tests/queries/0_stateless/02132_client_history_navigation.expect | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/02132_client_history_navigation.expect b/tests/queries/0_stateless/02132_client_history_navigation.expect index 82e994f4fc1..ec7b69ad90e 100755 --- a/tests/queries/0_stateless/02132_client_history_navigation.expect +++ b/tests/queries/0_stateless/02132_client_history_navigation.expect @@ -26,6 +26,7 @@ send -- "SELECT 1\r" expect "1" expect ":) " send -- "SELECT 2" +# NOTE: it does not work for alacritty with TERM=xterm send -- "\033\[A" expect "SELECT 1" send -- "\033\[B" From 34cda2bcb255c99c69ce4a5fbeeead3015f55e98 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Mon, 20 Feb 2023 12:54:13 +0100 Subject: [PATCH 445/717] Update 00170_s3_cache.sql --- tests/queries/1_stateful/00170_s3_cache.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/1_stateful/00170_s3_cache.sql b/tests/queries/1_stateful/00170_s3_cache.sql index 81592255428..43e85af0bc3 100644 --- a/tests/queries/1_stateful/00170_s3_cache.sql +++ b/tests/queries/1_stateful/00170_s3_cache.sql @@ -2,6 +2,7 @@ -- { echo } +SET allow_prefetched_read_pool_for_remote_filesystem=0; SET enable_filesystem_cache_on_write_operations=0; SET max_memory_usage='20G'; SYSTEM DROP FILESYSTEM CACHE; From 9fb7d4fd059994664744e11610979edcaf6b3c1f Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Mon, 20 Feb 2023 12:55:24 +0100 Subject: [PATCH 446/717] Update .reference --- tests/queries/1_stateful/00170_s3_cache.reference | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/1_stateful/00170_s3_cache.reference b/tests/queries/1_stateful/00170_s3_cache.reference index 04d610bc8d2..293fbd7f8cb 100644 --- a/tests/queries/1_stateful/00170_s3_cache.reference +++ b/tests/queries/1_stateful/00170_s3_cache.reference @@ -1,5 +1,6 @@ -- { echo } +SET allow_prefetched_read_pool_for_remote_filesystem=0; SET enable_filesystem_cache_on_write_operations=0; SET max_memory_usage='20G'; SYSTEM DROP FILESYSTEM CACHE; From 78c35ffc45172c8b9976d4dbc164f5c637e96919 Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 20 Feb 2023 12:28:44 +0000 Subject: [PATCH 447/717] finally fix attachProfileCountersScope --- src/Interpreters/ThreadStatusExt.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/ThreadStatusExt.cpp b/src/Interpreters/ThreadStatusExt.cpp index 8b84d4cae9b..84400fc3711 100644 --- a/src/Interpreters/ThreadStatusExt.cpp +++ b/src/Interpreters/ThreadStatusExt.cpp @@ -169,8 +169,10 @@ ProfileEvents::Counters * ThreadStatus::attachProfileCountersScope(ProfileEvents /// Allow to attach the same scope multiple times return prev_counters; - if (!performance_counters_scope->getParent()) + /// Avoid cycles when exiting local scope and attaching back to current thread counters + if (performance_counters_scope != &performance_counters) performance_counters_scope->setParent(&performance_counters); + current_performance_counters = performance_counters_scope; return prev_counters; From a6d420a1c64e99ddc0977af69e12ae6f7dbc8959 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 20 Feb 2023 13:55:42 +0100 Subject: [PATCH 448/717] One cannot use /dev/null as a history file since replxx run chmod Signed-off-by: Azat Khuzhin --- .../0_stateless/01179_insert_values_semicolon.expect | 3 ++- .../queries/0_stateless/01180_client_syntax_errors.expect | 3 ++- .../01293_client_interactive_vertical_singleline.expect | 3 ++- .../01370_client_autocomplete_word_break_characters.expect | 3 ++- .../queries/0_stateless/01520_client_print_query_id.expect | 3 ++- .../0_stateless/01565_reconnect_after_client_error.expect | 3 ++- ...5_client_highlight_multi_line_comment_regression.expect | 3 ++- tests/queries/0_stateless/01945_show_debug_warning.expect | 7 ++++--- .../0_stateless/02003_memory_limit_in_client.expect | 7 ++++--- tests/queries/0_stateless/02047_client_exception.expect | 3 ++- .../0_stateless/02105_backslash_letter_commands.expect | 3 ++- tests/queries/0_stateless/02116_interactive_hello.expect | 3 ++- .../0_stateless/02132_client_history_navigation.expect | 3 ++- .../02160_client_autocomplete_parse_query.expect | 3 ++- .../queries/0_stateless/02417_repeat_input_commands.expect | 3 ++- 15 files changed, 34 insertions(+), 19 deletions(-) diff --git a/tests/queries/0_stateless/01179_insert_values_semicolon.expect b/tests/queries/0_stateless/01179_insert_values_semicolon.expect index d66a84769f2..35713a90297 100755 --- a/tests/queries/0_stateless/01179_insert_values_semicolon.expect +++ b/tests/queries/0_stateless/01179_insert_values_semicolon.expect @@ -4,6 +4,7 @@ set basedir [file dirname $argv0] set basename [file tail $argv0] exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 +set history_file $env(CLICKHOUSE_TMP)/$basename.history log_user 0 set timeout 60 @@ -15,7 +16,7 @@ expect_after { timeout { exit 1 } } -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --history_file=/dev/null" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --history_file=$history_file" expect ":) " send -- "DROP TABLE IF EXISTS test_01179\r" diff --git a/tests/queries/0_stateless/01180_client_syntax_errors.expect b/tests/queries/0_stateless/01180_client_syntax_errors.expect index a031333313a..c1fd0f93510 100755 --- a/tests/queries/0_stateless/01180_client_syntax_errors.expect +++ b/tests/queries/0_stateless/01180_client_syntax_errors.expect @@ -3,6 +3,7 @@ set basedir [file dirname $argv0] set basename [file tail $argv0] exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 +set history_file $env(CLICKHOUSE_TMP)/$basename.history log_user 0 set timeout 60 @@ -14,7 +15,7 @@ expect_after { timeout { exit 1 } } -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --history_file=/dev/null" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --history_file=$history_file" expect ":) " # Make a query with syntax error diff --git a/tests/queries/0_stateless/01293_client_interactive_vertical_singleline.expect b/tests/queries/0_stateless/01293_client_interactive_vertical_singleline.expect index e64d71c6319..6b11b1eee15 100755 --- a/tests/queries/0_stateless/01293_client_interactive_vertical_singleline.expect +++ b/tests/queries/0_stateless/01293_client_interactive_vertical_singleline.expect @@ -3,6 +3,7 @@ set basedir [file dirname $argv0] set basename [file tail $argv0] exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 +set history_file $env(CLICKHOUSE_TMP)/$basename.history log_user 0 set timeout 60 @@ -14,7 +15,7 @@ expect_after { timeout { exit 1 } } -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --history_file=/dev/null" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --history_file=$history_file" expect ":) " send -- "SELECT 1\r" diff --git a/tests/queries/0_stateless/01370_client_autocomplete_word_break_characters.expect b/tests/queries/0_stateless/01370_client_autocomplete_word_break_characters.expect index bbd0fdd223b..8547be839d4 100755 --- a/tests/queries/0_stateless/01370_client_autocomplete_word_break_characters.expect +++ b/tests/queries/0_stateless/01370_client_autocomplete_word_break_characters.expect @@ -3,6 +3,7 @@ set basedir [file dirname $argv0] set basename [file tail $argv0] exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 +set history_file $env(CLICKHOUSE_TMP)/$basename.history log_user 0 set timeout 60 @@ -14,7 +15,7 @@ expect_after { timeout { exit 1 } } -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --history_file=/dev/null" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --history_file=$history_file" expect ":) " # Make a query diff --git a/tests/queries/0_stateless/01520_client_print_query_id.expect b/tests/queries/0_stateless/01520_client_print_query_id.expect index b16cd6d499b..cbeacc6a4ec 100755 --- a/tests/queries/0_stateless/01520_client_print_query_id.expect +++ b/tests/queries/0_stateless/01520_client_print_query_id.expect @@ -3,6 +3,7 @@ set basedir [file dirname $argv0] set basename [file tail $argv0] exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 +set history_file $env(CLICKHOUSE_TMP)/$basename.history log_user 0 set timeout 60 @@ -14,7 +15,7 @@ expect_after { timeout { exit 1 } } -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --history_file=/dev/null" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --history_file=$history_file" expect ":) " # Make a query diff --git a/tests/queries/0_stateless/01565_reconnect_after_client_error.expect b/tests/queries/0_stateless/01565_reconnect_after_client_error.expect index 076b91390bd..255248ba61a 100755 --- a/tests/queries/0_stateless/01565_reconnect_after_client_error.expect +++ b/tests/queries/0_stateless/01565_reconnect_after_client_error.expect @@ -7,6 +7,7 @@ set basedir [file dirname $argv0] set basename [file tail $argv0] exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 +set history_file $env(CLICKHOUSE_TMP)/$basename.history log_user 0 set timeout 60 @@ -19,7 +20,7 @@ expect_after { timeout { exit 1 } } -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion -mn --history_file=/dev/null" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion -mn --history_file=$history_file" expect "\n:) " send -- "DROP TABLE IF EXISTS t01565;\n" diff --git a/tests/queries/0_stateless/01755_client_highlight_multi_line_comment_regression.expect b/tests/queries/0_stateless/01755_client_highlight_multi_line_comment_regression.expect index 293fd2e13f4..223690f1f8b 100755 --- a/tests/queries/0_stateless/01755_client_highlight_multi_line_comment_regression.expect +++ b/tests/queries/0_stateless/01755_client_highlight_multi_line_comment_regression.expect @@ -3,6 +3,7 @@ set basedir [file dirname $argv0] set basename [file tail $argv0] exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 +set history_file $env(CLICKHOUSE_TMP)/$basename.history log_user 0 set timeout 60 @@ -14,7 +15,7 @@ expect_after { timeout { exit 1 } } -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --history_file=/dev/null" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --history_file=$history_file" expect ":) " # regression for heap-buffer-overflow issue (under ASAN) diff --git a/tests/queries/0_stateless/01945_show_debug_warning.expect b/tests/queries/0_stateless/01945_show_debug_warning.expect index 6714901ab20..c93635b3b27 100755 --- a/tests/queries/0_stateless/01945_show_debug_warning.expect +++ b/tests/queries/0_stateless/01945_show_debug_warning.expect @@ -6,6 +6,7 @@ set basedir [file dirname $argv0] set basename [file tail $argv0] exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 +set history_file $env(CLICKHOUSE_TMP)/$basename.history log_user 0 set timeout 60 @@ -20,7 +21,7 @@ expect_after { set Debug_type 0 -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --history_file=/dev/null" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --history_file=$history_file" expect ":) " # Check debug type @@ -38,7 +39,7 @@ expect eof if { $Debug_type > 0} { -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --history_file=/dev/null" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --history_file=$history_file" expect "Warnings:" expect " * Server was built in debug mode. It will work slowly." expect ":) " @@ -52,7 +53,7 @@ send -- "q\r" expect eof } -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --max_memory_usage_for_all_queries=123 --history_file=/dev/null" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --max_memory_usage_for_all_queries=123 --history_file=$history_file" expect "Warnings:" expect " * Some obsolete setting is changed." expect ":) " diff --git a/tests/queries/0_stateless/02003_memory_limit_in_client.expect b/tests/queries/0_stateless/02003_memory_limit_in_client.expect index 2b59b54b4dc..4f28fafc1e6 100755 --- a/tests/queries/0_stateless/02003_memory_limit_in_client.expect +++ b/tests/queries/0_stateless/02003_memory_limit_in_client.expect @@ -7,6 +7,7 @@ set basedir [file dirname $argv0] set basename [file tail $argv0] exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 +set history_file $env(CLICKHOUSE_TMP)/$basename.history log_user 0 set timeout 60 @@ -22,7 +23,7 @@ expect_after { # # Check that the query will fail in clickhouse-client # -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --max_memory_usage_in_client=1 --history_file=/dev/null" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --max_memory_usage_in_client=1 --history_file=$history_file" expect ":) " send -- "SELECT arrayMap(x -> range(x), range(number)) FROM numbers(1000)\r" @@ -37,7 +38,7 @@ expect eof # # Check that the query will fail in clickhouse-client # -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --max_memory_usage_in_client=1 --history_file=/dev/null" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --max_memory_usage_in_client=1 --history_file=$history_file" expect ":) " send -- "SELECT arrayMap(x -> range(x), range(number)) FROM numbers(1000)\r" @@ -52,7 +53,7 @@ expect eof # # Check that the query will not fail (due to max_untracked_memory) # -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --max_memory_usage_in_client=1 --history_file=/dev/null" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --max_memory_usage_in_client=1 --history_file=$history_file" expect ":) " send -- "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000\r" diff --git a/tests/queries/0_stateless/02047_client_exception.expect b/tests/queries/0_stateless/02047_client_exception.expect index 9ae8495cfcb..69f468907a3 100755 --- a/tests/queries/0_stateless/02047_client_exception.expect +++ b/tests/queries/0_stateless/02047_client_exception.expect @@ -3,6 +3,7 @@ set basedir [file dirname $argv0] set basename [file tail $argv0] exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 +set history_file $env(CLICKHOUSE_TMP)/$basename.history log_user 0 set timeout 20 @@ -15,7 +16,7 @@ expect_after { timeout { exit 1 } } -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --history_file=/dev/null" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --history_file=$history_file" expect ":) " send -- "DROP TABLE IF EXISTS test_02047\r" diff --git a/tests/queries/0_stateless/02105_backslash_letter_commands.expect b/tests/queries/0_stateless/02105_backslash_letter_commands.expect index a7340020611..8f8ec1f5abd 100755 --- a/tests/queries/0_stateless/02105_backslash_letter_commands.expect +++ b/tests/queries/0_stateless/02105_backslash_letter_commands.expect @@ -3,6 +3,7 @@ set basedir [file dirname $argv0] set basename [file tail $argv0] exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 +set history_file $env(CLICKHOUSE_TMP)/$basename.history log_user 0 set timeout 60 @@ -14,7 +15,7 @@ expect_after { timeout { exit 1 } } -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --history_file=/dev/null" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --history_file=$history_file" expect ":) " # Send a command diff --git a/tests/queries/0_stateless/02116_interactive_hello.expect b/tests/queries/0_stateless/02116_interactive_hello.expect index 443f835b76d..7e895196304 100755 --- a/tests/queries/0_stateless/02116_interactive_hello.expect +++ b/tests/queries/0_stateless/02116_interactive_hello.expect @@ -4,6 +4,7 @@ set basedir [file dirname $argv0] set basename [file tail $argv0] exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 +set history_file $env(CLICKHOUSE_TMP)/$basename.history log_user 0 set timeout 60 @@ -16,7 +17,7 @@ expect_after { timeout { exit 1 } } -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --history_file=/dev/null" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --history_file=$history_file" expect -re "ClickHouse client version \[\\d\]{2}.\[\\d\]{1,2}.\[\\d\]{1,2}.\[\\d\]{1,2}.\r" expect -re "Connecting to database .* at localhost:9000 as user default.\r" diff --git a/tests/queries/0_stateless/02132_client_history_navigation.expect b/tests/queries/0_stateless/02132_client_history_navigation.expect index ec7b69ad90e..3316f26d552 100755 --- a/tests/queries/0_stateless/02132_client_history_navigation.expect +++ b/tests/queries/0_stateless/02132_client_history_navigation.expect @@ -3,6 +3,7 @@ set basedir [file dirname $argv0] set basename [file tail $argv0] exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 +set history_file $env(CLICKHOUSE_TMP)/$basename.history log_user 0 set timeout 60 @@ -18,7 +19,7 @@ expect_after { # useful debugging configuration # exp_internal 1 -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --highlight 0 --history_file=/dev/null" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --highlight 0 --history_file=$history_file" expect ":) " # Make a query diff --git a/tests/queries/0_stateless/02160_client_autocomplete_parse_query.expect b/tests/queries/0_stateless/02160_client_autocomplete_parse_query.expect index 12bfcc7c041..41d32891e98 100755 --- a/tests/queries/0_stateless/02160_client_autocomplete_parse_query.expect +++ b/tests/queries/0_stateless/02160_client_autocomplete_parse_query.expect @@ -3,6 +3,7 @@ set basedir [file dirname $argv0] set basename [file tail $argv0] exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 +set history_file $env(CLICKHOUSE_TMP)/$basename.history log_user 0 set timeout 60 @@ -15,7 +16,7 @@ expect_after { timeout { exit 1 } } -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --history_file=/dev/null" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --history_file=$history_file" expect ":) " # Make a query diff --git a/tests/queries/0_stateless/02417_repeat_input_commands.expect b/tests/queries/0_stateless/02417_repeat_input_commands.expect index 85be0a02881..3658d5d8494 100755 --- a/tests/queries/0_stateless/02417_repeat_input_commands.expect +++ b/tests/queries/0_stateless/02417_repeat_input_commands.expect @@ -3,6 +3,7 @@ set basedir [file dirname $argv0] set basename [file tail $argv0] exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 +set history_file $env(CLICKHOUSE_TMP)/$basename.history log_user 0 set timeout 60 @@ -15,7 +16,7 @@ expect_after { timeout { exit 1 } } -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --history_file=/dev/null" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --history_file=$history_file" expect ":) " # ----------------------------------------- From 361678ad7327d1c41b78c6eb5b744c178410812a Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 20 Feb 2023 13:55:27 +0100 Subject: [PATCH 449/717] rabbitmq-test-fix --- src/Storages/RabbitMQ/RabbitMQHandler.cpp | 4 ++-- src/Storages/RabbitMQ/RabbitMQHandler.h | 2 +- src/Storages/RabbitMQ/RabbitMQProducer.cpp | 15 ++++++++++++++- tests/integration/test_storage_rabbitmq/test.py | 14 +++++++------- 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/src/Storages/RabbitMQ/RabbitMQHandler.cpp b/src/Storages/RabbitMQ/RabbitMQHandler.cpp index 934753257b4..745af0d20e3 100644 --- a/src/Storages/RabbitMQ/RabbitMQHandler.cpp +++ b/src/Storages/RabbitMQ/RabbitMQHandler.cpp @@ -56,10 +56,10 @@ int RabbitMQHandler::iterateLoop() /// Do not need synchronization as in iterateLoop(), because this method is used only for /// initial RabbitMQ setup - at this point there is no background loop thread. -void RabbitMQHandler::startBlockingLoop() +int RabbitMQHandler::startBlockingLoop() { LOG_DEBUG(log, "Started blocking loop."); - uv_run(loop, UV_RUN_DEFAULT); + return uv_run(loop, UV_RUN_DEFAULT); } void RabbitMQHandler::stopLoop() diff --git a/src/Storages/RabbitMQ/RabbitMQHandler.h b/src/Storages/RabbitMQ/RabbitMQHandler.h index 948d56416fd..4223732a4a0 100644 --- a/src/Storages/RabbitMQ/RabbitMQHandler.h +++ b/src/Storages/RabbitMQ/RabbitMQHandler.h @@ -38,7 +38,7 @@ public: /// Loop to wait for small tasks in a blocking mode. /// No synchronization is done with the main loop thread. - void startBlockingLoop(); + int startBlockingLoop(); void stopLoop(); diff --git a/src/Storages/RabbitMQ/RabbitMQProducer.cpp b/src/Storages/RabbitMQ/RabbitMQProducer.cpp index 5d639b77f53..246569060d0 100644 --- a/src/Storages/RabbitMQ/RabbitMQProducer.cpp +++ b/src/Storages/RabbitMQ/RabbitMQProducer.cpp @@ -262,7 +262,20 @@ void RabbitMQProducer::startProducingTaskLoop() LOG_TEST(log, "Waiting for pending callbacks to finish (count: {}, try: {})", res, try_num); } - LOG_DEBUG(log, "Producer on channel {} completed", channel_id); + producer_channel->close() + .onSuccess([&]() + { + LOG_TRACE(log, "Successfully closed producer channel"); + connection.getHandler().stopLoop(); + }) + .onError([&](const char * message) + { + LOG_ERROR(log, "Failed to close producer channel: {}", message); + connection.getHandler().stopLoop(); + }); + + int active = connection.getHandler().startBlockingLoop(); + LOG_DEBUG(log, "Producer on channel completed (not finished events: {})", active); } diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index 030d9507d4f..5ca6f2acedf 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -1086,6 +1086,9 @@ def test_rabbitmq_overloaded_insert(rabbitmq_cluster): time.sleep(random.uniform(0, 1)) thread.start() + for thread in threads: + thread.join() + while True: result = instance.query("SELECT count() FROM test.view_overload") expected = messages_num * threads_num @@ -1096,16 +1099,13 @@ def test_rabbitmq_overloaded_insert(rabbitmq_cluster): instance.query( """ - DROP TABLE test.consumer_overload; - DROP TABLE test.view_overload; - DROP TABLE test.rabbitmq_consume; - DROP TABLE test.rabbitmq_overload; + DROP TABLE test.consumer_overload NO DELAY; + DROP TABLE test.view_overload NO DELAY; + DROP TABLE test.rabbitmq_consume NO DELAY; + DROP TABLE test.rabbitmq_overload NO DELAY; """ ) - for thread in threads: - thread.join() - assert ( int(result) == messages_num * threads_num ), "ClickHouse lost some messages: {}".format(result) From 514e3cbf90487a304f718290aea4d2ca45e3b99c Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Mon, 20 Feb 2023 14:29:04 +0100 Subject: [PATCH 450/717] Update test.py --- .../test_storage_meilisearch/test.py | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_storage_meilisearch/test.py b/tests/integration/test_storage_meilisearch/test.py index 32da7f20155..66c32459251 100644 --- a/tests/integration/test_storage_meilisearch/test.py +++ b/tests/integration/test_storage_meilisearch/test.py @@ -58,6 +58,7 @@ def test_simple_select(started_cluster): push_data(client, table, data) node = started_cluster.instances["meili"] + node.query("DROP TABLE IF EXISTS simple_meili_table") node.query( "CREATE TABLE simple_meili_table(id UInt64, data String) ENGINE = MeiliSearch('http://meili1:7700', 'new_table', '')" ) @@ -83,6 +84,7 @@ def test_insert(started_cluster): big_table = client.index("big_table") node = started_cluster.instances["meili"] + node.query("DROP TABLE IF EXISTS new_table") node.query( "CREATE TABLE new_table(id UInt64, data String) ENGINE = MeiliSearch('http://meili1:7700', 'new_table', '')" ) @@ -90,9 +92,10 @@ def test_insert(started_cluster): node.query( "INSERT INTO new_table (id, data) VALUES (1, '1') (2, '2') (3, '3') (4, '4') (5, '5') (6, '6') (7, '7')" ) - sleep(1) + sleep(5) assert len(new_table.get_documents()) == 7 + node.query("DROP TABLE IF EXISTS big_table") node.query( "CREATE TABLE big_table(id UInt64, data String) ENGINE = MeiliSearch('http://meili1:7700', 'big_table', '')" ) @@ -124,6 +127,7 @@ def test_meilimatch(started_cluster): push_movies(client) node = started_cluster.instances["meili"] + node.query("DROP TABLE IF EXISTS movies_table") node.query( "CREATE TABLE movies_table(id String, title String, release_date Int64) ENGINE = MeiliSearch('http://meili1:7700', 'movies', '')" ) @@ -208,6 +212,7 @@ def test_incorrect_data_type(started_cluster): push_data(client, table, data) node = started_cluster.instances["meili"] + node.query("DROP TABLE IF EXISTS strange_meili_table") node.query( "CREATE TABLE strange_meili_table(id UInt64, data String, bbbb String) ENGINE = MeiliSearch('http://meili1:7700', 'new_table', '')" ) @@ -230,10 +235,12 @@ def test_simple_select_secure(started_cluster): push_data(client, table, data) node = started_cluster.instances["meili"] + node.query("DROP TABLE IF EXISTS simple_meili_table") node.query( "CREATE TABLE simple_meili_table(id UInt64, data String) ENGINE = MeiliSearch('http://meili_secure:7700', 'new_table', 'password')" ) + node.query("DROP TABLE IF EXISTS wrong_meili_table") node.query( "CREATE TABLE wrong_meili_table(id UInt64, data String) ENGINE = MeiliSearch('http://meili_secure:7700', 'new_table', 'wrong_password')" ) @@ -272,6 +279,7 @@ def test_meilimatch_secure(started_cluster): push_movies(client) node = started_cluster.instances["meili"] + node.query("DROP TABLE IF EXISTS movies_table") node.query( "CREATE TABLE movies_table(id String, title String, release_date Int64) ENGINE = MeiliSearch('http://meili_secure:7700', 'movies', 'password')" ) @@ -356,6 +364,7 @@ def test_incorrect_data_type_secure(started_cluster): push_data(client, table, data) node = started_cluster.instances["meili"] + node.query("DROP TABLE IF EXISTS strange_meili_table") node.query( "CREATE TABLE strange_meili_table(id UInt64, data String, bbbb String) ENGINE = MeiliSearch('http://meili_secure:7700', 'new_table', 'password')" ) @@ -374,6 +383,7 @@ def test_insert_secure(started_cluster): big_table = client.index("big_table") node = started_cluster.instances["meili"] + node.query("DROP TABLE IF EXISTS new_table") node.query( "CREATE TABLE new_table(id UInt64, data String) ENGINE = MeiliSearch('http://meili_secure:7700', 'new_table', 'password')" ) @@ -381,9 +391,10 @@ def test_insert_secure(started_cluster): node.query( "INSERT INTO new_table (id, data) VALUES (1, '1') (2, '2') (3, '3') (4, '4') (5, '5') (6, '6') (7, '7')" ) - sleep(1) + sleep(5) assert len(new_table.get_documents()) == 7 + node.query("DROP TABLE IF EXISTS big_table") node.query( "CREATE TABLE big_table(id UInt64, data String) ENGINE = MeiliSearch('http://meili_secure:7700', 'big_table', 'password')" ) @@ -417,9 +428,11 @@ def test_security_levels(started_cluster): values += "(" + str(i) + ", " + "'" + str(i) + "'" + ") " node = started_cluster.instances["meili"] + node.query("DROP TABLE IF EXISTS read_table") node.query( f"CREATE TABLE read_table(id UInt64, data String) ENGINE = MeiliSearch('http://meili_secure:7700', 'new_table', '{search_key}')" ) + node.query("DROP TABLE IF EXISTS write_table") node.query( f"CREATE TABLE write_table(id UInt64, data String) ENGINE = MeiliSearch('http://meili_secure:7700', 'new_table', '{admin_key}')" ) @@ -430,7 +443,7 @@ def test_security_levels(started_cluster): assert "MEILISEARCH_EXCEPTION" in error node.query("INSERT INTO write_table (id, data) VALUES " + values) - sleep(1) + sleep(5) assert len(new_table.get_documents({"limit": 40010})) == 100 ans1 = ( @@ -493,6 +506,7 @@ def test_types(started_cluster): push_data(client, table, data) node = started_cluster.instances["meili"] + node.query("DROP TABLE IF EXISTS types_table") node.query( "CREATE TABLE types_table(\ id UInt64,\ @@ -556,6 +570,7 @@ def test_named_collection(started_cluster): push_data(client, table, data) node = started_cluster.instances["meili"] + node.query("DROP TABLE IF EXISTS simple_meili_table") node.query( "CREATE TABLE simple_meili_table(id UInt64, data String) ENGINE = MeiliSearch( named_collection_for_meili )" ) @@ -589,14 +604,17 @@ def test_named_collection_secure(started_cluster): push_data(client_free, table_free, data) node = started_cluster.instances["meili"] + node.query("DROP TABLE IF EXISTS simple_meili_table") node.query( "CREATE TABLE simple_meili_table(id UInt64, data String) ENGINE = MeiliSearch( named_collection_for_meili_secure )" ) + node.query("DROP TABLE IF EXISTS wrong_meili_table") node.query( "CREATE TABLE wrong_meili_table(id UInt64, data String) ENGINE = MeiliSearch( named_collection_for_meili_secure_no_password )" ) + node.query("DROP TABLE IF EXISTS combine_meili_table") node.query( 'CREATE TABLE combine_meili_table(id UInt64, data String) ENGINE = MeiliSearch( named_collection_for_meili_secure_no_password, key="password" )' ) From 591510c20ece9b3f13caf29a68fe3bf2e8a40659 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 20 Feb 2023 14:36:53 +0100 Subject: [PATCH 451/717] Uodate test --- .../test_postgresql_replica_database_engine_1/test.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tests/integration/test_postgresql_replica_database_engine_1/test.py b/tests/integration/test_postgresql_replica_database_engine_1/test.py index c7a503f8f16..377b1c89efc 100644 --- a/tests/integration/test_postgresql_replica_database_engine_1/test.py +++ b/tests/integration/test_postgresql_replica_database_engine_1/test.py @@ -581,16 +581,6 @@ def test_virtual_columns(started_cluster): ) print(result) - instance.query( - f"INSERT INTO postgres_database.{table_name} SELECT number, number, number from numbers(20, 10)" - ) - check_tables_are_synchronized(instance, table_name) - - result = instance.query( - f"SELECT key, value, value2, _sign, _version FROM test_database.{table_name};" - ) - print(result) - def test_multiple_databases(started_cluster): NUM_TABLES = 5 From 1539f812b30657b4f96d63c562d1584ca8bfa219 Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 20 Feb 2023 13:48:52 +0000 Subject: [PATCH 452/717] Fix processing Const(LowCardinality) in arrayMap --- src/Functions/array/FunctionArrayMapped.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Functions/array/FunctionArrayMapped.h b/src/Functions/array/FunctionArrayMapped.h index 89599edd9d1..8d427bcf691 100644 --- a/src/Functions/array/FunctionArrayMapped.h +++ b/src/Functions/array/FunctionArrayMapped.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -341,9 +342,15 @@ public: replicated_column_function->appendArguments(arrays); auto lambda_result = replicated_column_function->reduce(); + + /// Convert LowCardinality(T) -> T and Const(LowCardinality(T)) -> Const(T), + /// because we removed LowCardinality from return type of lambda expression. if (lambda_result.column->lowCardinality()) lambda_result.column = lambda_result.column->convertToFullColumnIfLowCardinality(); + if (const auto * const_column = checkAndGetColumnConst(lambda_result.column.get())) + lambda_result.column = const_column->removeLowCardinality(); + if (Impl::needBoolean()) { /// If result column is Nothing or Nullable(Nothing), just create const UInt8 column with 0 value. From a80ce0219996bff88c8685087be5c8d03babe687 Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 20 Feb 2023 13:50:20 +0000 Subject: [PATCH 453/717] Update test --- .../0_stateless/02667_array_map_const_low_cardinality.reference | 1 + ...g_structure.sql => 02667_array_map_const_low_cardinality.sql} | 0 tests/queries/0_stateless/02667_wrong_structure.reference | 0 3 files changed, 1 insertion(+) create mode 100644 tests/queries/0_stateless/02667_array_map_const_low_cardinality.reference rename tests/queries/0_stateless/{02667_wrong_structure.sql => 02667_array_map_const_low_cardinality.sql} (100%) delete mode 100644 tests/queries/0_stateless/02667_wrong_structure.reference diff --git a/tests/queries/0_stateless/02667_array_map_const_low_cardinality.reference b/tests/queries/0_stateless/02667_array_map_const_low_cardinality.reference new file mode 100644 index 00000000000..22fd87520ef --- /dev/null +++ b/tests/queries/0_stateless/02667_array_map_const_low_cardinality.reference @@ -0,0 +1 @@ +[2] diff --git a/tests/queries/0_stateless/02667_wrong_structure.sql b/tests/queries/0_stateless/02667_array_map_const_low_cardinality.sql similarity index 100% rename from tests/queries/0_stateless/02667_wrong_structure.sql rename to tests/queries/0_stateless/02667_array_map_const_low_cardinality.sql diff --git a/tests/queries/0_stateless/02667_wrong_structure.reference b/tests/queries/0_stateless/02667_wrong_structure.reference deleted file mode 100644 index e69de29bb2d..00000000000 From 3272e7d7787fe32fc33a232d012d0939e2beb1f7 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Mon, 20 Feb 2023 15:23:24 +0100 Subject: [PATCH 454/717] Fix dependencies for InstallPackagesTestAarch64 --- .github/workflows/backport_branches.yml | 2 +- .github/workflows/master.yml | 2 +- .github/workflows/pull_request.yml | 2 +- .github/workflows/release_branches.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/backport_branches.yml b/.github/workflows/backport_branches.yml index 7d7efc51fa9..110c06631c7 100644 --- a/.github/workflows/backport_branches.yml +++ b/.github/workflows/backport_branches.yml @@ -549,7 +549,7 @@ jobs: docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" InstallPackagesTestAarch64: - needs: [BuilderDebRelease] + needs: [BuilderDebAarch64] runs-on: [self-hosted, style-checker-aarch64] steps: - name: Set envs diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 6e728b6bfb0..7e045992dee 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -983,7 +983,7 @@ jobs: docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" InstallPackagesTestAarch64: - needs: [BuilderDebRelease] + needs: [BuilderDebAarch64] runs-on: [self-hosted, style-checker-aarch64] steps: - name: Set envs diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 1a46cf9a3d4..bdf9c0615a8 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -1021,7 +1021,7 @@ jobs: docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" InstallPackagesTestAarch64: - needs: [BuilderDebRelease] + needs: [BuilderDebAarch64] runs-on: [self-hosted, style-checker-aarch64] steps: - name: Set envs diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 95ef60686a7..4d2a99c2106 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -641,7 +641,7 @@ jobs: docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" InstallPackagesTestAarch64: - needs: [BuilderDebRelease] + needs: [BuilderDebAarch64] runs-on: [self-hosted, style-checker-aarch64] steps: - name: Set envs From 669c4e94d50a4d4886ff4ffe6b9c3ee66dc921e0 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 20 Feb 2023 17:35:30 +0300 Subject: [PATCH 455/717] Update compare.sh --- docker/test/performance-comparison/compare.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 338a0c02a55..725dcbd7157 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -538,7 +538,7 @@ unset IFS numactl --show numactl --cpunodebind=all --membind=all numactl --show # Use less jobs to avoid OOM. Some queries can consume 8+ GB of memory. -jobs_count=$(($(grep -c ^processor /proc/cpuinfo) / 3)) +jobs_count=$(($(grep -c ^processor /proc/cpuinfo) / 4)) numactl --cpunodebind=all --membind=all parallel --jobs $jobs_count --joblog analyze/parallel-log.txt --null < analyze/commands.txt 2>> analyze/errors.log clickhouse-local --query " From 4715e2c172fc06728a18b32a3a6444b614ea6c77 Mon Sep 17 00:00:00 2001 From: Han Fei Date: Mon, 20 Feb 2023 15:36:45 +0100 Subject: [PATCH 456/717] update llvm-project to fix gwp-asan --- contrib/llvm-project | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/llvm-project b/contrib/llvm-project index e61a81aa6fc..a8bf69e9cd3 160000 --- a/contrib/llvm-project +++ b/contrib/llvm-project @@ -1 +1 @@ -Subproject commit e61a81aa6fc529b469e2a54b7ce788606e138b5d +Subproject commit a8bf69e9cd39a23140a2b633c172d201484172da From 32de43753375f99f883beda91efca15204afa109 Mon Sep 17 00:00:00 2001 From: Dan Roscigno Date: Mon, 20 Feb 2023 09:49:14 -0500 Subject: [PATCH 457/717] Update changelog README --- utils/changelog/README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/utils/changelog/README.md b/utils/changelog/README.md index 739229b49c9..ccc235c4990 100644 --- a/utils/changelog/README.md +++ b/utils/changelog/README.md @@ -12,8 +12,13 @@ python3 changelog.py -h Usage example: -``` -git fetch --tags # changelog.py depends on having the tags available, this will fetch them +Note: The working directory is ClickHouse/utils/changelog + +```bash +export GITHUB_TOKEN="" + +git fetch --tags # changelog.py depends on having the tags available, this will fetch them. + # If you are working from a branch in your personal fork, then you may need `git fetch --all` python3 changelog.py --output=changelog-v22.4.1.2305-prestable.md --gh-user-or-token="$GITHUB_TOKEN" v21.6.2.7-prestable python3 changelog.py --output=changelog-v22.4.1.2305-prestable.md --gh-user-or-token="$USER" --gh-password="$PASSWORD" v21.6.2.7-prestable From 9d16205c8a532f5b2600fe9049c99372535a6dc9 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 20 Feb 2023 15:29:09 +0100 Subject: [PATCH 458/717] Load named collections on first access --- programs/local/LocalServer.cpp | 5 -- programs/server/Server.cpp | 4 -- .../NamedCollections/NamedCollectionUtils.cpp | 51 ++++++++++++++++++- .../NamedCollections/NamedCollectionUtils.h | 2 + src/Storages/NamedCollectionsHelpers.cpp | 2 + .../System/StorageSystemNamedCollections.cpp | 2 + 6 files changed, 55 insertions(+), 11 deletions(-) diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 133d629bbb1..23ba65fed44 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -37,7 +37,6 @@ #include #include #include -#include #include #include #include @@ -131,8 +130,6 @@ void LocalServer::initialize(Poco::Util::Application & self) config().getUInt("max_io_thread_pool_size", 100), config().getUInt("max_io_thread_pool_free_size", 0), config().getUInt("io_thread_pool_queue_size", 10000)); - - NamedCollectionUtils::loadFromConfig(config()); } @@ -224,8 +221,6 @@ void LocalServer::tryInitPath() global_context->setUserFilesPath(""); // user's files are everywhere - NamedCollectionUtils::loadFromSQL(global_context); - /// top_level_domains_lists const std::string & top_level_domains_path = config().getString("top_level_domains_path", path + "top_level_domains/"); if (!top_level_domains_path.empty()) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index b97b48d9c68..10710d61b84 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -770,8 +770,6 @@ try config().getUInt("max_io_thread_pool_free_size", 0), config().getUInt("io_thread_pool_queue_size", 10000)); - NamedCollectionUtils::loadFromConfig(config()); - /// Initialize global local cache for remote filesystem. if (config().has("local_cache_for_remote_fs")) { @@ -1177,8 +1175,6 @@ try SensitiveDataMasker::setInstance(std::make_unique(config(), "query_masking_rules")); } - NamedCollectionUtils::loadFromSQL(global_context); - auto main_config_reloader = std::make_unique( config_path, include_from_path, diff --git a/src/Common/NamedCollections/NamedCollectionUtils.cpp b/src/Common/NamedCollections/NamedCollectionUtils.cpp index 8beaa38210e..6ec09fb8a77 100644 --- a/src/Common/NamedCollections/NamedCollectionUtils.cpp +++ b/src/Common/NamedCollections/NamedCollectionUtils.cpp @@ -32,6 +32,9 @@ namespace ErrorCodes namespace NamedCollectionUtils { +static std::atomic is_loaded_from_config = false; +static std::atomic is_loaded_from_sql = false; + class LoadFromConfig { private: @@ -329,10 +332,21 @@ std::unique_lock lockNamedCollectionsTransaction() return std::unique_lock(transaction_lock); } +void loadFromConfigUnlocked(const Poco::Util::AbstractConfiguration & config, std::unique_lock &) +{ + auto named_collections = LoadFromConfig(config).getAll(); + LOG_TRACE( + &Poco::Logger::get("NamedCollectionsUtils"), + "Loaded {} collections from config", named_collections.size()); + + NamedCollectionFactory::instance().add(std::move(named_collections)); + is_loaded_from_config = true; +} + void loadFromConfig(const Poco::Util::AbstractConfiguration & config) { auto lock = lockNamedCollectionsTransaction(); - NamedCollectionFactory::instance().add(LoadFromConfig(config).getAll()); + loadFromConfigUnlocked(config, lock); } void reloadFromConfig(const Poco::Util::AbstractConfiguration & config) @@ -342,17 +356,47 @@ void reloadFromConfig(const Poco::Util::AbstractConfiguration & config) auto & instance = NamedCollectionFactory::instance(); instance.removeById(SourceId::CONFIG); instance.add(collections); + is_loaded_from_config = true; +} + +void loadFromSQLUnlocked(ContextPtr context, std::unique_lock &) +{ + auto named_collections = LoadFromSQL(context).getAll(); + LOG_TRACE( + &Poco::Logger::get("NamedCollectionsUtils"), + "Loaded {} collections from SQL", named_collections.size()); + + NamedCollectionFactory::instance().add(std::move(named_collections)); + is_loaded_from_sql = true; } void loadFromSQL(ContextPtr context) { auto lock = lockNamedCollectionsTransaction(); - NamedCollectionFactory::instance().add(LoadFromSQL(context).getAll()); + loadFromSQLUnlocked(context, lock); +} + +void loadIfNotUnlocked(std::unique_lock & lock) +{ + auto global_context = Context::getGlobalContextInstance(); + if (!is_loaded_from_config) + loadFromConfigUnlocked(global_context->getConfigRef(), lock); + if (!is_loaded_from_sql) + loadFromSQLUnlocked(global_context, lock); +} + +void loadIfNot() +{ + if (is_loaded_from_sql && is_loaded_from_config) + return; + auto lock = lockNamedCollectionsTransaction(); + return loadIfNotUnlocked(lock); } void removeFromSQL(const std::string & collection_name, ContextPtr context) { auto lock = lockNamedCollectionsTransaction(); + loadIfNotUnlocked(lock); LoadFromSQL(context).remove(collection_name); NamedCollectionFactory::instance().remove(collection_name); } @@ -360,6 +404,7 @@ void removeFromSQL(const std::string & collection_name, ContextPtr context) void removeIfExistsFromSQL(const std::string & collection_name, ContextPtr context) { auto lock = lockNamedCollectionsTransaction(); + loadIfNotUnlocked(lock); LoadFromSQL(context).removeIfExists(collection_name); NamedCollectionFactory::instance().removeIfExists(collection_name); } @@ -367,12 +412,14 @@ void removeIfExistsFromSQL(const std::string & collection_name, ContextPtr conte void createFromSQL(const ASTCreateNamedCollectionQuery & query, ContextPtr context) { auto lock = lockNamedCollectionsTransaction(); + loadIfNotUnlocked(lock); NamedCollectionFactory::instance().add(query.collection_name, LoadFromSQL(context).create(query)); } void updateFromSQL(const ASTAlterNamedCollectionQuery & query, ContextPtr context) { auto lock = lockNamedCollectionsTransaction(); + loadIfNotUnlocked(lock); LoadFromSQL(context).update(query); auto collection = NamedCollectionFactory::instance().getMutable(query.collection_name); diff --git a/src/Common/NamedCollections/NamedCollectionUtils.h b/src/Common/NamedCollections/NamedCollectionUtils.h index 8befc9cac3c..c929abb5d74 100644 --- a/src/Common/NamedCollections/NamedCollectionUtils.h +++ b/src/Common/NamedCollections/NamedCollectionUtils.h @@ -35,6 +35,8 @@ void createFromSQL(const ASTCreateNamedCollectionQuery & query, ContextPtr conte /// Update definition of already existing collection from AST and update result in `context->getPath() / named_collections /`. void updateFromSQL(const ASTAlterNamedCollectionQuery & query, ContextPtr context); +void loadIfNot(); + } } diff --git a/src/Storages/NamedCollectionsHelpers.cpp b/src/Storages/NamedCollectionsHelpers.cpp index cefed555781..6c783beaecb 100644 --- a/src/Storages/NamedCollectionsHelpers.cpp +++ b/src/Storages/NamedCollectionsHelpers.cpp @@ -58,6 +58,8 @@ NamedCollectionPtr tryGetNamedCollectionWithOverrides(ASTs asts) if (asts.empty()) return nullptr; + NamedCollectionUtils::loadIfNot(); + auto collection = tryGetNamedCollectionFromASTs(asts); if (!collection) return nullptr; diff --git a/src/Storages/System/StorageSystemNamedCollections.cpp b/src/Storages/System/StorageSystemNamedCollections.cpp index bc1e3a45e6b..aa095f48179 100644 --- a/src/Storages/System/StorageSystemNamedCollections.cpp +++ b/src/Storages/System/StorageSystemNamedCollections.cpp @@ -31,6 +31,8 @@ void StorageSystemNamedCollections::fillData(MutableColumns & res_columns, Conte { context->checkAccess(AccessType::SHOW_NAMED_COLLECTIONS); + NamedCollectionUtils::loadIfNot(); + auto collections = NamedCollectionFactory::instance().getAll(); for (const auto & [name, collection] : collections) { From 0d9a54144148d268e606b8aced431fdc25595814 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Mon, 20 Feb 2023 16:43:10 +0100 Subject: [PATCH 459/717] Update test.py --- tests/integration/test_storage_rabbitmq/test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index 030d9507d4f..fa3f7f6102d 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -1019,6 +1019,7 @@ def test_rabbitmq_many_inserts(rabbitmq_cluster): ), "ClickHouse lost some messages: {}".format(result) +@pytest.mark.skip(reason="Flaky") def test_rabbitmq_overloaded_insert(rabbitmq_cluster): instance.query( """ From 2273acaa30548da72c31f39bb5f8adde06ce4a95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 20 Feb 2023 16:47:25 +0100 Subject: [PATCH 460/717] Fix incorrect alias recursion in QueryNormalizer --- src/Interpreters/QueryNormalizer.cpp | 4 +++ ...2667_order_by_aggregation_result.reference | 4 +++ .../02667_order_by_aggregation_result.sql | 36 +++++++++++++++++++ 3 files changed, 44 insertions(+) create mode 100644 tests/queries/0_stateless/02667_order_by_aggregation_result.reference create mode 100644 tests/queries/0_stateless/02667_order_by_aggregation_result.sql diff --git a/src/Interpreters/QueryNormalizer.cpp b/src/Interpreters/QueryNormalizer.cpp index 4db61501d3d..df454aecac1 100644 --- a/src/Interpreters/QueryNormalizer.cpp +++ b/src/Interpreters/QueryNormalizer.cpp @@ -118,6 +118,8 @@ void QueryNormalizer::visit(ASTIdentifier & node, ASTPtr & ast, Data & data) alias_node->checkSize(data.settings.max_expanded_ast_elements); ast = alias_node->clone(); ast->setAlias(node_alias); + if (data.finished_asts.contains(alias_node)) + data.finished_asts[ast] = ast; } } else @@ -127,6 +129,8 @@ void QueryNormalizer::visit(ASTIdentifier & node, ASTPtr & ast, Data & data) auto alias_name = ast->getAliasOrColumnName(); ast = alias_node->clone(); ast->setAlias(alias_name); + if (data.finished_asts.contains(alias_node)) + data.finished_asts[ast] = ast; } } } diff --git a/tests/queries/0_stateless/02667_order_by_aggregation_result.reference b/tests/queries/0_stateless/02667_order_by_aggregation_result.reference new file mode 100644 index 00000000000..642fc2ed645 --- /dev/null +++ b/tests/queries/0_stateless/02667_order_by_aggregation_result.reference @@ -0,0 +1,4 @@ +0 0 +0 1 █████████████████████████████████████████████████▉ +1 2 0.5 +1 0.1 1.1 diff --git a/tests/queries/0_stateless/02667_order_by_aggregation_result.sql b/tests/queries/0_stateless/02667_order_by_aggregation_result.sql new file mode 100644 index 00000000000..277430888d7 --- /dev/null +++ b/tests/queries/0_stateless/02667_order_by_aggregation_result.sql @@ -0,0 +1,36 @@ +-- Github issues: +-- - https://github.com/ClickHouse/ClickHouse/issues/46268 +-- - https://github.com/ClickHouse/ClickHouse/issues/46273 + +-- Queries that the original PR (https://github.com/ClickHouse/ClickHouse/pull/42827) tried to fix +SELECT (number = 1) AND (number = 2) AS value, sum(value) OVER () FROM numbers(1) WHERE 1; +SELECT time, round(exp_smooth, 10), bar(exp_smooth, -9223372036854775807, 1048575, 50) AS bar FROM (SELECT 2 OR (number = 0) OR (number >= 1) AS value, number AS time, exponentialTimeDecayedSum(2147483646)(value, time) OVER (RANGE BETWEEN CURRENT ROW AND CURRENT ROW) AS exp_smooth FROM numbers(1) WHERE 10) WHERE 25; + +CREATE TABLE ttttttt +( + `timestamp` DateTime, + `col1` Float64, + `col2` Float64, + `col3` Float64 +) +ENGINE = MergeTree() +ORDER BY tuple(); + +INSERT INTO ttttttt VALUES ('2023-02-20 00:00:00', 1, 2, 3); + +-- Query that https://github.com/ClickHouse/ClickHouse/pull/42827 broke +SELECT + argMax(col1, timestamp) AS col1, + argMax(col2, timestamp) AS col2, + col1 / col2 AS final_col +FROM ttttttt +GROUP BY + col3 +ORDER BY final_col DESC; + +SELECT + argMax(col1, timestamp) AS col1, + col1 / 10 AS final_col, + final_col + 1 AS final_col2 +FROM ttttttt +GROUP BY col3; From 0a3639884746a6bb1c057bb5480369d026a96854 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 20 Feb 2023 16:53:06 +0100 Subject: [PATCH 461/717] Update test --- tests/integration/test_storage_rabbitmq/test.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index 5ca6f2acedf..c3e1843a417 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -1033,8 +1033,7 @@ def test_rabbitmq_overloaded_insert(rabbitmq_cluster): rabbitmq_exchange_type = 'direct', rabbitmq_num_consumers = 2, rabbitmq_flush_interval_ms=1000, - rabbitmq_max_block_size = 1000, - rabbitmq_num_queues = 2, + rabbitmq_max_block_size = 100, rabbitmq_routing_key_list = 'over', rabbitmq_format = 'TSV', rabbitmq_row_delimiter = '\\n'; @@ -1044,8 +1043,6 @@ def test_rabbitmq_overloaded_insert(rabbitmq_cluster): rabbitmq_exchange_name = 'over', rabbitmq_exchange_type = 'direct', rabbitmq_routing_key_list = 'over', - rabbitmq_flush_interval_ms=1000, - rabbitmq_max_block_size = 1000, rabbitmq_format = 'TSV', rabbitmq_row_delimiter = '\\n'; CREATE TABLE test.view_overload (key UInt64, value UInt64) From c84a64d22fe287e0b6a5caa4624791fb0dd9d4b8 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Mon, 20 Feb 2023 17:15:03 +0100 Subject: [PATCH 462/717] Return chunks with 0 rows from MergeTreeSource to report progress when rows are filtered in PREWHERE --- .../MergeTree/MergeTreeBaseSelectProcessor.cpp | 8 ++++---- .../MergeTree/MergeTreeBaseSelectProcessor.h | 3 +++ src/Storages/MergeTree/MergeTreeSource.cpp | 13 +++++++------ src/Storages/MergeTree/MergeTreeSource.h | 2 +- 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp index 49458be4232..531f48377a9 100644 --- a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp @@ -170,16 +170,16 @@ ChunkAndProgress IMergeTreeSelectAlgorithm::read() return ChunkAndProgress{ .chunk = Chunk(ordered_columns, res.row_count), .num_read_rows = res.num_read_rows, - .num_read_bytes = res.num_read_bytes}; + .num_read_bytes = res.num_read_bytes, + .is_finished = false}; } else { - num_read_rows += res.num_read_rows; - num_read_bytes += res.num_read_bytes; + return {Chunk(), res.num_read_rows, res.num_read_bytes, false}; } } - return {Chunk(), num_read_rows, num_read_bytes}; + return {Chunk(), num_read_rows, num_read_bytes, true}; } void IMergeTreeSelectAlgorithm::initializeMergeTreeReadersForCurrentTask( diff --git a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.h b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.h index c6680676ce9..22c15635529 100644 --- a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.h +++ b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.h @@ -20,6 +20,9 @@ struct ChunkAndProgress Chunk chunk; size_t num_read_rows = 0; size_t num_read_bytes = 0; + /// Explicitly indicate that we have read all data. + /// This is needed to occasionally return empty chunk to indicate the progress while the rows are filtered out in PREWHERE. + bool is_finished = false; }; struct ParallelReadingExtension diff --git a/src/Storages/MergeTree/MergeTreeSource.cpp b/src/Storages/MergeTree/MergeTreeSource.cpp index ae1679cebfd..a37d1d3ec2c 100644 --- a/src/Storages/MergeTree/MergeTreeSource.cpp +++ b/src/Storages/MergeTree/MergeTreeSource.cpp @@ -176,15 +176,16 @@ ISource::Status MergeTreeSource::prepare() } -std::optional MergeTreeSource::reportProgress(ChunkAndProgress chunk) +Chunk MergeTreeSource::processReadResult(ChunkAndProgress chunk) { if (chunk.num_read_rows || chunk.num_read_bytes) progress(chunk.num_read_rows, chunk.num_read_bytes); - if (chunk.chunk.hasRows()) - return std::move(chunk.chunk); + finished = chunk.is_finished; - return {}; + /// We can return a chunk with no rows even if are not finished. + /// This allows to report progress when all the rows are filtered out inside MergeTreeBaseSelectProcessor by PREWHERE logic. + return std::move(chunk.chunk); } @@ -194,7 +195,7 @@ std::optional MergeTreeSource::tryGenerate() if (async_reading_state) { if (async_reading_state->getStage() == AsyncReadingState::Stage::IsFinished) - return reportProgress(async_reading_state->getResult()); + return processReadResult(async_reading_state->getResult()); chassert(async_reading_state->getStage() == AsyncReadingState::Stage::NotStarted); @@ -220,7 +221,7 @@ std::optional MergeTreeSource::tryGenerate() } #endif - return reportProgress(algorithm->read()); + return processReadResult(algorithm->read()); } #if defined(OS_LINUX) diff --git a/src/Storages/MergeTree/MergeTreeSource.h b/src/Storages/MergeTree/MergeTreeSource.h index bba0c0af80e..463faad0fab 100644 --- a/src/Storages/MergeTree/MergeTreeSource.h +++ b/src/Storages/MergeTree/MergeTreeSource.h @@ -36,7 +36,7 @@ private: std::unique_ptr async_reading_state; #endif - std::optional reportProgress(ChunkAndProgress chunk); + Chunk processReadResult(ChunkAndProgress chunk); }; } From 6ed71120fbb6d0b214e96ca75b759427774d1d1f Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Mon, 20 Feb 2023 14:46:45 -0300 Subject: [PATCH 463/717] possibly fix explain syntax test --- tests/queries/0_stateless/02420_final_setting.reference | 8 ++++---- .../0_stateless/02420_final_setting_analyzer.reference | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/queries/0_stateless/02420_final_setting.reference b/tests/queries/0_stateless/02420_final_setting.reference index 5b9780e27cd..42acc78c57e 100644 --- a/tests/queries/0_stateless/02420_final_setting.reference +++ b/tests/queries/0_stateless/02420_final_setting.reference @@ -82,16 +82,16 @@ FROM ALL INNER JOIN ( SELECT - val_middle, - id + id, + val_middle FROM middle_table ) AS middle_table ON `--left_table.id` = `--middle_table.id` ) AS `--.s` ALL INNER JOIN ( SELECT - val_right, - id + id, + val_right FROM right_table FINAL ) AS right_table ON `--middle_table.id` = id diff --git a/tests/queries/0_stateless/02420_final_setting_analyzer.reference b/tests/queries/0_stateless/02420_final_setting_analyzer.reference index 2f8bc48fc65..ee7c2541bcf 100644 --- a/tests/queries/0_stateless/02420_final_setting_analyzer.reference +++ b/tests/queries/0_stateless/02420_final_setting_analyzer.reference @@ -83,16 +83,16 @@ FROM ALL INNER JOIN ( SELECT - val_middle, - id + id, + val_middle FROM middle_table ) AS middle_table ON `--left_table.id` = `--middle_table.id` ) AS `--.s` ALL INNER JOIN ( SELECT - val_right, - id + id, + val_right FROM right_table FINAL ) AS right_table ON `--middle_table.id` = id From 87a4fb025275d1837868952f7f47e0305023b230 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 20 Feb 2023 19:08:54 +0100 Subject: [PATCH 464/717] Update aliases when clone happens --- src/Interpreters/QueryNormalizer.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/Interpreters/QueryNormalizer.cpp b/src/Interpreters/QueryNormalizer.cpp index df454aecac1..56b81b3d224 100644 --- a/src/Interpreters/QueryNormalizer.cpp +++ b/src/Interpreters/QueryNormalizer.cpp @@ -118,8 +118,15 @@ void QueryNormalizer::visit(ASTIdentifier & node, ASTPtr & ast, Data & data) alias_node->checkSize(data.settings.max_expanded_ast_elements); ast = alias_node->clone(); ast->setAlias(node_alias); + + /// If the cloned AST was finished, this one should also be considered finished if (data.finished_asts.contains(alias_node)) data.finished_asts[ast] = ast; + + /// If we had an alias for node_alias, point it instead to the new node so we don't have to revisit it + /// on subsequent calls + if (auto existing_alias = data.aliases.find(node_alias); existing_alias != data.aliases.end()) + existing_alias->second = ast; } } else @@ -129,8 +136,15 @@ void QueryNormalizer::visit(ASTIdentifier & node, ASTPtr & ast, Data & data) auto alias_name = ast->getAliasOrColumnName(); ast = alias_node->clone(); ast->setAlias(alias_name); + + /// If the cloned AST was finished, this one should also be considered finished if (data.finished_asts.contains(alias_node)) data.finished_asts[ast] = ast; + + /// If we had an alias for node_alias, point it instead to the new node so we don't have to revisit it + /// on subsequent calls + if (auto existing_alias = data.aliases.find(node_alias); existing_alias != data.aliases.end()) + existing_alias->second = ast; } } } From 3c4c527bce41b7418de94f5c730478c1ed715938 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Mon, 20 Feb 2023 18:40:57 +0000 Subject: [PATCH 465/717] Fix MemoryTracker counters for async inserts --- src/Common/CurrentThread.cpp | 22 +++++++++++ src/Common/CurrentThread.h | 6 +++ src/Interpreters/AsynchronousInsertQueue.cpp | 6 ++- src/Interpreters/AsynchronousInsertQueue.h | 39 +++++++++++++++++++- 4 files changed, 70 insertions(+), 3 deletions(-) diff --git a/src/Common/CurrentThread.cpp b/src/Common/CurrentThread.cpp index e54b2c8abe4..95b316671a4 100644 --- a/src/Common/CurrentThread.cpp +++ b/src/Common/CurrentThread.cpp @@ -110,4 +110,26 @@ ThreadGroupStatusPtr CurrentThread::getGroup() return current_thread->getThreadGroup(); } +MemoryTracker * CurrentThread::getUserMemoryTracker() +{ + if (unlikely(!current_thread)) + return nullptr; + + if (auto group = current_thread->getThreadGroup()) + return group->memory_tracker.getParent(); + + return nullptr; +} + +void CurrentThread::flushUntrackedMemory() +{ + if (unlikely(!current_thread)) + return; + if (current_thread->untracked_memory == 0) + return; + + current_thread->memory_tracker.adjustWithUntrackedMemory(current_thread->untracked_memory); + current_thread->untracked_memory = 0; +} + } diff --git a/src/Common/CurrentThread.h b/src/Common/CurrentThread.h index cbe60365798..382ae5f6e77 100644 --- a/src/Common/CurrentThread.h +++ b/src/Common/CurrentThread.h @@ -40,6 +40,12 @@ public: /// Group to which belongs current thread static ThreadGroupStatusPtr getGroup(); + /// MemoryTracker for user that owns current thread if any + static MemoryTracker * getUserMemoryTracker(); + + /// Adjust counters in MemoryTracker hierarchy if untracked_memory is not 0. + static void flushUntrackedMemory(); + /// A logs queue used by TCPHandler to pass logs to a client static void attachInternalTextLogsQueue(const std::shared_ptr & logs_queue, LogsLevel client_logs_level); diff --git a/src/Interpreters/AsynchronousInsertQueue.cpp b/src/Interpreters/AsynchronousInsertQueue.cpp index fa3e9915e8f..bfd9988fd10 100644 --- a/src/Interpreters/AsynchronousInsertQueue.cpp +++ b/src/Interpreters/AsynchronousInsertQueue.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -102,9 +103,10 @@ bool AsynchronousInsertQueue::InsertQuery::operator==(const InsertQuery & other) return query_str == other.query_str && settings == other.settings; } -AsynchronousInsertQueue::InsertData::Entry::Entry(String && bytes_, String && query_id_) +AsynchronousInsertQueue::InsertData::Entry::Entry(String && bytes_, String && query_id_, MemoryTracker * user_memory_tracker_) : bytes(std::move(bytes_)) , query_id(std::move(query_id_)) + , user_memory_tracker(user_memory_tracker_) , create_time(std::chrono::system_clock::now()) { } @@ -209,7 +211,7 @@ std::future AsynchronousInsertQueue::push(ASTPtr query, ContextPtr query_c if (auto quota = query_context->getQuota()) quota->used(QuotaType::WRITTEN_BYTES, bytes.size()); - auto entry = std::make_shared(std::move(bytes), query_context->getCurrentQueryId()); + auto entry = std::make_shared(std::move(bytes), query_context->getCurrentQueryId(), CurrentThread::getUserMemoryTracker()); InsertQuery key{query, settings}; InsertDataPtr data_to_process; diff --git a/src/Interpreters/AsynchronousInsertQueue.h b/src/Interpreters/AsynchronousInsertQueue.h index ee1265673a6..b8c7d9d285b 100644 --- a/src/Interpreters/AsynchronousInsertQueue.h +++ b/src/Interpreters/AsynchronousInsertQueue.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -41,6 +42,31 @@ private: UInt128 calculateHash() const; }; + struct UserMemoryTrackerSwitcher + { + explicit UserMemoryTrackerSwitcher(MemoryTracker * new_tracker) + { + auto * thread_tracker = CurrentThread::getMemoryTracker(); + prev_untracked_memory = current_thread->untracked_memory; + prev_memory_tracker_parent = thread_tracker->getParent(); + + current_thread->untracked_memory = 0; + thread_tracker->setParent(new_tracker); + } + + ~UserMemoryTrackerSwitcher() + { + CurrentThread::flushUntrackedMemory(); + auto * thread_tracker = CurrentThread::getMemoryTracker(); + + current_thread->untracked_memory = prev_untracked_memory; + thread_tracker->setParent(prev_memory_tracker_parent); + } + + MemoryTracker * prev_memory_tracker_parent; + Int64 prev_untracked_memory; + }; + struct InsertData { struct Entry @@ -48,9 +74,10 @@ private: public: const String bytes; const String query_id; + MemoryTracker * const user_memory_tracker; const std::chrono::time_point create_time; - Entry(String && bytes_, String && query_id_); + Entry(String && bytes_, String && query_id_, MemoryTracker * user_memory_tracker_); void finish(std::exception_ptr exception_ = nullptr); std::future getFuture() { return promise.get_future(); } @@ -61,6 +88,16 @@ private: std::atomic_bool finished = false; }; + ~InsertData() + { + auto it = entries.begin(); + while (it != entries.end()) + { + UserMemoryTrackerSwitcher switcher((*it)->user_memory_tracker); + it = entries.erase(it); + } + } + using EntryPtr = std::shared_ptr; std::list entries; From 186a29a2aad4e05be56c4462e55ee3eb5a0ec360 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 20 Feb 2023 19:46:39 +0100 Subject: [PATCH 466/717] Resurrect processors_profile_log docs. --- docs/en/operations/settings/settings.md | 9 +++ .../system-tables/processors_profile_log.md | 74 +++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 docs/en/operations/system-tables/processors_profile_log.md diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 1d1d8b32d1d..9b16188ad6d 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -851,6 +851,15 @@ Result: └─────────────┴───────────┘ ``` +## log_processors_profiles {#settings-log_processors_profiles} + +Write time that processor spent during execution/waiting for data to `system.processors_profile_log` table. + +See also: + +- [`system.processors_profile_log`](../../operations/system-tables/processors_profile_log.md#system-processors_profile_log) +- [`EXPLAIN PIPELINE`](../../sql-reference/statements/explain.md#explain-pipeline) + ## max_insert_block_size {#settings-max_insert_block_size} The size of blocks (in a count of rows) to form for insertion into a table. diff --git a/docs/en/operations/system-tables/processors_profile_log.md b/docs/en/operations/system-tables/processors_profile_log.md new file mode 100644 index 00000000000..269385deab6 --- /dev/null +++ b/docs/en/operations/system-tables/processors_profile_log.md @@ -0,0 +1,74 @@ +# system.processors_profile_log {#system-processors_profile_log} + +This table contains profiling on processors level (that you can find in [`EXPLAIN PIPELINE`](../../sql-reference/statements/explain.md#explain-pipeline)). + +Columns: + +- `event_date` ([Date](../../sql-reference/data-types/date.md)) — The date when the event happened. +- `event_time` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — The date and time when the event happened. +- `id` ([UInt64](../../sql-reference/data-types/int-uint.md)) — ID of processor +- `parent_ids` ([Array(UInt64)](../../sql-reference/data-types/array.md)) — Parent processors IDs +- `query_id` ([String](../../sql-reference/data-types/string.md)) — ID of the query +- `name` ([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md)) — Name of the processor. +- `elapsed_us` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Number of microseconds this processor was executed. +- `input_wait_elapsed_us` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Number of microseconds this processor was waiting for data (from other processor). +- `output_wait_elapsed_us` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Number of microseconds this processor was waiting because output port was full. +- `plan_step` ([UInt64](../../sql-reference/data-types/int-uint.md)) — ID of the query plan step which created this processor. The value is zero if the processor was not added from any step. +- `plan_group` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Group of the processor if it was created by query plan step. A group is a logical partitioning of processors added from the same query plan step. Group is used only for beautifying the result of EXPLAIN PIPELINE result. +- `input_rows` ([UInt64](../../sql-reference/data-types/int-uint.md)) — The number of rows consumed by processor. +- `input_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) — The number of bytes consumed by processor. +- `output_rows` ([UInt64](../../sql-reference/data-types/int-uint.md)) — The number of rows generated by processor. +- `output_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) — The number of bytes generated by processor. +**Example** + +Query: + +``` sql +EXPLAIN PIPELINE +SELECT sleep(1) +┌─explain─────────────────────────┐ +│ (Expression) │ +│ ExpressionTransform │ +│ (SettingQuotaAndLimits) │ +│ (ReadFromStorage) │ +│ SourceFromSingleChunk 0 → 1 │ +└─────────────────────────────────┘ +SELECT sleep(1) +SETTINGS log_processors_profiles = 1 +Query id: feb5ed16-1c24-4227-aa54-78c02b3b27d4 +┌─sleep(1)─┐ +│ 0 │ +└──────────┘ +1 rows in set. Elapsed: 1.018 sec. +SELECT + name, + elapsed_us, + input_wait_elapsed_us, + output_wait_elapsed_us +FROM system.processors_profile_log +WHERE query_id = 'feb5ed16-1c24-4227-aa54-78c02b3b27d4' +ORDER BY name ASC +``` + +Result: + +``` text +┌─name────────────────────┬─elapsed_us─┬─input_wait_elapsed_us─┬─output_wait_elapsed_us─┐ +│ ExpressionTransform │ 1000497 │ 2823 │ 197 │ +│ LazyOutputFormat │ 36 │ 1002188 │ 0 │ +│ LimitsCheckingTransform │ 10 │ 1002994 │ 106 │ +│ NullSource │ 5 │ 1002074 │ 0 │ +│ NullSource │ 1 │ 1002084 │ 0 │ +│ SourceFromSingleChunk │ 45 │ 4736 │ 1000819 │ +└─────────────────────────┴────────────┴───────────────────────┴────────────────────────┘ +``` + +Here you can see: + +- `ExpressionTransform` was executing `sleep(1)` function, so it `work` will takes 1e6, and so `elapsed_us` > 1e6. +- `SourceFromSingleChunk` need to wait, because `ExpressionTransform` does not accept any data during execution of `sleep(1)`, so it will be in `PortFull` state for 1e6 us, and so `output_wait_elapsed_us` > 1e6. +- `LimitsCheckingTransform`/`NullSource`/`LazyOutputFormat` need to wait until `ExpressionTransform` will execute `sleep(1)` to process the result, so `input_wait_elapsed_us` > 1e6. + +**See Also** + +- [`EXPLAIN PIPELINE`](../../sql-reference/statements/explain.md#explain-pipeline) \ No newline at end of file From 7f487f03e75a769722c6c1115f8120e1176c4033 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 20 Feb 2023 19:24:15 +0000 Subject: [PATCH 467/717] Automatic style fix --- tests/ci/workflow_approve_rerun_lambda/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/workflow_approve_rerun_lambda/app.py b/tests/ci/workflow_approve_rerun_lambda/app.py index b563a9786c4..fb14dfd2258 100644 --- a/tests/ci/workflow_approve_rerun_lambda/app.py +++ b/tests/ci/workflow_approve_rerun_lambda/app.py @@ -123,7 +123,7 @@ TRUSTED_CONTRIBUTORS = { "BoloniniD", # Seasoned contributor, HSE "tonickkozlov", # Cloudflare "tylerhannan", # ClickHouse Employee - "myrrc", # Mike Kot, DoubleCloud + "myrrc", # Mike Kot, DoubleCloud ] } From a568704d63a5402de9af2852a06f5419afc3da38 Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 20 Feb 2023 20:43:28 +0000 Subject: [PATCH 468/717] Fix avro --- src/Processors/Formats/Impl/AvroRowInputFormat.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp index bedf3cc932e..c3ea1b5e23b 100644 --- a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp @@ -778,15 +778,14 @@ AvroDeserializer::AvroDeserializer(const Block & header, avro::ValidSchema schem void AvroDeserializer::deserializeRow(MutableColumns & columns, avro::Decoder & decoder, RowReadExtension & ext) const { + size_t row = columns[0]->size() + 1; ext.read_columns.assign(columns.size(), false); row_action.execute(columns, decoder, ext); for (size_t i = 0; i < ext.read_columns.size(); ++i) { /// Insert default in missing columns. - if (!column_found[i]) - { + if (columns[i]->size() != row) columns[i]->insertDefault(); - } } } From b1e0e587e61dcf38c0647dfb196fcb206b380507 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 20 Feb 2023 23:43:11 +0100 Subject: [PATCH 469/717] Inhibit index_granularity_bytes randomization in some tests --- .../00755_avg_value_size_hint_passing.sql | 2 +- .../00804_test_alter_compression_codecs.sql | 2 +- .../00804_test_custom_compression_codecs.sql | 2 +- .../queries/0_stateless/00837_minmax_index.sh | 4 ++-- ...minmax_index_replicated_zookeeper_long.sql | 4 ++-- .../queries/0_stateless/00838_unique_index.sh | 2 +- .../0_stateless/00907_set_index_max_rows.sh | 4 ++-- .../0_stateless/00908_bloom_filter_index.sh | 6 +++--- ...tom_compression_codecs_replicated_long.sql | 2 +- .../queries/0_stateless/00942_mutate_index.sh | 2 +- .../0_stateless/00943_materialize_index.sh | 2 +- .../00944_clear_index_in_partition.sh | 2 +- .../0_stateless/00945_bloom_filter_index.sql | 18 ++++++++--------- ...hecksums_in_system_parts_columns_table.sql | 3 +-- .../00964_bloom_index_string_functions.sh | 2 +- .../00965_set_index_string_functions.sh | 2 +- ...ices_mutation_replicated_zookeeper_long.sh | 4 ++-- .../0_stateless/00980_alter_settings_race.sh | 2 +- .../00980_merge_alter_settings.reference | 20 +++++++++---------- .../00980_merge_alter_settings.sql | 4 ++-- .../00990_hasToken_and_tokenbf.sql | 4 ++-- .../01030_final_mark_empty_primary_key.sql | 2 +- .../01055_minmax_index_compact_parts.sh | 4 ++-- ...tal_streaming_from_2_src_with_feedback.sql | 2 +- ..._materialize_clear_index_compact_parts.sql | 2 +- .../01201_read_single_thread_in_order.sql | 2 +- ...oom_filter_index_string_multi_granulas.sql | 2 +- ...with_constant_string_in_index_analysis.sql | 2 +- ...515_mv_and_array_join_optimisation_bag.sql | 2 +- .../01605_skip_idx_compact_parts.sql | 2 +- .../01681_bloom_filter_nullable_column.sql | 2 +- .../0_stateless/01780_column_sparse_full.sql | 2 +- .../01781_token_extractor_buffer_overflow.sql | 2 +- .../0_stateless/01926_order_by_desc_limit.sql | 2 +- .../0_stateless/02030_tuple_filter.sql | 2 +- .../02149_read_in_order_fixed_prefix.sql | 2 +- .../02155_read_in_order_max_rows_to_read.sql | 2 +- .../02267_empty_arrays_read_reverse.sql | 2 +- .../0_stateless/02346_full_text_search.sql | 13 ++++++------ tests/queries/0_stateless/02354_annoy.sh | 20 +++++++++---------- .../0_stateless/02374_in_tuple_index.sql | 2 +- .../02465_limit_trivial_max_rows_to_read.sql | 2 +- .../02467_set_with_lowcardinality_type.sql | 4 ++-- ...default_value_used_in_row_level_filter.sql | 2 +- ...ewhere_filtered_rows_div_by_zero.reference | 2 +- ...481_prewhere_filtered_rows_div_by_zero.sql | 2 +- 46 files changed, 87 insertions(+), 89 deletions(-) diff --git a/tests/queries/0_stateless/00755_avg_value_size_hint_passing.sql b/tests/queries/0_stateless/00755_avg_value_size_hint_passing.sql index 8ce6716e03c..1d033b7aa47 100644 --- a/tests/queries/0_stateless/00755_avg_value_size_hint_passing.sql +++ b/tests/queries/0_stateless/00755_avg_value_size_hint_passing.sql @@ -1,5 +1,5 @@ DROP TABLE IF EXISTS size_hint; -CREATE TABLE size_hint (s Array(String)) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 1000; +CREATE TABLE size_hint (s Array(String)) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 1000, index_granularity_bytes = '10Mi'; SET max_block_size = 1000; SET max_memory_usage = 1000000000; diff --git a/tests/queries/0_stateless/00804_test_alter_compression_codecs.sql b/tests/queries/0_stateless/00804_test_alter_compression_codecs.sql index 9e911262ed7..1c83d2ae36d 100644 --- a/tests/queries/0_stateless/00804_test_alter_compression_codecs.sql +++ b/tests/queries/0_stateless/00804_test_alter_compression_codecs.sql @@ -65,7 +65,7 @@ CREATE TABLE large_alter_table_00804 ( somedate Date CODEC(ZSTD, ZSTD, ZSTD(12), LZ4HC(12)), id UInt64 CODEC(LZ4, ZSTD, NONE, LZ4HC), data String CODEC(ZSTD(2), LZ4HC, NONE, LZ4, LZ4) -) ENGINE = MergeTree() PARTITION BY somedate ORDER BY id SETTINGS index_granularity = 2, min_bytes_for_wide_part = 0; +) ENGINE = MergeTree() PARTITION BY somedate ORDER BY id SETTINGS index_granularity = 2, index_granularity_bytes = '10Mi', min_bytes_for_wide_part = 0; INSERT INTO large_alter_table_00804 SELECT toDate('2019-01-01'), number, toString(number + rand()) FROM system.numbers LIMIT 300000; diff --git a/tests/queries/0_stateless/00804_test_custom_compression_codecs.sql b/tests/queries/0_stateless/00804_test_custom_compression_codecs.sql index c3573588ae9..593d349caf9 100644 --- a/tests/queries/0_stateless/00804_test_custom_compression_codecs.sql +++ b/tests/queries/0_stateless/00804_test_custom_compression_codecs.sql @@ -114,7 +114,7 @@ CREATE TABLE compression_codec_multiple_with_key ( somedate Date CODEC(ZSTD, ZSTD, ZSTD(12), LZ4HC(12), Delta, Delta), id UInt64 CODEC(LZ4, ZSTD, Delta, NONE, LZ4HC, Delta), data String CODEC(ZSTD(2), Delta(1), LZ4HC, NONE, LZ4, LZ4) -) ENGINE = MergeTree() PARTITION BY somedate ORDER BY id SETTINGS index_granularity = 2; +) ENGINE = MergeTree() PARTITION BY somedate ORDER BY id SETTINGS index_granularity = 2, index_granularity_bytes = '10Mi'; INSERT INTO compression_codec_multiple_with_key VALUES(toDate('2018-10-12'), 100000, 'hello'), (toDate('2018-10-12'), 100002, 'world'), (toDate('2018-10-12'), 1111, '!'); diff --git a/tests/queries/0_stateless/00837_minmax_index.sh b/tests/queries/0_stateless/00837_minmax_index.sh index 39e39f9d628..e4de0b9ebfc 100755 --- a/tests/queries/0_stateless/00837_minmax_index.sh +++ b/tests/queries/0_stateless/00837_minmax_index.sh @@ -23,7 +23,7 @@ CREATE TABLE minmax_idx INDEX idx_2 (u64 + toYear(dt), substring(s, 2, 4)) TYPE minmax GRANULARITY 3 ) ENGINE = MergeTree() ORDER BY u64 -SETTINGS index_granularity = 2;" +SETTINGS index_granularity = 2, index_granularity_bytes = '10Mi';" $CLICKHOUSE_CLIENT --query="INSERT INTO minmax_idx VALUES @@ -48,4 +48,4 @@ $CLICKHOUSE_CLIENT --query="SELECT * FROM minmax_idx WHERE i32 = 5 AND i32 + f64 $CLICKHOUSE_CLIENT --query="SELECT * FROM minmax_idx WHERE (u64 < 2 OR u64 > 10) AND e != 'b' ORDER BY dt" $CLICKHOUSE_CLIENT --query="SELECT * FROM minmax_idx WHERE (u64 < 2 OR u64 > 10) AND e != 'b' ORDER BY dt FORMAT JSON" | grep "rows_read" -$CLICKHOUSE_CLIENT --query="DROP TABLE minmax_idx" \ No newline at end of file +$CLICKHOUSE_CLIENT --query="DROP TABLE minmax_idx" diff --git a/tests/queries/0_stateless/00837_minmax_index_replicated_zookeeper_long.sql b/tests/queries/0_stateless/00837_minmax_index_replicated_zookeeper_long.sql index a2fe1ebd63a..ba080949548 100644 --- a/tests/queries/0_stateless/00837_minmax_index_replicated_zookeeper_long.sql +++ b/tests/queries/0_stateless/00837_minmax_index_replicated_zookeeper_long.sql @@ -19,7 +19,7 @@ CREATE TABLE minmax_idx1 idx_2 (u64 + toYear(dt), substring(s, 2, 4)) TYPE minmax GRANULARITY 3 ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/test_00837/minmax', 'r1') ORDER BY u64 -SETTINGS index_granularity = 2; +SETTINGS index_granularity = 2, index_granularity_bytes = '10Mi'; CREATE TABLE minmax_idx2 ( @@ -36,7 +36,7 @@ CREATE TABLE minmax_idx2 idx_2 (u64 + toYear(dt), substring(s, 2, 4)) TYPE minmax GRANULARITY 3 ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/test_00837/minmax', 'r2') ORDER BY u64 -SETTINGS index_granularity = 2; +SETTINGS index_granularity = 2, index_granularity_bytes = '10Mi'; /* many small inserts => table will make merges */ diff --git a/tests/queries/0_stateless/00838_unique_index.sh b/tests/queries/0_stateless/00838_unique_index.sh index 36504b754a7..b267b6a8eb3 100755 --- a/tests/queries/0_stateless/00838_unique_index.sh +++ b/tests/queries/0_stateless/00838_unique_index.sh @@ -22,7 +22,7 @@ CREATE TABLE set_idx INDEX idx_2 (u64 + toYear(dt), substring(s, 2, 4)) TYPE set(6) GRANULARITY 3 ) ENGINE = MergeTree() ORDER BY u64 -SETTINGS index_granularity = 2;" +SETTINGS index_granularity = 2, index_granularity_bytes = '10Mi';" $CLICKHOUSE_CLIENT --query="INSERT INTO set_idx VALUES (0, 5, 4.7, 6.5, 'cba', 'b', '2014-01-04'), diff --git a/tests/queries/0_stateless/00907_set_index_max_rows.sh b/tests/queries/0_stateless/00907_set_index_max_rows.sh index f780517934d..3707aaf2ca6 100755 --- a/tests/queries/0_stateless/00907_set_index_max_rows.sh +++ b/tests/queries/0_stateless/00907_set_index_max_rows.sh @@ -14,7 +14,7 @@ CREATE TABLE set_idx INDEX idx (i32) TYPE set(2) GRANULARITY 1 ) ENGINE = MergeTree() ORDER BY u64 -SETTINGS index_granularity = 6;" +SETTINGS index_granularity = 6, index_granularity_bytes = '10Mi';" $CLICKHOUSE_CLIENT --query=" INSERT INTO set_idx @@ -24,4 +24,4 @@ SELECT number, number FROM system.numbers LIMIT 100" $CLICKHOUSE_CLIENT --query="SELECT * FROM set_idx WHERE i32 > 0 FORMAT JSON" | grep "rows_read" -$CLICKHOUSE_CLIENT --query="DROP TABLE set_idx;" \ No newline at end of file +$CLICKHOUSE_CLIENT --query="DROP TABLE set_idx;" diff --git a/tests/queries/0_stateless/00908_bloom_filter_index.sh b/tests/queries/0_stateless/00908_bloom_filter_index.sh index f1f07ca360b..92b5634c1db 100755 --- a/tests/queries/0_stateless/00908_bloom_filter_index.sh +++ b/tests/queries/0_stateless/00908_bloom_filter_index.sh @@ -20,7 +20,7 @@ CREATE TABLE bloom_filter_idx INDEX bf (s, lower(s)) TYPE ngrambf_v1(3, 512, 2, 0) GRANULARITY 1 ) ENGINE = MergeTree() ORDER BY k -SETTINGS index_granularity = 2;" +SETTINGS index_granularity = 2, index_granularity_bytes = '10Mi';" $CLICKHOUSE_CLIENT -n --query=" CREATE TABLE bloom_filter_idx2 @@ -30,7 +30,7 @@ CREATE TABLE bloom_filter_idx2 INDEX bf (s, lower(s)) TYPE ngrambf_v1(3, 512, 2, 0) GRANULARITY 1 ) ENGINE = MergeTree() ORDER BY k -SETTINGS index_granularity = 2;" +SETTINGS index_granularity = 2, index_granularity_bytes = '10Mi';" $CLICKHOUSE_CLIENT --query="INSERT INTO bloom_filter_idx VALUES @@ -113,7 +113,7 @@ CREATE TABLE bloom_filter_idx3 INDEX bf (s, lower(s)) TYPE tokenbf_v1(512, 3, 0) GRANULARITY 1 ) ENGINE = MergeTree() ORDER BY k -SETTINGS index_granularity = 2;" +SETTINGS index_granularity = 2, index_granularity_bytes = '10Mi';" $CLICKHOUSE_CLIENT --query="INSERT INTO bloom_filter_idx3 VALUES (0, 'ClickHouse is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP).'), diff --git a/tests/queries/0_stateless/00910_zookeeper_custom_compression_codecs_replicated_long.sql b/tests/queries/0_stateless/00910_zookeeper_custom_compression_codecs_replicated_long.sql index 369d897fafd..ed4d9dd4934 100644 --- a/tests/queries/0_stateless/00910_zookeeper_custom_compression_codecs_replicated_long.sql +++ b/tests/queries/0_stateless/00910_zookeeper_custom_compression_codecs_replicated_long.sql @@ -126,7 +126,7 @@ CREATE TABLE compression_codec_multiple_with_key_replicated ( somedate Date CODEC(ZSTD, ZSTD, ZSTD(12), LZ4HC(12), Delta, Delta), id UInt64 CODEC(LZ4, ZSTD, Delta, NONE, LZ4HC, Delta), data String CODEC(ZSTD(2), Delta(1), LZ4HC, NONE, LZ4, LZ4) -) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/test_00910/compression_codec_multiple_with_key_replicated', '1') PARTITION BY somedate ORDER BY id SETTINGS index_granularity = 2; +) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/test_00910/compression_codec_multiple_with_key_replicated', '1') PARTITION BY somedate ORDER BY id SETTINGS index_granularity = 2, index_granularity_bytes = '10Mi'; INSERT INTO compression_codec_multiple_with_key_replicated VALUES(toDate('2018-10-12'), 100000, 'hello'), (toDate('2018-10-12'), 100002, 'world'), (toDate('2018-10-12'), 1111, '!'); diff --git a/tests/queries/0_stateless/00942_mutate_index.sh b/tests/queries/0_stateless/00942_mutate_index.sh index 4eebdd1147f..6ebb30c25b9 100755 --- a/tests/queries/0_stateless/00942_mutate_index.sh +++ b/tests/queries/0_stateless/00942_mutate_index.sh @@ -15,7 +15,7 @@ CREATE TABLE minmax_idx INDEX idx (i64, u64 * i64) TYPE minmax GRANULARITY 1 ) ENGINE = MergeTree() ORDER BY u64 -SETTINGS index_granularity = 2;" +SETTINGS index_granularity = 2, index_granularity_bytes = '10Mi';" $CLICKHOUSE_CLIENT --query="INSERT INTO minmax_idx VALUES (0, 1, 1), diff --git a/tests/queries/0_stateless/00943_materialize_index.sh b/tests/queries/0_stateless/00943_materialize_index.sh index 43c9af84672..30ef46e5cb0 100755 --- a/tests/queries/0_stateless/00943_materialize_index.sh +++ b/tests/queries/0_stateless/00943_materialize_index.sh @@ -16,7 +16,7 @@ CREATE TABLE minmax_idx ) ENGINE = MergeTree() PARTITION BY i32 ORDER BY u64 -SETTINGS index_granularity = 2;" +SETTINGS index_granularity = 2, index_granularity_bytes = '10Mi';" $CLICKHOUSE_CLIENT --query="INSERT INTO minmax_idx VALUES diff --git a/tests/queries/0_stateless/00944_clear_index_in_partition.sh b/tests/queries/0_stateless/00944_clear_index_in_partition.sh index 1f349cf5946..8b74bd94f2c 100755 --- a/tests/queries/0_stateless/00944_clear_index_in_partition.sh +++ b/tests/queries/0_stateless/00944_clear_index_in_partition.sh @@ -17,7 +17,7 @@ CREATE TABLE minmax_idx ) ENGINE = MergeTree() PARTITION BY i32 ORDER BY u64 -SETTINGS index_granularity = 2;" +SETTINGS index_granularity = 2, index_granularity_bytes = '10Mi';" $CLICKHOUSE_CLIENT --query="INSERT INTO minmax_idx VALUES diff --git a/tests/queries/0_stateless/00945_bloom_filter_index.sql b/tests/queries/0_stateless/00945_bloom_filter_index.sql index fc18a4a4dc5..dc47e858c4d 100644 --- a/tests/queries/0_stateless/00945_bloom_filter_index.sql +++ b/tests/queries/0_stateless/00945_bloom_filter_index.sql @@ -1,7 +1,7 @@ DROP TABLE IF EXISTS single_column_bloom_filter; -CREATE TABLE single_column_bloom_filter (u64 UInt64, i32 Int32, i64 UInt64, INDEX idx (i32) TYPE bloom_filter GRANULARITY 1) ENGINE = MergeTree() ORDER BY u64 SETTINGS index_granularity = 6; +CREATE TABLE single_column_bloom_filter (u64 UInt64, i32 Int32, i64 UInt64, INDEX idx (i32) TYPE bloom_filter GRANULARITY 1) ENGINE = MergeTree() ORDER BY u64 SETTINGS index_granularity = 6, index_granularity_bytes = '10Mi'; INSERT INTO single_column_bloom_filter SELECT number AS u64, number AS i32, number AS i64 FROM system.numbers LIMIT 100; @@ -28,7 +28,7 @@ DROP TABLE IF EXISTS single_column_bloom_filter; DROP TABLE IF EXISTS bloom_filter_types_test; -CREATE TABLE bloom_filter_types_test (order_key UInt64, i8 Int8, i16 Int16, i32 Int32, i64 Int64, u8 UInt8, u16 UInt16, u32 UInt32, u64 UInt64, f32 Float32, f64 Float64, date Date, date_time DateTime('Asia/Istanbul'), str String, fixed_string FixedString(5), INDEX idx (i8, i16, i32, i64, u8, u16, u32, u64, f32, f64, date, date_time, str, fixed_string) TYPE bloom_filter GRANULARITY 1) ENGINE = MergeTree() ORDER BY order_key SETTINGS index_granularity = 6; +CREATE TABLE bloom_filter_types_test (order_key UInt64, i8 Int8, i16 Int16, i32 Int32, i64 Int64, u8 UInt8, u16 UInt16, u32 UInt32, u64 UInt64, f32 Float32, f64 Float64, date Date, date_time DateTime('Asia/Istanbul'), str String, fixed_string FixedString(5), INDEX idx (i8, i16, i32, i64, u8, u16, u32, u64, f32, f64, date, date_time, str, fixed_string) TYPE bloom_filter GRANULARITY 1) ENGINE = MergeTree() ORDER BY order_key SETTINGS index_granularity = 6, index_granularity_bytes = '10Mi'; INSERT INTO bloom_filter_types_test SELECT number AS order_key, toInt8(number) AS i8, toInt16(number) AS i16, toInt32(number) AS i32, toInt64(number) AS i64, toUInt8(number) AS u8, toUInt16(number) AS u16, toUInt32(number) AS u32, toUInt64(number) AS u64, toFloat32(number) AS f32, toFloat64(number) AS f64, toDate(number, 'Asia/Istanbul') AS date, toDateTime(number, 'Asia/Istanbul') AS date_time, toString(number) AS str, toFixedString(toString(number), 5) AS fixed_string FROM system.numbers LIMIT 100; SELECT COUNT() FROM bloom_filter_types_test WHERE i8 = 1 SETTINGS max_rows_to_read = 6; @@ -52,7 +52,7 @@ DROP TABLE IF EXISTS bloom_filter_types_test; DROP TABLE IF EXISTS bloom_filter_array_types_test; -CREATE TABLE bloom_filter_array_types_test (order_key Array(UInt64), i8 Array(Int8), i16 Array(Int16), i32 Array(Int32), i64 Array(Int64), u8 Array(UInt8), u16 Array(UInt16), u32 Array(UInt32), u64 Array(UInt64), f32 Array(Float32), f64 Array(Float64), date Array(Date), date_time Array(DateTime('Asia/Istanbul')), str Array(String), fixed_string Array(FixedString(5)), INDEX idx (i8, i16, i32, i64, u8, u16, u32, u64, f32, f64, date, date_time, str, fixed_string) TYPE bloom_filter GRANULARITY 1) ENGINE = MergeTree() ORDER BY order_key SETTINGS index_granularity = 6; +CREATE TABLE bloom_filter_array_types_test (order_key Array(UInt64), i8 Array(Int8), i16 Array(Int16), i32 Array(Int32), i64 Array(Int64), u8 Array(UInt8), u16 Array(UInt16), u32 Array(UInt32), u64 Array(UInt64), f32 Array(Float32), f64 Array(Float64), date Array(Date), date_time Array(DateTime('Asia/Istanbul')), str Array(String), fixed_string Array(FixedString(5)), INDEX idx (i8, i16, i32, i64, u8, u16, u32, u64, f32, f64, date, date_time, str, fixed_string) TYPE bloom_filter GRANULARITY 1) ENGINE = MergeTree() ORDER BY order_key SETTINGS index_granularity = 6, index_granularity_bytes = '10Mi'; INSERT INTO bloom_filter_array_types_test SELECT groupArray(number) AS order_key, groupArray(toInt8(number)) AS i8, groupArray(toInt16(number)) AS i16, groupArray(toInt32(number)) AS i32, groupArray(toInt64(number)) AS i64, groupArray(toUInt8(number)) AS u8, groupArray(toUInt16(number)) AS u16, groupArray(toUInt32(number)) AS u32, groupArray(toUInt64(number)) AS u64, groupArray(toFloat32(number)) AS f32, groupArray(toFloat64(number)) AS f64, groupArray(toDate(number, 'Asia/Istanbul')) AS date, groupArray(toDateTime(number, 'Asia/Istanbul')) AS date_time, groupArray(toString(number)) AS str, groupArray(toFixedString(toString(number), 5)) AS fixed_string FROM (SELECT number FROM system.numbers LIMIT 15); INSERT INTO bloom_filter_array_types_test SELECT groupArray(number) AS order_key, groupArray(toInt8(number)) AS i8, groupArray(toInt16(number)) AS i16, groupArray(toInt32(number)) AS i32, groupArray(toInt64(number)) AS i64, groupArray(toUInt8(number)) AS u8, groupArray(toUInt16(number)) AS u16, groupArray(toUInt32(number)) AS u32, groupArray(toUInt64(number)) AS u64, groupArray(toFloat32(number)) AS f32, groupArray(toFloat64(number)) AS f64, groupArray(toDate(number, 'Asia/Istanbul')) AS date, groupArray(toDateTime(number, 'Asia/Istanbul')) AS date_time, groupArray(toString(number)) AS str, groupArray(toFixedString(toString(number), 5)) AS fixed_string FROM (SELECT number FROM system.numbers WHERE number >= 5 LIMIT 15); INSERT INTO bloom_filter_array_types_test SELECT groupArray(number) AS order_key, groupArray(toInt8(number)) AS i8, groupArray(toInt16(number)) AS i16, groupArray(toInt32(number)) AS i32, groupArray(toInt64(number)) AS i64, groupArray(toUInt8(number)) AS u8, groupArray(toUInt16(number)) AS u16, groupArray(toUInt32(number)) AS u32, groupArray(toUInt64(number)) AS u64, groupArray(toFloat32(number)) AS f32, groupArray(toFloat64(number)) AS f64, groupArray(toDate(number, 'Asia/Istanbul')) AS date, groupArray(toDateTime(number, 'Asia/Istanbul')) AS date_time, groupArray(toString(number)) AS str, groupArray(toFixedString(toString(number), 5)) AS fixed_string FROM (SELECT number FROM system.numbers WHERE number >= 10 LIMIT 15); @@ -106,7 +106,7 @@ DROP TABLE IF EXISTS bloom_filter_array_types_test; DROP TABLE IF EXISTS bloom_filter_null_types_test; -CREATE TABLE bloom_filter_null_types_test (order_key UInt64, i8 Nullable(Int8), i16 Nullable(Int16), i32 Nullable(Int32), i64 Nullable(Int64), u8 Nullable(UInt8), u16 Nullable(UInt16), u32 Nullable(UInt32), u64 Nullable(UInt64), f32 Nullable(Float32), f64 Nullable(Float64), date Nullable(Date), date_time Nullable(DateTime('Asia/Istanbul')), str Nullable(String), fixed_string Nullable(FixedString(5)), INDEX idx (i8, i16, i32, i64, u8, u16, u32, u64, f32, f64, date, date_time, str, fixed_string) TYPE bloom_filter GRANULARITY 1) ENGINE = MergeTree() ORDER BY order_key SETTINGS index_granularity = 6; +CREATE TABLE bloom_filter_null_types_test (order_key UInt64, i8 Nullable(Int8), i16 Nullable(Int16), i32 Nullable(Int32), i64 Nullable(Int64), u8 Nullable(UInt8), u16 Nullable(UInt16), u32 Nullable(UInt32), u64 Nullable(UInt64), f32 Nullable(Float32), f64 Nullable(Float64), date Nullable(Date), date_time Nullable(DateTime('Asia/Istanbul')), str Nullable(String), fixed_string Nullable(FixedString(5)), INDEX idx (i8, i16, i32, i64, u8, u16, u32, u64, f32, f64, date, date_time, str, fixed_string) TYPE bloom_filter GRANULARITY 1) ENGINE = MergeTree() ORDER BY order_key SETTINGS index_granularity = 6, index_granularity_bytes = '10Mi'; INSERT INTO bloom_filter_null_types_test SELECT number AS order_key, toInt8(number) AS i8, toInt16(number) AS i16, toInt32(number) AS i32, toInt64(number) AS i64, toUInt8(number) AS u8, toUInt16(number) AS u16, toUInt32(number) AS u32, toUInt64(number) AS u64, toFloat32(number) AS f32, toFloat64(number) AS f64, toDate(number, 'Asia/Istanbul') AS date, toDateTime(number, 'Asia/Istanbul') AS date_time, toString(number) AS str, toFixedString(toString(number), 5) AS fixed_string FROM system.numbers LIMIT 100; INSERT INTO bloom_filter_null_types_test SELECT 0 AS order_key, NULL AS i8, NULL AS i16, NULL AS i32, NULL AS i64, NULL AS u8, NULL AS u16, NULL AS u32, NULL AS u64, NULL AS f32, NULL AS f64, NULL AS date, NULL AS date_time, NULL AS str, NULL AS fixed_string; @@ -146,7 +146,7 @@ DROP TABLE IF EXISTS bloom_filter_null_types_test; DROP TABLE IF EXISTS bloom_filter_lc_null_types_test; -CREATE TABLE bloom_filter_lc_null_types_test (order_key UInt64, str LowCardinality(Nullable(String)), fixed_string LowCardinality(Nullable(FixedString(5))), INDEX idx (str, fixed_string) TYPE bloom_filter GRANULARITY 1) ENGINE = MergeTree() ORDER BY order_key SETTINGS index_granularity = 6; +CREATE TABLE bloom_filter_lc_null_types_test (order_key UInt64, str LowCardinality(Nullable(String)), fixed_string LowCardinality(Nullable(FixedString(5))), INDEX idx (str, fixed_string) TYPE bloom_filter GRANULARITY 1) ENGINE = MergeTree() ORDER BY order_key SETTINGS index_granularity = 6, index_granularity_bytes = '10Mi'; INSERT INTO bloom_filter_lc_null_types_test SELECT number AS order_key, toString(number) AS str, toFixedString(toString(number), 5) AS fixed_string FROM system.numbers LIMIT 100; INSERT INTO bloom_filter_lc_null_types_test SELECT 0 AS order_key, NULL AS str, NULL AS fixed_string; @@ -183,7 +183,7 @@ CREATE TABLE bloom_filter_array_lc_null_types_test ( fixed_string Array(LowCardinality(Nullable(FixedString(5)))), INDEX idx (i8, i16, i32, i64, u8, u16, u32, u64, f32, f64, date, date_time, str, fixed_string) TYPE bloom_filter GRANULARITY 1) -ENGINE = MergeTree() ORDER BY order_key SETTINGS index_granularity = 6, allow_nullable_key = 1; +ENGINE = MergeTree() ORDER BY order_key SETTINGS index_granularity = 6, index_granularity_bytes = '10Mi', allow_nullable_key = 1; INSERT INTO bloom_filter_array_lc_null_types_test SELECT groupArray(number) AS order_key, @@ -286,19 +286,19 @@ SELECT COUNT() FROM bloom_filter_array_lc_null_types_test WHERE has(fixed_string DROP TABLE IF EXISTS bloom_filter_array_lc_null_types_test; DROP TABLE IF EXISTS bloom_filter_array_offsets_lc_str; -CREATE TABLE bloom_filter_array_offsets_lc_str (order_key int, str Array(LowCardinality(String)), INDEX idx str TYPE bloom_filter(1.) GRANULARITY 1024) ENGINE = MergeTree() ORDER BY order_key SETTINGS index_granularity = 1024; +CREATE TABLE bloom_filter_array_offsets_lc_str (order_key int, str Array(LowCardinality(String)), INDEX idx str TYPE bloom_filter(1.) GRANULARITY 1024) ENGINE = MergeTree() ORDER BY order_key SETTINGS index_granularity = 1024, index_granularity_bytes = '10Mi'; INSERT INTO bloom_filter_array_offsets_lc_str SELECT number AS i, if(i%2, ['value'], []) FROM system.numbers LIMIT 10000; SELECT count() FROM bloom_filter_array_offsets_lc_str WHERE has(str, 'value'); DROP TABLE IF EXISTS bloom_filter_array_offsets_lc_str; DROP TABLE IF EXISTS bloom_filter_array_offsets_str; -CREATE TABLE bloom_filter_array_offsets_str (order_key int, str Array(String), INDEX idx str TYPE bloom_filter(1.) GRANULARITY 1024) ENGINE = MergeTree() ORDER BY order_key SETTINGS index_granularity = 1024; +CREATE TABLE bloom_filter_array_offsets_str (order_key int, str Array(String), INDEX idx str TYPE bloom_filter(1.) GRANULARITY 1024) ENGINE = MergeTree() ORDER BY order_key SETTINGS index_granularity = 1024, index_granularity_bytes = '10Mi'; INSERT INTO bloom_filter_array_offsets_str SELECT number AS i, if(i%2, ['value'], []) FROM system.numbers LIMIT 10000; SELECT count() FROM bloom_filter_array_offsets_str WHERE has(str, 'value'); DROP TABLE IF EXISTS bloom_filter_array_offsets_str; DROP TABLE IF EXISTS bloom_filter_array_offsets_i; -CREATE TABLE bloom_filter_array_offsets_i (order_key int, i Array(int), INDEX idx i TYPE bloom_filter(1.) GRANULARITY 1024) ENGINE = MergeTree() ORDER BY order_key SETTINGS index_granularity = 1024; +CREATE TABLE bloom_filter_array_offsets_i (order_key int, i Array(int), INDEX idx i TYPE bloom_filter(1.) GRANULARITY 1024) ENGINE = MergeTree() ORDER BY order_key SETTINGS index_granularity = 1024, index_granularity_bytes = '10Mi'; INSERT INTO bloom_filter_array_offsets_i SELECT number AS i, if(i%2, [99999], []) FROM system.numbers LIMIT 10000; SELECT count() FROM bloom_filter_array_offsets_i WHERE has(i, 99999); DROP TABLE IF EXISTS bloom_filter_array_offsets_i; diff --git a/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.sql b/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.sql index 40da12baddc..43b7775e816 100644 --- a/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.sql +++ b/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.sql @@ -4,7 +4,7 @@ DROP TABLE IF EXISTS test_00961; CREATE TABLE test_00961 (d Date, a String, b UInt8, x String, y Int8, z UInt32) ENGINE = MergeTree PARTITION BY d ORDER BY (a, b) - SETTINGS index_granularity = 111, min_bytes_for_wide_part = 0, compress_marks = 0, compress_primary_key = 0; + SETTINGS index_granularity = 111, min_bytes_for_wide_part = 0, compress_marks = 0, compress_primary_key = 0, index_granularity_bytes = '10Mi'; INSERT INTO test_00961 VALUES ('2000-01-01', 'Hello, world!', 123, 'xxx yyy', -123, 123456789); @@ -18,4 +18,3 @@ FROM system.parts WHERE table = 'test_00961' and database = currentDatabase(); DROP TABLE test_00961; - diff --git a/tests/queries/0_stateless/00964_bloom_index_string_functions.sh b/tests/queries/0_stateless/00964_bloom_index_string_functions.sh index 7697578ea66..e2ec7fd42e4 100755 --- a/tests/queries/0_stateless/00964_bloom_index_string_functions.sh +++ b/tests/queries/0_stateless/00964_bloom_index_string_functions.sh @@ -15,7 +15,7 @@ CREATE TABLE bloom_filter_idx INDEX bf (s, lower(s)) TYPE ngrambf_v1(3, 512, 2, 0) GRANULARITY 1 ) ENGINE = MergeTree() ORDER BY k -SETTINGS index_granularity = 2;" +SETTINGS index_granularity = 2, index_granularity_bytes = '10Mi';" $CLICKHOUSE_CLIENT --query="INSERT INTO bloom_filter_idx VALUES (0, 'ClickHouse - столбцовая система управления базами данных (СУБД)'), diff --git a/tests/queries/0_stateless/00965_set_index_string_functions.sh b/tests/queries/0_stateless/00965_set_index_string_functions.sh index dba33d9abcf..8892fb11752 100755 --- a/tests/queries/0_stateless/00965_set_index_string_functions.sh +++ b/tests/queries/0_stateless/00965_set_index_string_functions.sh @@ -14,7 +14,7 @@ CREATE TABLE set_idx INDEX idx (s) TYPE set(2) GRANULARITY 1 ) ENGINE = MergeTree() ORDER BY k -SETTINGS index_granularity = 2;" +SETTINGS index_granularity = 2, index_granularity_bytes = '10Mi';" $CLICKHOUSE_CLIENT --query="INSERT INTO set_idx VALUES (0, 'ClickHouse - столбцовая система управления базами данных (СУБД)'), diff --git a/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper_long.sh b/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper_long.sh index 5ad4007b873..89b17ffe2bf 100755 --- a/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper_long.sh +++ b/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper_long.sh @@ -21,7 +21,7 @@ CREATE TABLE indices_mutaions1 ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/indices_mutaions', 'r1') PARTITION BY i32 ORDER BY u64 -SETTINGS index_granularity = 2; +SETTINGS index_granularity = 2, index_granularity_bytes = '10Mi'; CREATE TABLE indices_mutaions2 ( @@ -32,7 +32,7 @@ CREATE TABLE indices_mutaions2 ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/indices_mutaions', 'r2') PARTITION BY i32 ORDER BY u64 -SETTINGS index_granularity = 2;" +SETTINGS index_granularity = 2, index_granularity_bytes = '10Mi';" $CLICKHOUSE_CLIENT --query="INSERT INTO indices_mutaions1 VALUES diff --git a/tests/queries/0_stateless/00980_alter_settings_race.sh b/tests/queries/0_stateless/00980_alter_settings_race.sh index 6eb1df8964c..2fad7b79301 100755 --- a/tests/queries/0_stateless/00980_alter_settings_race.sh +++ b/tests/queries/0_stateless/00980_alter_settings_race.sh @@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS table_for_concurrent_alter" -$CLICKHOUSE_CLIENT --query="CREATE TABLE table_for_concurrent_alter (id UInt64, Data String) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity=4096;"; +$CLICKHOUSE_CLIENT --query="CREATE TABLE table_for_concurrent_alter (id UInt64, Data String) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity=4096, index_granularity_bytes = '10Mi';"; n=0 while [ "$n" -lt 50 ]; diff --git a/tests/queries/0_stateless/00980_merge_alter_settings.reference b/tests/queries/0_stateless/00980_merge_alter_settings.reference index 7a958c40651..706b64184ca 100644 --- a/tests/queries/0_stateless/00980_merge_alter_settings.reference +++ b/tests/queries/0_stateless/00980_merge_alter_settings.reference @@ -1,12 +1,12 @@ -CREATE TABLE default.table_for_alter\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096 -CREATE TABLE default.table_for_alter\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096, parts_to_throw_insert = 1, parts_to_delay_insert = 1 -CREATE TABLE default.table_for_alter\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096, parts_to_throw_insert = 100, parts_to_delay_insert = 100 +CREATE TABLE default.table_for_alter\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096, index_granularity_bytes = '10Mi' +CREATE TABLE default.table_for_alter\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096, index_granularity_bytes = '10Mi', parts_to_throw_insert = 1, parts_to_delay_insert = 1 +CREATE TABLE default.table_for_alter\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096, index_granularity_bytes = '10Mi', parts_to_throw_insert = 100, parts_to_delay_insert = 100 2 -CREATE TABLE default.table_for_alter\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096, parts_to_throw_insert = 100, parts_to_delay_insert = 100, check_delay_period = 30 -CREATE TABLE default.table_for_alter\n(\n `id` UInt64,\n `Data` String,\n `Data2` UInt64\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096, parts_to_throw_insert = 100, parts_to_delay_insert = 100, check_delay_period = 15 -CREATE TABLE default.table_for_reset_setting\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096 +CREATE TABLE default.table_for_alter\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096, index_granularity_bytes = '10Mi', parts_to_throw_insert = 100, parts_to_delay_insert = 100, check_delay_period = 30 +CREATE TABLE default.table_for_alter\n(\n `id` UInt64,\n `Data` String,\n `Data2` UInt64\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096, index_granularity_bytes = '10Mi', parts_to_throw_insert = 100, parts_to_delay_insert = 100, check_delay_period = 15 +CREATE TABLE default.table_for_reset_setting\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096, index_granularity_bytes = '10Mi' CREATE TABLE default.table_for_reset_setting\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096, parts_to_throw_insert = 1, parts_to_delay_insert = 1 -CREATE TABLE default.table_for_reset_setting\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096 -CREATE TABLE default.table_for_reset_setting\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096 -CREATE TABLE default.table_for_reset_setting\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096, merge_with_ttl_timeout = 300, max_concurrent_queries = 1 -CREATE TABLE default.table_for_reset_setting\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096 +CREATE TABLE default.table_for_reset_setting\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096, index_granularity_bytes = '10Mi' +CREATE TABLE default.table_for_reset_setting\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096, index_granularity_bytes = '10Mi' +CREATE TABLE default.table_for_reset_setting\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096, index_granularity_bytes = '10Mi', merge_with_ttl_timeout = 300, max_concurrent_queries = 1 +CREATE TABLE default.table_for_reset_setting\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096, index_granularity_bytes = '10Mi' diff --git a/tests/queries/0_stateless/00980_merge_alter_settings.sql b/tests/queries/0_stateless/00980_merge_alter_settings.sql index f595a09970d..02728a6ba8b 100644 --- a/tests/queries/0_stateless/00980_merge_alter_settings.sql +++ b/tests/queries/0_stateless/00980_merge_alter_settings.sql @@ -17,7 +17,7 @@ DROP TABLE IF EXISTS table_for_alter; CREATE TABLE table_for_alter ( id UInt64, Data String -) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity=4096; +) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity=4096, index_granularity_bytes = '10Mi'; ALTER TABLE table_for_alter MODIFY SETTING index_granularity=555; -- { serverError 472 } @@ -62,7 +62,7 @@ DROP TABLE IF EXISTS table_for_reset_setting; CREATE TABLE table_for_reset_setting ( id UInt64, Data String -) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity=4096; +) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity=4096, index_granularity_bytes = '10Mi'; ALTER TABLE table_for_reset_setting MODIFY SETTING index_granularity=555; -- { serverError 472 } diff --git a/tests/queries/0_stateless/00990_hasToken_and_tokenbf.sql b/tests/queries/0_stateless/00990_hasToken_and_tokenbf.sql index d3b36cda0d8..41676905771 100644 --- a/tests/queries/0_stateless/00990_hasToken_and_tokenbf.sql +++ b/tests/queries/0_stateless/00990_hasToken_and_tokenbf.sql @@ -5,7 +5,7 @@ CREATE TABLE bloom_filter id UInt64, s String, INDEX tok_bf (s, lower(s)) TYPE tokenbf_v1(512, 3, 0) GRANULARITY 1 -) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity = 8; +) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity = 8, index_granularity_bytes = '10Mi'; insert into bloom_filter select number, 'yyy,uuu' from numbers(1024); insert into bloom_filter select number+2000, 'abc,def,zzz' from numbers(8); @@ -38,4 +38,4 @@ SELECT max(id) FROM bloom_filter WHERE hasToken(s, 'yyy'); -- { serverError 158 -- this syntax is not supported by tokenbf SELECT max(id) FROM bloom_filter WHERE hasToken(s, 'zzz') == 1; -- { serverError 158 } -DROP TABLE bloom_filter; \ No newline at end of file +DROP TABLE bloom_filter; diff --git a/tests/queries/0_stateless/01030_final_mark_empty_primary_key.sql b/tests/queries/0_stateless/01030_final_mark_empty_primary_key.sql index c64b40dfd1d..7bf2e3e737b 100644 --- a/tests/queries/0_stateless/01030_final_mark_empty_primary_key.sql +++ b/tests/queries/0_stateless/01030_final_mark_empty_primary_key.sql @@ -1,5 +1,5 @@ DROP TABLE IF EXISTS empty_pk; -CREATE TABLE empty_pk (x UInt64) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 256; +CREATE TABLE empty_pk (x UInt64) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 256, index_granularity_bytes = '10Mi'; INSERT INTO empty_pk SELECT number FROM numbers(100000); diff --git a/tests/queries/0_stateless/01055_minmax_index_compact_parts.sh b/tests/queries/0_stateless/01055_minmax_index_compact_parts.sh index 030a32dcdcb..0b14ef8f6fa 100755 --- a/tests/queries/0_stateless/01055_minmax_index_compact_parts.sh +++ b/tests/queries/0_stateless/01055_minmax_index_compact_parts.sh @@ -25,7 +25,7 @@ CREATE TABLE minmax_idx INDEX idx_2 (u64 + toYear(dt), substring(s, 2, 4)) TYPE minmax GRANULARITY 3 ) ENGINE = MergeTree() ORDER BY u64 -SETTINGS index_granularity = 2, min_rows_for_wide_part = 1000000" +SETTINGS index_granularity = 2, index_granularity_bytes = '10Mi', min_rows_for_wide_part = 1000000" $CLICKHOUSE_CLIENT --query="INSERT INTO minmax_idx VALUES @@ -50,4 +50,4 @@ $CLICKHOUSE_CLIENT --query="SELECT * FROM minmax_idx WHERE i32 = 5 AND i32 + f64 $CLICKHOUSE_CLIENT --query="SELECT * FROM minmax_idx WHERE (u64 < 2 OR u64 > 10) AND e != 'b' ORDER BY dt" $CLICKHOUSE_CLIENT --query="SELECT * FROM minmax_idx WHERE (u64 < 2 OR u64 > 10) AND e != 'b' ORDER BY dt FORMAT JSON" | grep "rows_read" -$CLICKHOUSE_CLIENT --query="DROP TABLE minmax_idx" \ No newline at end of file +$CLICKHOUSE_CLIENT --query="DROP TABLE minmax_idx" diff --git a/tests/queries/0_stateless/01064_incremental_streaming_from_2_src_with_feedback.sql b/tests/queries/0_stateless/01064_incremental_streaming_from_2_src_with_feedback.sql index 9a439180265..3d75fb0ccc9 100644 --- a/tests/queries/0_stateless/01064_incremental_streaming_from_2_src_with_feedback.sql +++ b/tests/queries/0_stateless/01064_incremental_streaming_from_2_src_with_feedback.sql @@ -13,7 +13,7 @@ DROP TABLE IF EXISTS mv_checkouts2target; -- that is the final table, which is filled incrementally from 2 different sources CREATE TABLE target_table Engine=SummingMergeTree() ORDER BY id -SETTINGS index_granularity=128 +SETTINGS index_granularity=128, index_granularity_bytes = '10Mi' AS SELECT number as id, diff --git a/tests/queries/0_stateless/01114_materialize_clear_index_compact_parts.sql b/tests/queries/0_stateless/01114_materialize_clear_index_compact_parts.sql index 831cb25d967..767ca0e4073 100644 --- a/tests/queries/0_stateless/01114_materialize_clear_index_compact_parts.sql +++ b/tests/queries/0_stateless/01114_materialize_clear_index_compact_parts.sql @@ -8,7 +8,7 @@ CREATE TABLE minmax_compact ) ENGINE = MergeTree() PARTITION BY i32 ORDER BY u64 -SETTINGS index_granularity = 2, min_rows_for_wide_part = 1000000; +SETTINGS index_granularity = 2, index_granularity_bytes = '10Mi', min_rows_for_wide_part = 1000000; INSERT INTO minmax_compact VALUES (0, 2, 1), (1, 1, 1), (2, 1, 1), (3, 1, 1), (4, 1, 1), (5, 2, 1), (6, 1, 2), (7, 1, 2), (8, 1, 2), (9, 1, 2); diff --git a/tests/queries/0_stateless/01201_read_single_thread_in_order.sql b/tests/queries/0_stateless/01201_read_single_thread_in_order.sql index bfe03192891..33ccbbbe84d 100644 --- a/tests/queries/0_stateless/01201_read_single_thread_in_order.sql +++ b/tests/queries/0_stateless/01201_read_single_thread_in_order.sql @@ -6,7 +6,7 @@ CREATE TABLE t ) ENGINE = MergeTree ORDER BY number -SETTINGS index_granularity = 128; +SETTINGS index_granularity = 128, index_granularity_bytes = '10Mi'; SET min_insert_block_size_rows = 0, min_insert_block_size_bytes = 0; INSERT INTO t SELECT number FROM numbers(10000000); diff --git a/tests/queries/0_stateless/01307_bloom_filter_index_string_multi_granulas.sql b/tests/queries/0_stateless/01307_bloom_filter_index_string_multi_granulas.sql index e96c70bef7f..cfb1f45c19a 100644 --- a/tests/queries/0_stateless/01307_bloom_filter_index_string_multi_granulas.sql +++ b/tests/queries/0_stateless/01307_bloom_filter_index_string_multi_granulas.sql @@ -1,6 +1,6 @@ DROP TABLE IF EXISTS test_01307; -CREATE TABLE test_01307 (id UInt64, val String, INDEX ind val TYPE bloom_filter() GRANULARITY 1) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity = 2; +CREATE TABLE test_01307 (id UInt64, val String, INDEX ind val TYPE bloom_filter() GRANULARITY 1) ENGINE = MergeTree() ORDER BY id SETTINGS index_granularity = 2, index_granularity_bytes = '10Mi'; INSERT INTO test_01307 (id, val) select number as id, toString(number) as val from numbers(4); SELECT count() FROM test_01307 WHERE identity(val) = '2'; SELECT count() FROM test_01307 WHERE val = '2'; diff --git a/tests/queries/0_stateless/01312_comparison_with_constant_string_in_index_analysis.sql b/tests/queries/0_stateless/01312_comparison_with_constant_string_in_index_analysis.sql index e37f647e81f..50aa434a28c 100644 --- a/tests/queries/0_stateless/01312_comparison_with_constant_string_in_index_analysis.sql +++ b/tests/queries/0_stateless/01312_comparison_with_constant_string_in_index_analysis.sql @@ -1,5 +1,5 @@ DROP TABLE IF EXISTS test; -CREATE TABLE test (x UInt64) ENGINE = MergeTree ORDER BY x SETTINGS index_granularity = 1000; +CREATE TABLE test (x UInt64) ENGINE = MergeTree ORDER BY x SETTINGS index_granularity = 1000, index_granularity_bytes = '10Mi'; INSERT INTO test SELECT * FROM numbers(1000000); OPTIMIZE TABLE test; diff --git a/tests/queries/0_stateless/01515_mv_and_array_join_optimisation_bag.sql b/tests/queries/0_stateless/01515_mv_and_array_join_optimisation_bag.sql index ad762ea65fb..56dabc2a7cf 100644 --- a/tests/queries/0_stateless/01515_mv_and_array_join_optimisation_bag.sql +++ b/tests/queries/0_stateless/01515_mv_and_array_join_optimisation_bag.sql @@ -43,7 +43,7 @@ CREATE TABLE goal `GoalID` UInt32, `Visits` AggregateFunction(sumIf, Int8, UInt8), `GoalReaches` AggregateFunction(sum, Int8) -) ENGINE = AggregatingMergeTree PARTITION BY toStartOfMonth(StartDate) ORDER BY (CounterID, StartDate, GoalID) SETTINGS index_granularity = 256; +) ENGINE = AggregatingMergeTree PARTITION BY toStartOfMonth(StartDate) ORDER BY (CounterID, StartDate, GoalID) SETTINGS index_granularity = 256, index_granularity_bytes = '10Mi'; INSERT INTO visits (`CounterID`,`StartDate`,`StartTime`,`Sign`,`GoalsID`) VALUES (1, toDate('2000-01-01'), toDateTime(toDate('2000-01-01')), 1, [1]); diff --git a/tests/queries/0_stateless/01605_skip_idx_compact_parts.sql b/tests/queries/0_stateless/01605_skip_idx_compact_parts.sql index d57ccdcf9fc..9d44550c0ba 100644 --- a/tests/queries/0_stateless/01605_skip_idx_compact_parts.sql +++ b/tests/queries/0_stateless/01605_skip_idx_compact_parts.sql @@ -1,7 +1,7 @@ DROP TABLE IF EXISTS skip_idx_comp_parts; CREATE TABLE skip_idx_comp_parts (a Int, b Int, index b_idx b TYPE minmax GRANULARITY 4) ENGINE = MergeTree ORDER BY a - SETTINGS index_granularity=256, merge_max_block_size=100; + SETTINGS index_granularity=256, index_granularity_bytes = '10Mi', merge_max_block_size=100; SYSTEM STOP MERGES skip_idx_comp_parts; diff --git a/tests/queries/0_stateless/01681_bloom_filter_nullable_column.sql b/tests/queries/0_stateless/01681_bloom_filter_nullable_column.sql index 4af1f74fca6..50663654b10 100644 --- a/tests/queries/0_stateless/01681_bloom_filter_nullable_column.sql +++ b/tests/queries/0_stateless/01681_bloom_filter_nullable_column.sql @@ -7,7 +7,7 @@ CREATE TABLE bloom_filter_nullable_index INDEX idx (str) TYPE bloom_filter GRANULARITY 1 ) ENGINE = MergeTree() - ORDER BY order_key SETTINGS index_granularity = 6; + ORDER BY order_key SETTINGS index_granularity = 6, index_granularity_bytes = '10Mi'; INSERT INTO bloom_filter_nullable_index VALUES (1, 'test'); INSERT INTO bloom_filter_nullable_index VALUES (2, 'test2'); diff --git a/tests/queries/0_stateless/01780_column_sparse_full.sql b/tests/queries/0_stateless/01780_column_sparse_full.sql index 08a1c0699a4..57da1974373 100644 --- a/tests/queries/0_stateless/01780_column_sparse_full.sql +++ b/tests/queries/0_stateless/01780_column_sparse_full.sql @@ -5,7 +5,7 @@ DROP TABLE IF EXISTS t_sparse_full; CREATE TABLE t_sparse_full (id UInt64, u UInt64, s String) ENGINE = MergeTree ORDER BY id -SETTINGS index_granularity = 32, +SETTINGS index_granularity = 32, index_granularity_bytes = '10Mi', ratio_of_defaults_for_sparse_serialization = 0.1; SYSTEM STOP MERGES t_sparse_full; diff --git a/tests/queries/0_stateless/01781_token_extractor_buffer_overflow.sql b/tests/queries/0_stateless/01781_token_extractor_buffer_overflow.sql index 4cc216955b3..400792df8be 100644 --- a/tests/queries/0_stateless/01781_token_extractor_buffer_overflow.sql +++ b/tests/queries/0_stateless/01781_token_extractor_buffer_overflow.sql @@ -1,7 +1,7 @@ SET max_block_size = 10, min_insert_block_size_rows = 0, min_insert_block_size_bytes = 0, max_threads = 20; DROP TABLE IF EXISTS bloom_filter; -CREATE TABLE bloom_filter (`id` UInt64, `s` String, INDEX tok_bf (s, lower(s)) TYPE tokenbf_v1(512, 3, 0) GRANULARITY 1) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 8; +CREATE TABLE bloom_filter (`id` UInt64, `s` String, INDEX tok_bf (s, lower(s)) TYPE tokenbf_v1(512, 3, 0) GRANULARITY 1) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 8, index_granularity_bytes = '10Mi'; INSERT INTO bloom_filter SELECT number, 'yyy,uuu' FROM numbers(1024); SELECT max(id) FROM bloom_filter WHERE hasToken(s, 'abc'); diff --git a/tests/queries/0_stateless/01926_order_by_desc_limit.sql b/tests/queries/0_stateless/01926_order_by_desc_limit.sql index 92c7a27bc9a..a0047a2925a 100644 --- a/tests/queries/0_stateless/01926_order_by_desc_limit.sql +++ b/tests/queries/0_stateless/01926_order_by_desc_limit.sql @@ -6,7 +6,7 @@ SET enable_filesystem_cache=0; CREATE TABLE order_by_desc (u UInt32, s String) ENGINE MergeTree ORDER BY u PARTITION BY u % 100 -SETTINGS index_granularity = 1024; +SETTINGS index_granularity = 1024, index_granularity_bytes = '10Mi'; INSERT INTO order_by_desc SELECT number, repeat('a', 1024) FROM numbers(1024 * 300); OPTIMIZE TABLE order_by_desc FINAL; diff --git a/tests/queries/0_stateless/02030_tuple_filter.sql b/tests/queries/0_stateless/02030_tuple_filter.sql index d2a114a89f9..f2fc3a30aa6 100644 --- a/tests/queries/0_stateless/02030_tuple_filter.sql +++ b/tests/queries/0_stateless/02030_tuple_filter.sql @@ -2,7 +2,7 @@ SET allow_experimental_analyzer = 1; DROP TABLE IF EXISTS test_tuple_filter; -CREATE TABLE test_tuple_filter (id UInt32, value String, log_date Date) Engine=MergeTree() ORDER BY id PARTITION BY log_date SETTINGS index_granularity = 3; +CREATE TABLE test_tuple_filter (id UInt32, value String, log_date Date) Engine=MergeTree() ORDER BY id PARTITION BY log_date SETTINGS index_granularity = 3, index_granularity_bytes = '10Mi'; INSERT INTO test_tuple_filter VALUES (1,'A','2021-01-01'),(2,'B','2021-01-01'),(3,'C','2021-01-01'),(4,'D','2021-01-02'),(5,'E','2021-01-02'); diff --git a/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.sql b/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.sql index 65a8cadb996..f50aab67d77 100644 --- a/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.sql +++ b/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.sql @@ -38,7 +38,7 @@ DROP TABLE IF EXISTS t_read_in_order; CREATE TABLE t_read_in_order(a UInt32, b UInt32) ENGINE = MergeTree ORDER BY (a, b) -SETTINGS index_granularity = 3; +SETTINGS index_granularity = 3, index_granularity_bytes = '10Mi'; SYSTEM STOP MERGES t_read_in_order; diff --git a/tests/queries/0_stateless/02155_read_in_order_max_rows_to_read.sql b/tests/queries/0_stateless/02155_read_in_order_max_rows_to_read.sql index 314d0610d12..b387582296d 100644 --- a/tests/queries/0_stateless/02155_read_in_order_max_rows_to_read.sql +++ b/tests/queries/0_stateless/02155_read_in_order_max_rows_to_read.sql @@ -2,7 +2,7 @@ DROP TABLE IF EXISTS t_max_rows_to_read; CREATE TABLE t_max_rows_to_read (a UInt64) ENGINE = MergeTree ORDER BY a -SETTINGS index_granularity = 4; +SETTINGS index_granularity = 4, index_granularity_bytes = '10Mi'; INSERT INTO t_max_rows_to_read SELECT number FROM numbers(100); diff --git a/tests/queries/0_stateless/02267_empty_arrays_read_reverse.sql b/tests/queries/0_stateless/02267_empty_arrays_read_reverse.sql index 7c1cf47c540..0c6c1a46ee1 100644 --- a/tests/queries/0_stateless/02267_empty_arrays_read_reverse.sql +++ b/tests/queries/0_stateless/02267_empty_arrays_read_reverse.sql @@ -8,7 +8,7 @@ CREATE TABLE t_02267 ) ENGINE = MergeTree ORDER BY b -SETTINGS index_granularity = 500; +SETTINGS index_granularity = 500, index_granularity_bytes = '10Mi'; INSERT INTO t_02267 (b, a, c) SELECT 0, ['x'], ['1','2','3','4','5','6'] FROM numbers(1) ; INSERT INTO t_02267 (b, a, c) SELECT 1, [], ['1','2','3','4','5','6'] FROM numbers(300000); diff --git a/tests/queries/0_stateless/02346_full_text_search.sql b/tests/queries/0_stateless/02346_full_text_search.sql index cead2ce0666..af49c5d52c2 100644 --- a/tests/queries/0_stateless/02346_full_text_search.sql +++ b/tests/queries/0_stateless/02346_full_text_search.sql @@ -8,7 +8,7 @@ DROP TABLE IF EXISTS tab; CREATE TABLE tab(k UInt64, s String, INDEX af(s) TYPE inverted(2)) ENGINE = MergeTree() ORDER BY k - SETTINGS index_granularity = 2; + SETTINGS index_granularity = 2, index_granularity_bytes = '10Mi'; INSERT INTO tab VALUES (101, 'Alick a01'), (102, 'Blick a02'), (103, 'Click a03'), (104, 'Dlick a04'), (105, 'Elick a05'), (106, 'Alick a06'), (107, 'Blick a07'), (108, 'Click a08'), (109, 'Dlick a09'), (110, 'Elick a10'), (111, 'Alick b01'), (112, 'Blick b02'), (113, 'Click b03'), (114, 'Dlick b04'), (115, 'Elick b05'), (116, 'Alick b06'), (117, 'Blick b07'), (118, 'Click b08'), (119, 'Dlick b09'), (120, 'Elick b10'); @@ -61,7 +61,7 @@ DROP TABLE IF EXISTS tab_x; CREATE TABLE tab_x(k UInt64, s String, INDEX af(s) TYPE inverted()) ENGINE = MergeTree() ORDER BY k - SETTINGS index_granularity = 2; + SETTINGS index_granularity = 2, index_granularity_bytes = '10Mi'; INSERT INTO tab_x VALUES (101, 'Alick a01'), (102, 'Blick a02'), (103, 'Click a03'), (104, 'Dlick a04'), (105, 'Elick a05'), (106, 'Alick a06'), (107, 'Blick a07'), (108, 'Click a08'), (109, 'Dlick a09'), (110, 'Elick a10'), (111, 'Alick b01'), (112, 'Blick b02'), (113, 'Click b03'), (114, 'Dlick b04'), (115, 'Elick b05'), (116, 'Alick b06'), (117, 'Blick b07'), (118, 'Click b08'), (119, 'Dlick b09'), (120, 'Elick b10'); @@ -114,7 +114,7 @@ DROP TABLE IF EXISTS tab; create table tab (k UInt64, s Array(String), INDEX af(s) TYPE inverted(2)) ENGINE = MergeTree() ORDER BY k - SETTINGS index_granularity = 2; + SETTINGS index_granularity = 2, index_granularity_bytes = '10Mi'; INSERT INTO tab SELECT rowNumberInBlock(), groupArray(s) FROM tab_x GROUP BY k%10; @@ -141,7 +141,7 @@ DROP TABLE IF EXISTS tab; CREATE TABLE tab (k UInt64, s Map(String,String), INDEX af(mapKeys(s)) TYPE inverted(2)) ENGINE = MergeTree() ORDER BY k - SETTINGS index_granularity = 2; + SETTINGS index_granularity = 2, index_granularity_bytes = '10Mi'; INSERT INTO tab VALUES (101, {'Alick':'Alick a01'}), (102, {'Blick':'Blick a02'}), (103, {'Click':'Click a03'}), (104, {'Dlick':'Dlick a04'}), (105, {'Elick':'Elick a05'}), (106, {'Alick':'Alick a06'}), (107, {'Blick':'Blick a07'}), (108, {'Click':'Click a08'}), (109, {'Dlick':'Dlick a09'}), (110, {'Elick':'Elick a10'}), (111, {'Alick':'Alick b01'}), (112, {'Blick':'Blick b02'}), (113, {'Click':'Click b03'}), (114, {'Dlick':'Dlick b04'}), (115, {'Elick':'Elick b05'}), (116, {'Alick':'Alick b06'}), (117, {'Blick':'Blick b07'}), (118, {'Click':'Click b08'}), (119, {'Dlick':'Dlick b09'}), (120, {'Elick':'Elick b10'}); @@ -181,7 +181,7 @@ DROP TABLE IF EXISTS tab; CREATE TABLE tab(k UInt64, s String, INDEX af(s) TYPE inverted(2)) ENGINE = MergeTree() ORDER BY k - SETTINGS index_granularity = 2; + SETTINGS index_granularity = 2, index_granularity_bytes = '10Mi'; INSERT INTO tab VALUES (101, 'Alick a01'), (102, 'Blick a02'), (103, 'Click a03'), (104, 'Dlick a04'), (105, 'Elick a05'), (106, 'Alick a06'), (107, 'Blick a07'), (108, 'Click a08'), (109, 'Dlick a09'), (110, 'Elick b10'), (111, 'Alick b01'), (112, 'Blick b02'), (113, 'Click b03'), (114, 'Dlick b04'), (115, 'Elick b05'), (116, 'Alick b06'), (117, 'Blick b07'), (118, 'Click b08'), (119, 'Dlick b09'), (120, 'Elick b10'); INSERT INTO tab VALUES (201, 'rick c01'), (202, 'mick c02'), (203, 'nick c03'); @@ -210,7 +210,7 @@ DROP TABLE IF EXISTS tab; CREATE TABLE tab(k UInt64, s String, INDEX af(s) TYPE inverted(2)) ENGINE = MergeTree() ORDER BY k - SETTINGS index_granularity = 2; + SETTINGS index_granularity = 2, index_granularity_bytes = '10Mi'; INSERT INTO tab VALUES (101, 'Alick 好'), (102, 'clickhouse你好'), (103, 'Click 你'), (104, 'Dlick 你a好'), (105, 'Elick 好好你你'), (106, 'Alick 好a好a你a你'); @@ -332,4 +332,3 @@ SELECT read_rows==512 from system.query_log AND type='QueryFinish' AND result_rows==1 LIMIT 1; - diff --git a/tests/queries/0_stateless/02354_annoy.sh b/tests/queries/0_stateless/02354_annoy.sh index 526886ec68d..7b49a338955 100755 --- a/tests/queries/0_stateless/02354_annoy.sh +++ b/tests/queries/0_stateless/02354_annoy.sh @@ -17,7 +17,7 @@ CREATE TABLE 02354_annoy_l2 ) ENGINE = MergeTree ORDER BY id -SETTINGS index_granularity=5; +SETTINGS index_granularity=5, index_granularity_bytes = '10Mi'; INSERT INTO 02354_annoy_l2 VALUES (1, [0.0, 0.0, 10.0]), (2, [0.0, 0.0, 10.5]), (3, [0.0, 0.0, 9.5]), (4, [0.0, 0.0, 9.7]), (5, [0.0, 0.0, 10.2]), (6, [10.0, 0.0, 0.0]), (7, [9.5, 0.0, 0.0]), (8, [9.7, 0.0, 0.0]), (9, [10.2, 0.0, 0.0]), (10, [10.5, 0.0, 0.0]), (11, [0.0, 10.0, 0.0]), (12, [0.0, 9.5, 0.0]), (13, [0.0, 9.7, 0.0]), (14, [0.0, 10.2, 0.0]), (15, [0.0, 10.5, 0.0]); @@ -64,7 +64,7 @@ CREATE TABLE 02354_annoy_l2 ) ENGINE = MergeTree ORDER BY id -SETTINGS index_granularity=5; +SETTINGS index_granularity=5, index_granularity_bytes = '10Mi'; INSERT INTO 02354_annoy_l2 VALUES (1, [0.0, 0.0, 10.0]), (2, [0.0, 0.0, 10.5]), (3, [0.0, 0.0, 9.5]), (4, [0.0, 0.0, 9.7]), (5, [0.0, 0.0, 10.2]), (6, [10.0, 0.0, 0.0]), (7, [9.5, 0.0, 0.0]), (8, [9.7, 0.0, 0.0]), (9, [10.2, 0.0, 0.0]), (10, [10.5, 0.0, 0.0]), (11, [0.0, 10.0, 0.0]), (12, [0.0, 9.5, 0.0]), (13, [0.0, 9.7, 0.0]), (14, [0.0, 10.2, 0.0]), (15, [0.0, 10.5, 0.0]); @@ -95,7 +95,7 @@ CREATE TABLE 02354_annoy_cosine ) ENGINE = MergeTree ORDER BY id -SETTINGS index_granularity=5; +SETTINGS index_granularity=5, index_granularity_bytes = '10Mi'; INSERT INTO 02354_annoy_cosine VALUES (1, [0.0, 0.0, 10.0]), (2, [0.2, 0.0, 10.0]), (3, [-0.3, 0.0, 10.0]), (4, [0.5, 0.0, 10.1]), (5, [0.8, 0.0, 10.0]), (6, [10.0, 0.0, 0.0]), (7, [9.5, 0.0, 0.0]), (8, [9.7, 0.0, 0.0]), (9, [10.2, 0.0, 0.0]), (10, [10.5, 0.0, 0.0]), (11, [0.0, 10.0, 0.0]), (12, [0.0, 9.5, 0.0]), (13, [0.0, 9.7, 0.0]), (14, [0.0, 10.2, 0.0]), (15, [0.0, 10.5, 0.0]); @@ -124,7 +124,7 @@ CREATE TABLE 02354_annoy_cosine ) ENGINE = MergeTree ORDER BY id -SETTINGS index_granularity=5; +SETTINGS index_granularity=5, index_granularity_bytes = '10Mi'; INSERT INTO 02354_annoy_cosine VALUES (1, [0.0, 0.0, 10.0]), (2, [0.2, 0.0, 10.0]), (3, [-0.3, 0.0, 10.0]), (4, [0.5, 0.0, 10.1]), (5, [0.8, 0.0, 10.0]), (6, [10.0, 0.0, 0.0]), (7, [9.5, 0.0, 0.0]), (8, [9.7, 0.0, 0.0]), (9, [10.2, 0.0, 0.0]), (10, [10.5, 0.0, 0.0]), (11, [0.0, 10.0, 0.0]), (12, [0.0, 9.5, 0.0]), (13, [0.0, 9.7, 0.0]), (14, [0.0, 10.2, 0.0]), (15, [0.0, 10.5, 0.0]); @@ -156,7 +156,7 @@ CREATE TABLE 02354_annoy ) ENGINE = MergeTree ORDER BY id -SETTINGS index_granularity=5; -- {serverError 7 } +SETTINGS index_granularity=5, index_granularity_bytes = '10Mi'; -- {serverError 7 } -- Index must be created on Array(Float32) or Tuple(Float32) @@ -168,7 +168,7 @@ CREATE TABLE 02354_annoy ) ENGINE = MergeTree ORDER BY id -SETTINGS index_granularity=5; -- {serverError 44 } +SETTINGS index_granularity=5, index_granularity_bytes = '10Mi'; -- {serverError 44 } CREATE TABLE 02354_annoy @@ -179,7 +179,7 @@ CREATE TABLE 02354_annoy ) ENGINE = MergeTree ORDER BY id -SETTINGS index_granularity=5; -- {serverError 44 } +SETTINGS index_granularity=5, index_granularity_bytes = '10Mi'; -- {serverError 44 } CREATE TABLE 02354_annoy ( @@ -189,7 +189,7 @@ CREATE TABLE 02354_annoy ) ENGINE = MergeTree ORDER BY id -SETTINGS index_granularity=5; -- {serverError 44 } +SETTINGS index_granularity=5, index_granularity_bytes = '10Mi'; -- {serverError 44 } CREATE TABLE 02354_annoy ( @@ -199,7 +199,7 @@ CREATE TABLE 02354_annoy ) ENGINE = MergeTree ORDER BY id -SETTINGS index_granularity=5; -- {serverError 44 } +SETTINGS index_granularity=5, index_granularity_bytes = '10Mi'; -- {serverError 44 } CREATE TABLE 02354_annoy ( @@ -209,4 +209,4 @@ CREATE TABLE 02354_annoy ) ENGINE = MergeTree ORDER BY id -SETTINGS index_granularity=5; -- {serverError 44 }" +SETTINGS index_granularity=5, index_granularity_bytes = '10Mi'; -- {serverError 44 }" diff --git a/tests/queries/0_stateless/02374_in_tuple_index.sql b/tests/queries/0_stateless/02374_in_tuple_index.sql index 7f9b7b5470e..4f489f74eda 100644 --- a/tests/queries/0_stateless/02374_in_tuple_index.sql +++ b/tests/queries/0_stateless/02374_in_tuple_index.sql @@ -8,7 +8,7 @@ CREATE TABLE t_in_tuple_index ) ENGINE = MergeTree() ORDER BY (PLATFORM, USER_ID, ID) -SETTINGS index_granularity = 2048; +SETTINGS index_granularity = 2048, index_granularity_bytes = '10Mi'; INSERT INTO t_in_tuple_index VALUES ('1', 33, 'insta'), ('2', 33, 'insta'); diff --git a/tests/queries/0_stateless/02465_limit_trivial_max_rows_to_read.sql b/tests/queries/0_stateless/02465_limit_trivial_max_rows_to_read.sql index ee7a4e6b6b5..c2e97c8c704 100644 --- a/tests/queries/0_stateless/02465_limit_trivial_max_rows_to_read.sql +++ b/tests/queries/0_stateless/02465_limit_trivial_max_rows_to_read.sql @@ -2,7 +2,7 @@ DROP TABLE IF EXISTS t_max_rows_to_read; CREATE TABLE t_max_rows_to_read (a UInt64) ENGINE = MergeTree ORDER BY a -SETTINGS index_granularity = 4; +SETTINGS index_granularity = 4, index_granularity_bytes = '10Mi'; INSERT INTO t_max_rows_to_read SELECT number FROM numbers(100); diff --git a/tests/queries/0_stateless/02467_set_with_lowcardinality_type.sql b/tests/queries/0_stateless/02467_set_with_lowcardinality_type.sql index dee6f7de74a..1607d96977b 100644 --- a/tests/queries/0_stateless/02467_set_with_lowcardinality_type.sql +++ b/tests/queries/0_stateless/02467_set_with_lowcardinality_type.sql @@ -6,7 +6,7 @@ CREATE TABLE bloom_filter_nullable_index__fuzz_0 `str` Nullable(String), INDEX idx str TYPE bloom_filter GRANULARITY 1 ) -ENGINE = MergeTree ORDER BY order_key SETTINGS index_granularity = 6; +ENGINE = MergeTree ORDER BY order_key SETTINGS index_granularity = 6, index_granularity_bytes = '10Mi'; INSERT INTO bloom_filter_nullable_index__fuzz_0 VALUES (1, 'test'); INSERT INTO bloom_filter_nullable_index__fuzz_0 VALUES (2, 'test2'); @@ -18,7 +18,7 @@ CREATE TABLE bloom_filter_nullable_index__fuzz_1 `str` String, INDEX idx str TYPE bloom_filter GRANULARITY 1 ) -ENGINE = MergeTree ORDER BY order_key SETTINGS index_granularity = 6; +ENGINE = MergeTree ORDER BY order_key SETTINGS index_granularity = 6, index_granularity_bytes = '10Mi'; INSERT INTO bloom_filter_nullable_index__fuzz_0 VALUES (1, 'test'); INSERT INTO bloom_filter_nullable_index__fuzz_0 VALUES (2, 'test2'); diff --git a/tests/queries/0_stateless/02481_default_value_used_in_row_level_filter.sql b/tests/queries/0_stateless/02481_default_value_used_in_row_level_filter.sql index 6835a3a57ea..ce1662699a1 100644 --- a/tests/queries/0_stateless/02481_default_value_used_in_row_level_filter.sql +++ b/tests/queries/0_stateless/02481_default_value_used_in_row_level_filter.sql @@ -1,6 +1,6 @@ DROP TABLE IF EXISTS test_rlp; -CREATE TABLE test_rlp (a Int32, b Int32) ENGINE=MergeTree() ORDER BY a SETTINGS index_granularity=5; +CREATE TABLE test_rlp (a Int32, b Int32) ENGINE=MergeTree() ORDER BY a SETTINGS index_granularity=5, index_granularity_bytes = '10Mi'; INSERT INTO test_rlp SELECT number, number FROM numbers(15); diff --git a/tests/queries/0_stateless/02481_prewhere_filtered_rows_div_by_zero.reference b/tests/queries/0_stateless/02481_prewhere_filtered_rows_div_by_zero.reference index bb8ce4a8396..f9c1c174a7d 100644 --- a/tests/queries/0_stateless/02481_prewhere_filtered_rows_div_by_zero.reference +++ b/tests/queries/0_stateless/02481_prewhere_filtered_rows_div_by_zero.reference @@ -1,5 +1,5 @@ -- { echoOn } -CREATE TABLE test_filter(a Int32, b Int32, c Int32) ENGINE = MergeTree() ORDER BY a SETTINGS index_granularity = 3; +CREATE TABLE test_filter(a Int32, b Int32, c Int32) ENGINE = MergeTree() ORDER BY a SETTINGS index_granularity = 3, index_granularity_bytes = '10Mi'; INSERT INTO test_filter SELECT number, number+1, (number/2 + 1) % 2 FROM numbers(15); SELECT _part_offset, intDiv(_part_offset, 3) as granule, * FROM test_filter ORDER BY _part_offset; 0 0 0 1 1 diff --git a/tests/queries/0_stateless/02481_prewhere_filtered_rows_div_by_zero.sql b/tests/queries/0_stateless/02481_prewhere_filtered_rows_div_by_zero.sql index 94ffb1b8730..ab675df75cf 100644 --- a/tests/queries/0_stateless/02481_prewhere_filtered_rows_div_by_zero.sql +++ b/tests/queries/0_stateless/02481_prewhere_filtered_rows_div_by_zero.sql @@ -1,7 +1,7 @@ DROP TABLE IF EXISTS test_filter; -- { echoOn } -CREATE TABLE test_filter(a Int32, b Int32, c Int32) ENGINE = MergeTree() ORDER BY a SETTINGS index_granularity = 3; +CREATE TABLE test_filter(a Int32, b Int32, c Int32) ENGINE = MergeTree() ORDER BY a SETTINGS index_granularity = 3, index_granularity_bytes = '10Mi'; INSERT INTO test_filter SELECT number, number+1, (number/2 + 1) % 2 FROM numbers(15); From b1d9ba9da19c2f23f21d01c0b3d998b80936b201 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 21 Feb 2023 01:46:10 +0300 Subject: [PATCH 470/717] Fix #46620 --- docs/en/development/continuous-integration.md | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/docs/en/development/continuous-integration.md b/docs/en/development/continuous-integration.md index 9784c0863b4..232eee5b3cf 100644 --- a/docs/en/development/continuous-integration.md +++ b/docs/en/development/continuous-integration.md @@ -43,11 +43,6 @@ Tries to build the ClickHouse documentation website. It can fail if you changed something in the documentation. Most probable reason is that some cross-link in the documentation is wrong. Go to the check report and look for `ERROR` and `WARNING` messages. -### Report Details - -- [Status page example](https://clickhouse-test-reports.s3.yandex.net/12550/eabcc293eb02214caa6826b7c15f101643f67a6b/docs_check.html) -- `docs_output.txt` contains the building log. [Successful result example](https://clickhouse-test-reports.s3.yandex.net/12550/eabcc293eb02214caa6826b7c15f101643f67a6b/docs_check/docs_output.txt) - ## Description Check @@ -72,10 +67,6 @@ This check means that the CI system started to process the pull request. When it Performs some simple regex-based checks of code style, using the [`utils/check-style/check-style`](https://github.com/ClickHouse/ClickHouse/blob/master/utils/check-style/check-style) binary (note that it can be run locally). If it fails, fix the style errors following the [code style guide](style.md). -### Report Details -- [Status page example](https://clickhouse-test-reports.s3.yandex.net/12550/659c78c7abb56141723af6a81bfae39335aa8cb2/style_check.html) -- `output.txt` contains the check resulting errors (invalid tabulation etc), blank page means no errors. [Successful result example](https://clickhouse-test-reports.s3.yandex.net/12550/659c78c7abb56141723af6a81bfae39335aa8cb2/style_check/output.txt). - ## Fast Test Normally this is the first check that is ran for a PR. It builds ClickHouse and @@ -84,8 +75,6 @@ some. If it fails, further checks are not started until it is fixed. Look at the report to see which tests fail, then reproduce the failure locally as described [here](tests.md#functional-test-locally). -### Report Details -[Status page example](https://clickhouse-test-reports.s3.yandex.net/12550/67d716b5cc3987801996c31a67b31bf141bc3486/fast_test.html) #### Status Page Files - `runlog.out.log` is the general log that includes all other logs. @@ -113,9 +102,7 @@ Builds ClickHouse in various configurations for use in further steps. You have t ### Report Details -[Status page example](https://clickhouse-builds.s3.yandex.net/12550/67d716b5cc3987801996c31a67b31bf141bc3486/clickhouse_build_check/report.html). - -- **Compiler**: `gcc-9` or `clang-10` (or `clang-10-xx` for other architectures e.g. `clang-10-freebsd`). +- **Compiler**: `clang-15`, optionally with the name of a target platform - **Build type**: `Debug` or `RelWithDebInfo` (cmake). - **Sanitizer**: `none` (without sanitizers), `address` (ASan), `memory` (MSan), `undefined` (UBSan), or `thread` (TSan). - **Status**: `success` or `fail` From 4e467d842dda9a7dddd45adc2186f8ff8e1c1ca3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 20 Feb 2023 23:48:46 +0100 Subject: [PATCH 471/717] Inhibit index_granularity_bytes randomization in some tests --- tests/queries/0_stateless/01710_projection_vertical_merges.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01710_projection_vertical_merges.sql b/tests/queries/0_stateless/01710_projection_vertical_merges.sql index d54fef7e71d..e8a4a384017 100644 --- a/tests/queries/0_stateless/01710_projection_vertical_merges.sql +++ b/tests/queries/0_stateless/01710_projection_vertical_merges.sql @@ -2,7 +2,7 @@ drop table if exists t; -create table t (c1 Int64, c2 String, c3 DateTime, c4 Int8, c5 String, c6 String, c7 String, c8 String, c9 String, c10 String, c11 String, c12 String, c13 Int8, c14 Int64, c15 String, c16 String, c17 String, c18 Int64, c19 Int64, c20 Int64) engine MergeTree order by c18; +create table t (c1 Int64, c2 String, c3 DateTime, c4 Int8, c5 String, c6 String, c7 String, c8 String, c9 String, c10 String, c11 String, c12 String, c13 Int8, c14 Int64, c15 String, c16 String, c17 String, c18 Int64, c19 Int64, c20 Int64) engine MergeTree order by c18 SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi'; insert into t (c1, c18) select number, -number from numbers(2000000); From 67d808f49ada57940d257ff9b9a457c1628f4e72 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 21 Feb 2023 00:30:38 +0100 Subject: [PATCH 472/717] Add settings changes history --- src/Core/SettingsChangesHistory.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index 5684e4f3114..48dcded1868 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -83,7 +83,8 @@ static std::map sett {"23.2", {{"output_format_parquet_fixed_string_as_fixed_byte_array", false, true, "Use Parquet FIXED_LENGTH_BYTE_ARRAY type for FixedString by default"}, {"output_format_arrow_fixed_string_as_fixed_byte_array", false, true, "Use Arrow FIXED_SIZE_BINARY type for FixedString by default"}, {"query_plan_remove_redundant_distinct", false, true, "Remove redundant Distinct step in query plan"}, - {"optimize_duplicate_order_by_and_distinct", true, false, "Remove duplicate ORDER BY and DISTINCT if it's possible"}}}, + {"optimize_duplicate_order_by_and_distinct", true, false, "Remove duplicate ORDER BY and DISTINCT if it's possible"}, + {"insert_keeper_max_retries", 0, 20, "Enable reconnections to Keeper on INSERT, improve reliability"}}}, {"23.1", {{"input_format_json_read_objects_as_strings", 0, 1, "Enable reading nested json objects as strings while object type is experimental"}, {"input_format_json_defaults_for_missing_elements_in_named_tuple", false, true, "Allow missing elements in JSON objects while reading named tuples by default"}, {"input_format_csv_detect_header", false, true, "Detect header in CSV format by default"}, From 40d11ce422ef62161463df7abf32619c6fa8e236 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 21 Feb 2023 00:39:32 +0100 Subject: [PATCH 473/717] Adapt a test --- .../test.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/integration/test_keeper_multinode_blocade_leader/test.py b/tests/integration/test_keeper_multinode_blocade_leader/test.py index a7a80d90a58..b11b0a0b64c 100644 --- a/tests/integration/test_keeper_multinode_blocade_leader/test.py +++ b/tests/integration/test_keeper_multinode_blocade_leader/test.py @@ -89,7 +89,7 @@ def test_blocade_leader(started_cluster): print("Got exception from node", smaller_exception(ex)) time.sleep(0.1) - node2.query("INSERT INTO ordinary.t1 SELECT number FROM numbers(10)") + node2.query("INSERT INTO ordinary.t1 SELECT number FROM numbers(10) SETTINGS insert_keeper_max_retries = 0") node1.query("SYSTEM SYNC REPLICA ordinary.t1", timeout=10) node3.query("SYSTEM SYNC REPLICA ordinary.t1", timeout=10) @@ -107,7 +107,7 @@ def test_blocade_leader(started_cluster): restart_replica_for_sure( node2, "ordinary.t1", "/clickhouse/t1/replicas/2" ) - node2.query("INSERT INTO ordinary.t1 SELECT rand() FROM numbers(100)") + node2.query("INSERT INTO ordinary.t1 SELECT rand() FROM numbers(100) SETTINGS insert_keeper_max_retries = 0") break except Exception as ex: try: @@ -128,7 +128,7 @@ def test_blocade_leader(started_cluster): restart_replica_for_sure( node3, "ordinary.t1", "/clickhouse/t1/replicas/3" ) - node3.query("INSERT INTO ordinary.t1 SELECT rand() FROM numbers(100)") + node3.query("INSERT INTO ordinary.t1 SELECT rand() FROM numbers(100) SETTINGS insert_keeper_max_retries = 0") break except Exception as ex: try: @@ -167,7 +167,7 @@ def test_blocade_leader(started_cluster): for i in range(100): try: - node1.query("INSERT INTO ordinary.t1 SELECT rand() FROM numbers(100)") + node1.query("INSERT INTO ordinary.t1 SELECT rand() FROM numbers(100) SETTINGS insert_keeper_max_retries = 0") break except Exception as ex: print("Got exception node1", smaller_exception(ex)) @@ -293,7 +293,7 @@ def test_blocade_leader_twice(started_cluster): print("Got exception from node", smaller_exception(ex)) time.sleep(0.1) - node2.query("INSERT INTO ordinary.t2 SELECT number FROM numbers(10)") + node2.query("INSERT INTO ordinary.t2 SELECT number FROM numbers(10) SETTINGS insert_keeper_max_retries = 0") node1.query("SYSTEM SYNC REPLICA ordinary.t2", timeout=10) node3.query("SYSTEM SYNC REPLICA ordinary.t2", timeout=10) @@ -311,7 +311,7 @@ def test_blocade_leader_twice(started_cluster): restart_replica_for_sure( node2, "ordinary.t2", "/clickhouse/t2/replicas/2" ) - node2.query("INSERT INTO ordinary.t2 SELECT rand() FROM numbers(100)") + node2.query("INSERT INTO ordinary.t2 SELECT rand() FROM numbers(100) SETTINGS insert_keeper_max_retries = 0") break except Exception as ex: try: @@ -333,7 +333,7 @@ def test_blocade_leader_twice(started_cluster): node3, "ordinary.t2", "/clickhouse/t2/replicas/3" ) node3.query("SYSTEM SYNC REPLICA ordinary.t2", timeout=10) - node3.query("INSERT INTO ordinary.t2 SELECT rand() FROM numbers(100)") + node3.query("INSERT INTO ordinary.t2 SELECT rand() FROM numbers(100) SETTINGS insert_keeper_max_retries = 0") break except Exception as ex: try: @@ -359,14 +359,14 @@ def test_blocade_leader_twice(started_cluster): for i in range(10): try: - node3.query("INSERT INTO ordinary.t2 SELECT rand() FROM numbers(100)") + node3.query("INSERT INTO ordinary.t2 SELECT rand() FROM numbers(100) SETTINGS insert_keeper_max_retries = 0") assert False, "Node3 became leader?" except Exception as ex: time.sleep(0.5) for i in range(10): try: - node2.query("INSERT INTO ordinary.t2 SELECT rand() FROM numbers(100)") + node2.query("INSERT INTO ordinary.t2 SELECT rand() FROM numbers(100) SETTINGS insert_keeper_max_retries = 0") assert False, "Node2 became leader?" except Exception as ex: time.sleep(0.5) @@ -399,7 +399,7 @@ def test_blocade_leader_twice(started_cluster): for n, node in enumerate([node1, node2, node3]): for i in range(100): try: - node.query("INSERT INTO ordinary.t2 SELECT rand() FROM numbers(100)") + node.query("INSERT INTO ordinary.t2 SELECT rand() FROM numbers(100) SETTINGS insert_keeper_max_retries = 0") break except Exception as ex: print("Got exception node{}".format(n + 1), smaller_exception(ex)) From dc4b52e54ebc61b378e78f8f3377a6ccb94b7a7d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 21 Feb 2023 00:41:40 +0100 Subject: [PATCH 474/717] Adapt a test --- tests/integration/test_storage_kafka/configs/users.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/test_storage_kafka/configs/users.xml b/tests/integration/test_storage_kafka/configs/users.xml index 2cef0a6de3c..0f2f6d7c424 100644 --- a/tests/integration/test_storage_kafka/configs/users.xml +++ b/tests/integration/test_storage_kafka/configs/users.xml @@ -2,6 +2,8 @@ 1 + + 0 From 4a2bd6e0aaf55e5fc35840c0c865d83c27fcc375 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 21 Feb 2023 00:43:10 +0100 Subject: [PATCH 475/717] Adapt a test --- tests/integration/test_storage_kafka/configs/users.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_storage_kafka/configs/users.xml b/tests/integration/test_storage_kafka/configs/users.xml index 0f2f6d7c424..992464a0ac2 100644 --- a/tests/integration/test_storage_kafka/configs/users.xml +++ b/tests/integration/test_storage_kafka/configs/users.xml @@ -3,7 +3,7 @@ 1 - 0 + 0 From d0d63061024b83f0123b962a376c7551d64cf3d8 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 21 Feb 2023 00:46:27 +0100 Subject: [PATCH 476/717] Fix coverity --- .github/workflows/nightly.yml | 2 +- docker/packager/binary/build.sh | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index b4ef794c4ca..27c4f5811da 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -107,7 +107,7 @@ jobs: run: | curl --form token="${COVERITY_TOKEN}" \ --form email='security+coverity@clickhouse.com' \ - --form file="@$TEMP_PATH/$BUILD_NAME/coverity-scan.tar.zst" \ + --form file="@$TEMP_PATH/$BUILD_NAME/coverity-scan.tar.gz" \ --form version="${GITHUB_REF#refs/heads/}-${GITHUB_SHA::6}" \ --form description="Nighly Scan: $(date +'%Y-%m-%dT%H:%M:%S')" \ https://scan.coverity.com/builds?project=ClickHouse%2FClickHouse diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh index 7499aceae2e..24dca72e946 100755 --- a/docker/packager/binary/build.sh +++ b/docker/packager/binary/build.sh @@ -174,8 +174,9 @@ fi if [ "coverity" == "$COMBINED_OUTPUT" ] then - tar -cv --zstd -f "coverity-scan.tar.zst" cov-int - mv "coverity-scan.tar.zst" /output + # Coverity does not understand ZSTD. + tar -cvz -f "coverity-scan.tar.gz" cov-int + mv "coverity-scan.tar.gz" /output fi ccache_status From abeb1c5ba8834cec406dc42a3c287f37a9b792af Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 20 Feb 2023 23:49:25 +0000 Subject: [PATCH 477/717] Automatic style fix --- .../test.py | 40 ++++++++++++++----- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/tests/integration/test_keeper_multinode_blocade_leader/test.py b/tests/integration/test_keeper_multinode_blocade_leader/test.py index b11b0a0b64c..3af0751b0fd 100644 --- a/tests/integration/test_keeper_multinode_blocade_leader/test.py +++ b/tests/integration/test_keeper_multinode_blocade_leader/test.py @@ -89,7 +89,9 @@ def test_blocade_leader(started_cluster): print("Got exception from node", smaller_exception(ex)) time.sleep(0.1) - node2.query("INSERT INTO ordinary.t1 SELECT number FROM numbers(10) SETTINGS insert_keeper_max_retries = 0") + node2.query( + "INSERT INTO ordinary.t1 SELECT number FROM numbers(10) SETTINGS insert_keeper_max_retries = 0" + ) node1.query("SYSTEM SYNC REPLICA ordinary.t1", timeout=10) node3.query("SYSTEM SYNC REPLICA ordinary.t1", timeout=10) @@ -107,7 +109,9 @@ def test_blocade_leader(started_cluster): restart_replica_for_sure( node2, "ordinary.t1", "/clickhouse/t1/replicas/2" ) - node2.query("INSERT INTO ordinary.t1 SELECT rand() FROM numbers(100) SETTINGS insert_keeper_max_retries = 0") + node2.query( + "INSERT INTO ordinary.t1 SELECT rand() FROM numbers(100) SETTINGS insert_keeper_max_retries = 0" + ) break except Exception as ex: try: @@ -128,7 +132,9 @@ def test_blocade_leader(started_cluster): restart_replica_for_sure( node3, "ordinary.t1", "/clickhouse/t1/replicas/3" ) - node3.query("INSERT INTO ordinary.t1 SELECT rand() FROM numbers(100) SETTINGS insert_keeper_max_retries = 0") + node3.query( + "INSERT INTO ordinary.t1 SELECT rand() FROM numbers(100) SETTINGS insert_keeper_max_retries = 0" + ) break except Exception as ex: try: @@ -167,7 +173,9 @@ def test_blocade_leader(started_cluster): for i in range(100): try: - node1.query("INSERT INTO ordinary.t1 SELECT rand() FROM numbers(100) SETTINGS insert_keeper_max_retries = 0") + node1.query( + "INSERT INTO ordinary.t1 SELECT rand() FROM numbers(100) SETTINGS insert_keeper_max_retries = 0" + ) break except Exception as ex: print("Got exception node1", smaller_exception(ex)) @@ -293,7 +301,9 @@ def test_blocade_leader_twice(started_cluster): print("Got exception from node", smaller_exception(ex)) time.sleep(0.1) - node2.query("INSERT INTO ordinary.t2 SELECT number FROM numbers(10) SETTINGS insert_keeper_max_retries = 0") + node2.query( + "INSERT INTO ordinary.t2 SELECT number FROM numbers(10) SETTINGS insert_keeper_max_retries = 0" + ) node1.query("SYSTEM SYNC REPLICA ordinary.t2", timeout=10) node3.query("SYSTEM SYNC REPLICA ordinary.t2", timeout=10) @@ -311,7 +321,9 @@ def test_blocade_leader_twice(started_cluster): restart_replica_for_sure( node2, "ordinary.t2", "/clickhouse/t2/replicas/2" ) - node2.query("INSERT INTO ordinary.t2 SELECT rand() FROM numbers(100) SETTINGS insert_keeper_max_retries = 0") + node2.query( + "INSERT INTO ordinary.t2 SELECT rand() FROM numbers(100) SETTINGS insert_keeper_max_retries = 0" + ) break except Exception as ex: try: @@ -333,7 +345,9 @@ def test_blocade_leader_twice(started_cluster): node3, "ordinary.t2", "/clickhouse/t2/replicas/3" ) node3.query("SYSTEM SYNC REPLICA ordinary.t2", timeout=10) - node3.query("INSERT INTO ordinary.t2 SELECT rand() FROM numbers(100) SETTINGS insert_keeper_max_retries = 0") + node3.query( + "INSERT INTO ordinary.t2 SELECT rand() FROM numbers(100) SETTINGS insert_keeper_max_retries = 0" + ) break except Exception as ex: try: @@ -359,14 +373,18 @@ def test_blocade_leader_twice(started_cluster): for i in range(10): try: - node3.query("INSERT INTO ordinary.t2 SELECT rand() FROM numbers(100) SETTINGS insert_keeper_max_retries = 0") + node3.query( + "INSERT INTO ordinary.t2 SELECT rand() FROM numbers(100) SETTINGS insert_keeper_max_retries = 0" + ) assert False, "Node3 became leader?" except Exception as ex: time.sleep(0.5) for i in range(10): try: - node2.query("INSERT INTO ordinary.t2 SELECT rand() FROM numbers(100) SETTINGS insert_keeper_max_retries = 0") + node2.query( + "INSERT INTO ordinary.t2 SELECT rand() FROM numbers(100) SETTINGS insert_keeper_max_retries = 0" + ) assert False, "Node2 became leader?" except Exception as ex: time.sleep(0.5) @@ -399,7 +417,9 @@ def test_blocade_leader_twice(started_cluster): for n, node in enumerate([node1, node2, node3]): for i in range(100): try: - node.query("INSERT INTO ordinary.t2 SELECT rand() FROM numbers(100) SETTINGS insert_keeper_max_retries = 0") + node.query( + "INSERT INTO ordinary.t2 SELECT rand() FROM numbers(100) SETTINGS insert_keeper_max_retries = 0" + ) break except Exception as ex: print("Got exception node{}".format(n + 1), smaller_exception(ex)) From c0b50c981abad16b2c344c8eb7a5c5d974916f6f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 21 Feb 2023 02:53:36 +0300 Subject: [PATCH 478/717] Installation instructions in README --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index dee60cafb33..5b5dc4ae528 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,11 @@ ClickHouse® is an open-source column-oriented database management system that allows generating analytical data reports in real-time. +## How To Install +``` +curl https://clickhouse.com/ | sh +``` + ## Useful Links * [Official website](https://clickhouse.com/) has a quick high-level overview of ClickHouse on the main page. From 16a9ac9118cfd8eb8ed579fd12becb2eff8858f0 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Tue, 21 Feb 2023 00:51:39 +0000 Subject: [PATCH 479/717] Try to add test --- .../test_async_insert_memory/__init__.py | 0 .../test_async_insert_memory/test.py | 35 +++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 tests/integration/test_async_insert_memory/__init__.py create mode 100644 tests/integration/test_async_insert_memory/test.py diff --git a/tests/integration/test_async_insert_memory/__init__.py b/tests/integration/test_async_insert_memory/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_async_insert_memory/test.py b/tests/integration/test_async_insert_memory/test.py new file mode 100644 index 00000000000..cf7b59c8d4a --- /dev/null +++ b/tests/integration/test_async_insert_memory/test.py @@ -0,0 +1,35 @@ +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +node = cluster.add_instance("node") + + +@pytest.fixture(scope="module", autouse=True) +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def test_memory_usage(): + node.query( + "CREATE TABLE async_table(data Array(UInt64)) ENGINE=MergeTree() ORDER BY data" + ) + + response = node.get_query_request( + "SELECT groupArray(number + sleepEachRow(0.0001)) FROM numbers(1000000) SETTINGS max_memory_usage_for_user={}".format(30 * (2 ** 23)) + ) + + INSERT_QUERY = "INSERT INTO async_table SETTINGS async_insert=1, wait_for_async_insert=1 VALUES ({})" + for i in range(10): + node.query(INSERT_QUERY.format([i in range(i * 5000000, (i + 1) * 5000000)])) + + _, err = response.get_answer_and_error() + assert err == "", "Query failed" + + node.query("DROP TABLE async_table") From 573ce5040315ebd8dd82ef90796498280052dc57 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Tue, 21 Feb 2023 01:43:01 +0000 Subject: [PATCH 480/717] Fix test --- .../integration/test_async_insert_memory/test.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/integration/test_async_insert_memory/test.py b/tests/integration/test_async_insert_memory/test.py index cf7b59c8d4a..93e3e771c71 100644 --- a/tests/integration/test_async_insert_memory/test.py +++ b/tests/integration/test_async_insert_memory/test.py @@ -21,15 +21,20 @@ def test_memory_usage(): "CREATE TABLE async_table(data Array(UInt64)) ENGINE=MergeTree() ORDER BY data" ) - response = node.get_query_request( - "SELECT groupArray(number + sleepEachRow(0.0001)) FROM numbers(1000000) SETTINGS max_memory_usage_for_user={}".format(30 * (2 ** 23)) + node.get_query_request( + "SELECT count() FROM system.numbers" ) INSERT_QUERY = "INSERT INTO async_table SETTINGS async_insert=1, wait_for_async_insert=1 VALUES ({})" - for i in range(10): - node.query(INSERT_QUERY.format([i in range(i * 5000000, (i + 1) * 5000000)])) + for iter in range(10): + values = list(range(iter * 5000000, (iter + 1) * 5000000)) + node.query(INSERT_QUERY.format(values)) + + response = node.get_query_request( + "SELECT groupArray(number) FROM numbers(1000000) SETTINGS max_memory_usage_for_user={}".format(30 * (2 ** 23)) + ) _, err = response.get_answer_and_error() - assert err == "", "Query failed" + assert err == "", "Query failed with error {}".format(err) node.query("DROP TABLE async_table") From 93aabf8c66006d892583d4a3f5478897b166ce4c Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 21 Feb 2023 02:06:16 +0000 Subject: [PATCH 481/717] Automatic style fix --- tests/integration/test_async_insert_memory/test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_async_insert_memory/test.py b/tests/integration/test_async_insert_memory/test.py index 93e3e771c71..279542f087c 100644 --- a/tests/integration/test_async_insert_memory/test.py +++ b/tests/integration/test_async_insert_memory/test.py @@ -21,9 +21,7 @@ def test_memory_usage(): "CREATE TABLE async_table(data Array(UInt64)) ENGINE=MergeTree() ORDER BY data" ) - node.get_query_request( - "SELECT count() FROM system.numbers" - ) + node.get_query_request("SELECT count() FROM system.numbers") INSERT_QUERY = "INSERT INTO async_table SETTINGS async_insert=1, wait_for_async_insert=1 VALUES ({})" for iter in range(10): @@ -31,7 +29,9 @@ def test_memory_usage(): node.query(INSERT_QUERY.format(values)) response = node.get_query_request( - "SELECT groupArray(number) FROM numbers(1000000) SETTINGS max_memory_usage_for_user={}".format(30 * (2 ** 23)) + "SELECT groupArray(number) FROM numbers(1000000) SETTINGS max_memory_usage_for_user={}".format( + 30 * (2**23) + ) ) _, err = response.get_answer_and_error() From fcd3e442971c427b6c7557013f0481668d811699 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Tue, 21 Feb 2023 10:55:13 +0800 Subject: [PATCH 482/717] finish dev --- src/Functions/JSONArrayLength.cpp | 110 ++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 src/Functions/JSONArrayLength.cpp diff --git a/src/Functions/JSONArrayLength.cpp b/src/Functions/JSONArrayLength.cpp new file mode 100644 index 00000000000..4f784ce140d --- /dev/null +++ b/src/Functions/JSONArrayLength.cpp @@ -0,0 +1,110 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "config.h" + +#if USE_SIMDJSON +# include +#elif USE_RAPIDJSON +# include +#else +# include +#endif + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ILLEGAL_COLUMN; +} + +namespace +{ + /// JSONArrayLength(json) + class FunctionJSONArrayLength : public IFunction + { + public: + static constexpr auto name = "JSONArrayLength"; + static FunctionPtr create(ContextPtr) { return std::make_shared(); } + + String getName() const override { return name; } + + bool isVariadic() const override { return false; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } + + size_t getNumberOfArguments() const override { return 1; } + bool useDefaultImplementationForConstants() const override { return true; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + auto args = FunctionArgumentDescriptors{ + {"json", &isString, nullptr, "String"}, + }; + + validateFunctionArgumentTypes(*this, arguments, args); + return std::make_shared(std::make_shared()); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const ColumnPtr column = arguments[0].column; + const ColumnString * col = typeid_cast(column.get()); + if (!col) + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "First argument of function {} must be string", getName()); + + auto null_map = ColumnUInt8::create(); + auto data = ColumnUInt64::create(); + null_map->reserve(input_rows_count); + data->reserve(input_rows_count); + +#if USE_SIMDJSON + SimdJSONParser parser; + SimdJSONParser::Element element; +#elif USE_RAPIDJSON + RapidJSONParser parser; + RapidJSONParser::Element element; +#else + DummyJSONParser parser; + DummyJSONParser::Element element; +#endif + + for (size_t i = 0; i < input_rows_count; ++i) + { + auto str_ref = col->getDataAt(i); + std::string_view str_view(str_ref.data, str_ref.size); + bool ok = parser.parse(std::move(str_view), element); + if (!ok || !element.isArray()) + { + null_map->insertValue(1); + data->insertDefault(); + } + else + { + auto array = element.getArray(); + null_map->insertValue(0); + data->insertValue(array.size()); + } + } + return ColumnNullable::create(std::move(data), std::move(null_map)); + } + }; + +} + +REGISTER_FUNCTION(JSONArrayLength) +{ + factory.registerFunction(Documentation{ + "Returns the number of elements in the outermost JSON array. The function returns NULL if input JSON string is invalid."}); + + /// For Spark compatibility. + factory.registerAlias("JSON_ARRAY_LENGTH", "JSONArrayLength", FunctionFactory::CaseInsensitive); +} + +} From 6b5a102a5b12835208007a7f7824dc39dd26dba6 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Tue, 21 Feb 2023 11:08:13 +0800 Subject: [PATCH 483/717] add tests --- .../02667_json_array_length.reference | 21 +++++++++++++++++++ .../0_stateless/02667_json_array_length.sql | 13 ++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 tests/queries/0_stateless/02667_json_array_length.reference create mode 100644 tests/queries/0_stateless/02667_json_array_length.sql diff --git a/tests/queries/0_stateless/02667_json_array_length.reference b/tests/queries/0_stateless/02667_json_array_length.reference new file mode 100644 index 00000000000..b86f094d0fb --- /dev/null +++ b/tests/queries/0_stateless/02667_json_array_length.reference @@ -0,0 +1,21 @@ +-- { echoOn } +select json_array_length(null); +\N +select json_array_length(''); +\N +select json_array_length('[]'); +0 +select json_array_length('[1,2,3]'); +3 +select json_array_length('[[1,2],[5,6,7]]'); +2 +select json_array_length('[{"a":123},{"b":"hello"}]'); +2 +select json_array_length('[1,2,3,[33,44],{"key":[2,3,4]}]'); +5 +select json_array_length('{"key":"not a json array"}'); +\N +select json_array_length('[1,2,3,4,5'); +\N +select json_array_length(2); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +select json_array_length(); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } diff --git a/tests/queries/0_stateless/02667_json_array_length.sql b/tests/queries/0_stateless/02667_json_array_length.sql new file mode 100644 index 00000000000..7ea3dffbca6 --- /dev/null +++ b/tests/queries/0_stateless/02667_json_array_length.sql @@ -0,0 +1,13 @@ +-- { echoOn } +select json_array_length(null); +select json_array_length(''); +select json_array_length('[]'); +select json_array_length('[1,2,3]'); +select json_array_length('[[1,2],[5,6,7]]'); +select json_array_length('[{"a":123},{"b":"hello"}]'); +select json_array_length('[1,2,3,[33,44],{"key":[2,3,4]}]'); +select json_array_length('{"key":"not a json array"}'); +select json_array_length('[1,2,3,4,5'); + +select json_array_length(2); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +select json_array_length(); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } From eafac0f9a35937cb8715bc8d54809721d2f8a967 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Tue, 21 Feb 2023 11:17:44 +0800 Subject: [PATCH 484/717] add docs --- .../sql-reference/functions/json-functions.md | 35 +++++++++++++++++++ .../0_stateless/02667_json_array_length.sql | 25 +++++++------ 2 files changed, 49 insertions(+), 11 deletions(-) diff --git a/docs/en/sql-reference/functions/json-functions.md b/docs/en/sql-reference/functions/json-functions.md index ddea4b5c396..bfe2a541647 100644 --- a/docs/en/sql-reference/functions/json-functions.md +++ b/docs/en/sql-reference/functions/json-functions.md @@ -471,3 +471,38 @@ Result: - [output_format_json_quote_64bit_integers](../../operations/settings/settings.md#session_settings-output_format_json_quote_64bit_integers) - [output_format_json_quote_denormals](../../operations/settings/settings.md#settings-output_format_json_quote_denormals) + + +## JSONArrayLength + +Returns the number of elements in the outermost JSON array. The function returns NULL if input JSON string is invalid. + +**Syntax** + +``` sql +JSONArrayLength(json) +``` + +Alias: `JSON_ARRAY_LENGTH(json)`. + +**Arguments** + +- `json` — [String](../../sql-reference/data-types/string.md) with valid JSON. + +**Returned value** + +- If `json` is a valid JSON array string, returns the number of array elements, otherwise returns NULL. + +Type: [Nullable(UInt64)](../../sql-reference/data-types/int-uint.md). + +**Example** + +``` sql +SELECT + JSONArrayLength(''), + JSONArrayLength('[1,2,3]') + +┌─JSONArrayLength('')─┬─JSONArrayLength('[1,2,3]')─┐ +│ ᴺᵁᴸᴸ │ 3 │ +└─────────────────────┴────────────────────────────┘ +``` diff --git a/tests/queries/0_stateless/02667_json_array_length.sql b/tests/queries/0_stateless/02667_json_array_length.sql index 7ea3dffbca6..4f2127b9c81 100644 --- a/tests/queries/0_stateless/02667_json_array_length.sql +++ b/tests/queries/0_stateless/02667_json_array_length.sql @@ -1,13 +1,16 @@ -- { echoOn } -select json_array_length(null); -select json_array_length(''); -select json_array_length('[]'); -select json_array_length('[1,2,3]'); -select json_array_length('[[1,2],[5,6,7]]'); -select json_array_length('[{"a":123},{"b":"hello"}]'); -select json_array_length('[1,2,3,[33,44],{"key":[2,3,4]}]'); -select json_array_length('{"key":"not a json array"}'); -select json_array_length('[1,2,3,4,5'); +select JSONArrayLength(null); +select JSONArrayLength(''); +select JSONArrayLength('[]'); +select JSONArrayLength('[1,2,3]'); +select JSONArrayLength('[[1,2],[5,6,7]]'); +select JSONArrayLength('[{"a":123},{"b":"hello"}]'); +select JSONArrayLength('[1,2,3,[33,44],{"key":[2,3,4]}]'); +select JSONArrayLength('{"key":"not a json array"}'); +select JSONArrayLength('[1,2,3,4,5'); -select json_array_length(2); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } -select json_array_length(); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +select JSON_ARRAY_LENGTH('[1,2,3,4,5'); +select JSON_ARRAY_LENGTH('[1,2,3,4,5]'); + +select JSONArrayLength(2); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +select JSONArrayLength(); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } From 3eaad70b16a0e442e1e2e4d8dc8cf495d12ac715 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Tue, 21 Feb 2023 12:31:02 +0800 Subject: [PATCH 485/717] fix fast test --- .../02667_json_array_length.reference | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/tests/queries/0_stateless/02667_json_array_length.reference b/tests/queries/0_stateless/02667_json_array_length.reference index b86f094d0fb..dbe66902f35 100644 --- a/tests/queries/0_stateless/02667_json_array_length.reference +++ b/tests/queries/0_stateless/02667_json_array_length.reference @@ -1,21 +1,25 @@ -- { echoOn } -select json_array_length(null); +select JSONArrayLength(null); \N -select json_array_length(''); +select JSONArrayLength(''); \N -select json_array_length('[]'); +select JSONArrayLength('[]'); 0 -select json_array_length('[1,2,3]'); +select JSONArrayLength('[1,2,3]'); 3 -select json_array_length('[[1,2],[5,6,7]]'); +select JSONArrayLength('[[1,2],[5,6,7]]'); 2 -select json_array_length('[{"a":123},{"b":"hello"}]'); +select JSONArrayLength('[{"a":123},{"b":"hello"}]'); 2 -select json_array_length('[1,2,3,[33,44],{"key":[2,3,4]}]'); +select JSONArrayLength('[1,2,3,[33,44],{"key":[2,3,4]}]'); 5 -select json_array_length('{"key":"not a json array"}'); +select JSONArrayLength('{"key":"not a json array"}'); \N -select json_array_length('[1,2,3,4,5'); +select JSONArrayLength('[1,2,3,4,5'); \N -select json_array_length(2); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } -select json_array_length(); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +select JSON_ARRAY_LENGTH('[1,2,3,4,5'); +\N +select JSON_ARRAY_LENGTH('[1,2,3,4,5]'); +5 +select JSONArrayLength(2); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +select JSONArrayLength(); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } From 2945125a16a63d07f91c2ff55e057cf273240210 Mon Sep 17 00:00:00 2001 From: Jus <40656180+jus1096@users.noreply.github.com> Date: Tue, 21 Feb 2023 10:00:47 +0400 Subject: [PATCH 486/717] Update docs/ru/sql-reference/functions/other-functions.md Co-authored-by: Nikolay Degterinsky <43110995+evillique@users.noreply.github.com> --- docs/ru/sql-reference/functions/other-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/functions/other-functions.md b/docs/ru/sql-reference/functions/other-functions.md index 6e0c4ba146d..f457b54ae28 100644 --- a/docs/ru/sql-reference/functions/other-functions.md +++ b/docs/ru/sql-reference/functions/other-functions.md @@ -672,7 +672,7 @@ formatReadableTimeDelta(column[, maximum_unit]) **Аргументы** - `column` — Столбец с числовой дельтой времени. -- `maximum_unit` — Опицонально. Максимальная единица измерения для отображения. Допустимые значения: секунды, минуты, часы, дни, месяцы, годы. +- `maximum_unit` — Опциональный параметр. Максимальная единица измерения для отображения. Допустимые значения: секунды, минуты, часы, дни, месяцы, годы. Пример: From d905969006bd0619fc319dab638167185c574fdb Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 20 Feb 2023 16:14:35 +0100 Subject: [PATCH 487/717] Fix 01565_reconnect_after_client_error test (wrong expect code for multi-line mode) Signed-off-by: Azat Khuzhin --- .../01565_reconnect_after_client_error.expect | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/queries/0_stateless/01565_reconnect_after_client_error.expect b/tests/queries/0_stateless/01565_reconnect_after_client_error.expect index 255248ba61a..4e710fa8262 100755 --- a/tests/queries/0_stateless/01565_reconnect_after_client_error.expect +++ b/tests/queries/0_stateless/01565_reconnect_after_client_error.expect @@ -24,18 +24,27 @@ spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \ expect "\n:) " send -- "DROP TABLE IF EXISTS t01565;\n" +# NOTE: this is important for -mn mode, you should send "\r" only after reading echoed command +expect "DROP" +send -- "\r" expect "\nOk." expect "\n:)" send -- "CREATE TABLE t01565 (c0 String, c1 Int32) ENGINE = Memory() ;\n" +expect "CREATE" +send -- "\r" expect "\nOk." expect "\n:) " send -- "INSERT INTO t01565(c0, c1) VALUES (\"1\",1) ;\n" +expect "INSERT" +send -- "\r" expect "\nConnected" expect "\n:) " send -- "INSERT INTO t01565(c0, c1) VALUES ('1', 1) ;\n" +expect "INSERT" +send -- "\r" expect "\nOk." expect "\n:) " From 2a0cebfc493ae521cc88c06ca235bc8a0c9b6433 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 20 Feb 2023 19:27:11 +0100 Subject: [PATCH 488/717] Fix timeouts in 01565_reconnect_after_client_error Signed-off-by: Azat Khuzhin --- .../0_stateless/01565_reconnect_after_client_error.expect | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01565_reconnect_after_client_error.expect b/tests/queries/0_stateless/01565_reconnect_after_client_error.expect index 4e710fa8262..143e94ae1f3 100755 --- a/tests/queries/0_stateless/01565_reconnect_after_client_error.expect +++ b/tests/queries/0_stateless/01565_reconnect_after_client_error.expect @@ -15,9 +15,9 @@ match_max 100000 expect_after { # Do not ignore eof from expect - eof { exp_continue } + -i $any_spawn_id eof { exp_continue } # A default timeout action is to do nothing, change it to fail - timeout { exit 1 } + -i $any_spawn_id timeout { exit 1 } } spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion -mn --history_file=$history_file" From e169b36b8891840595085593bd77ef8bc885bdb3 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 21 Feb 2023 08:22:23 +0100 Subject: [PATCH 489/717] Replace 01565_reconnect_after_client_error with 01565_query_loop_after_client_error There is no need in explicit reconnect in case of client errors (#19353), so just rewrite the test to ensure that everything works. Signed-off-by: Azat Khuzhin --- ...t_error.expect => 01565_query_loop_after_client_error.expect} | 1 - ...r.reference => 01565_query_loop_after_client_error.reference} | 0 2 files changed, 1 deletion(-) rename tests/queries/0_stateless/{01565_reconnect_after_client_error.expect => 01565_query_loop_after_client_error.expect} (98%) rename tests/queries/0_stateless/{01565_reconnect_after_client_error.reference => 01565_query_loop_after_client_error.reference} (100%) diff --git a/tests/queries/0_stateless/01565_reconnect_after_client_error.expect b/tests/queries/0_stateless/01565_query_loop_after_client_error.expect similarity index 98% rename from tests/queries/0_stateless/01565_reconnect_after_client_error.expect rename to tests/queries/0_stateless/01565_query_loop_after_client_error.expect index 143e94ae1f3..bf701225605 100755 --- a/tests/queries/0_stateless/01565_reconnect_after_client_error.expect +++ b/tests/queries/0_stateless/01565_query_loop_after_client_error.expect @@ -39,7 +39,6 @@ expect "\n:) " send -- "INSERT INTO t01565(c0, c1) VALUES (\"1\",1) ;\n" expect "INSERT" send -- "\r" -expect "\nConnected" expect "\n:) " send -- "INSERT INTO t01565(c0, c1) VALUES ('1', 1) ;\n" diff --git a/tests/queries/0_stateless/01565_reconnect_after_client_error.reference b/tests/queries/0_stateless/01565_query_loop_after_client_error.reference similarity index 100% rename from tests/queries/0_stateless/01565_reconnect_after_client_error.reference rename to tests/queries/0_stateless/01565_query_loop_after_client_error.reference From 367cf9e77cefccbf40b9489ac59c3e64d6449a5b Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 20 Feb 2023 16:37:59 +0100 Subject: [PATCH 490/717] Fix 01179_insert_values_semicolon test Back in #19925 a check for reading data after semicolon had been added, but after #40474 it does not work, the test does not show the problem because of timeout does not work without stdin before (a more generic fix for timeouts in expect tests I will submit later). To make this test works, the only type that I can found that will work right now is DateTime64, other types does use peeking, or even if they do, they will fail while parsing the query as SQL expression. Signed-off-by: Azat Khuzhin --- .../0_stateless/01179_insert_values_semicolon.expect | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/01179_insert_values_semicolon.expect b/tests/queries/0_stateless/01179_insert_values_semicolon.expect index 9d35941ae40..c0b67de5302 100755 --- a/tests/queries/0_stateless/01179_insert_values_semicolon.expect +++ b/tests/queries/0_stateless/01179_insert_values_semicolon.expect @@ -21,7 +21,7 @@ expect ":) " send -- "DROP TABLE IF EXISTS test_01179\r" expect "Ok." -send -- "CREATE TABLE test_01179 (date DateTime) ENGINE=Memory()\r" +send -- "CREATE TABLE test_01179 (date DateTime64(3)) ENGINE=Memory()\r" expect "Ok." send -- "INSERT INTO test_01179 values ('2020-01-01')\r" @@ -30,11 +30,11 @@ expect "Ok." send -- "INSERT INTO test_01179 values ('2020-01-01'); \r" expect "Ok." -send -- "INSERT INTO test_01179 values ('2020-01-01'); (1) \r" +send -- "INSERT INTO test_01179 values ('2020-01-01 0'); (1) \r" expect "Cannot read data after semicolon" send -- "SELECT date, count() FROM test_01179 GROUP BY date FORMAT TSV\r" -expect "2020-01-01 00:00:00\t3" +expect "2020-01-01 00:00:00.000\t2" send -- "DROP TABLE test_01179\r" expect "Ok." From 5cc183ac39836aca060590eb96a572e46b653058 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 20 Feb 2023 19:27:09 +0100 Subject: [PATCH 491/717] Fix timeouts in 01179_insert_values_semicolon Signed-off-by: Azat Khuzhin --- .../queries/0_stateless/01179_insert_values_semicolon.expect | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01179_insert_values_semicolon.expect b/tests/queries/0_stateless/01179_insert_values_semicolon.expect index c0b67de5302..16c62443856 100755 --- a/tests/queries/0_stateless/01179_insert_values_semicolon.expect +++ b/tests/queries/0_stateless/01179_insert_values_semicolon.expect @@ -10,9 +10,9 @@ set timeout 60 match_max 100000 expect_after { # Do not ignore eof from expect - eof { exp_continue } + -i $any_spawn_id eof { exp_continue } # A default timeout action is to do nothing, change it to fail - timeout { exit 1 } + -i $any_spawn_id timeout { exit 1 } } spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion" From 19d0c929d66d53b6f1ded3fdc7d3b67d7f02511c Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 20 Feb 2023 17:26:49 +0100 Subject: [PATCH 492/717] Disable timeout logic for starting clickhouse-server from systemd service After ClickHouse became systemd aware (#43400), it waits not more then TimeoutStartSec (1m30sec by default), while before it simply ensures that the process is there. And likely 1m30sec can be not enough for some cluster, and this will lead to endless restarts. At first I've increased it to 10min, but there was a comment about that this is not enough, and I agree with this. But I'm not sure that using "inifinity" is a good option, but I cannot think of any downsides of this. Signed-off-by: Azat Khuzhin --- packages/clickhouse-server.service | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/clickhouse-server.service b/packages/clickhouse-server.service index ace304e0c5e..037be826b97 100644 --- a/packages/clickhouse-server.service +++ b/packages/clickhouse-server.service @@ -17,6 +17,8 @@ User=clickhouse Group=clickhouse Restart=always RestartSec=30 +# Since ClickHouse is systemd aware default 1m30sec may not be enough +TimeoutStartSec=inifinity # %p is resolved to the systemd unit name RuntimeDirectory=%p ExecStart=/usr/bin/clickhouse-server --config=/etc/clickhouse-server/config.xml --pid-file=%t/%p/%p.pid From 593de3416bc7ef61fcbe5d103a65c4a2c614dc6d Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Tue, 21 Feb 2023 12:27:08 +0100 Subject: [PATCH 493/717] Fix typo in read prefetch --- src/Processors/QueryPlan/ReadFromMergeTree.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index f638bd803f9..f90506b95a2 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -350,7 +350,7 @@ Pipe ReadFromMergeTree::readFromPool( && MergeTreePrefetchedReadPool::checkReadMethodAllowed(reader_settings.read_settings.remote_fs_method)) || (!all_parts_are_local && settings.allow_prefetched_read_pool_for_local_filesystem - && MergeTreePrefetchedReadPool::checkReadMethodAllowed(reader_settings.read_settings.remote_fs_method))) + && MergeTreePrefetchedReadPool::checkReadMethodAllowed(reader_settings.read_settings.local_fs_method))) { pool = std::make_shared( max_streams, From 07e96aafdac867e573184a1c0a73d7cdbd8f38c8 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Mon, 20 Feb 2023 18:42:47 +0100 Subject: [PATCH 494/717] Test progress bar while filtering all rows in PREWHERE --- ...gress_when_no_rows_from_prewhere.reference | 0 ...666_progress_when_no_rows_from_prewhere.sh | 23 +++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 tests/queries/0_stateless/02666_progress_when_no_rows_from_prewhere.reference create mode 100755 tests/queries/0_stateless/02666_progress_when_no_rows_from_prewhere.sh diff --git a/tests/queries/0_stateless/02666_progress_when_no_rows_from_prewhere.reference b/tests/queries/0_stateless/02666_progress_when_no_rows_from_prewhere.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02666_progress_when_no_rows_from_prewhere.sh b/tests/queries/0_stateless/02666_progress_when_no_rows_from_prewhere.sh new file mode 100755 index 00000000000..a25d72f357d --- /dev/null +++ b/tests/queries/0_stateless/02666_progress_when_no_rows_from_prewhere.sh @@ -0,0 +1,23 @@ +#!/usr/bin/expect -f +# Tags: no-tsan, no-asan, no-ubsan, no-msan, no-debug, no-fasttest + +log_user 0 +set timeout 60 +match_max 10000000 + +# Run query that filters all rows in PREWHERE +spawn clickhouse-local --progress -m -n --query "CREATE TABLE test_progress(n UInt64) ENGINE=MergeTree ORDER BY tuple() SETTINGS index_granularity=10 AS SELECT number FROM numbers(10000); SELECT count() FROM test_progress PREWHERE sleepEachRow(0.01) OR n > 1000000 SETTINGS max_block_size=10;" + +# Expect that progress is updated +expect { + "10.00 rows," { exit 0 } + "20.00 rows," { exit 0 } + "30.00 rows," { exit 0 } + "40.00 rows," { exit 0 } + "50.00 rows," { exit 0 } + "60.00 rows," { exit 0 } + "70.00 rows," { exit 0 } + "80.00 rows," { exit 0 } + "90.00 rows," { exit 0 } + timeout { exit 1 } +} From 2dfc0d008b5f120a468f91ef66d9fb8b93743d9a Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 21 Feb 2023 11:48:28 +0000 Subject: [PATCH 495/717] Fix: remove redundant sorting optimization + incorrect sorting step removal in case of parent step has more than 1 children --- .../QueryPlan/Optimizations/removeRedundantSorting.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/Processors/QueryPlan/Optimizations/removeRedundantSorting.cpp b/src/Processors/QueryPlan/Optimizations/removeRedundantSorting.cpp index 20d964dcb4f..c7b945b755c 100644 --- a/src/Processors/QueryPlan/Optimizations/removeRedundantSorting.cpp +++ b/src/Processors/QueryPlan/Optimizations/removeRedundantSorting.cpp @@ -188,7 +188,15 @@ private: return false; /// remove sorting - parent_node->children.front() = sorting_node->children.front(); + // parent_node->children.front() = sorting_node->children.front(); + for (auto & child : parent_node->children) + { + if (child == sorting_node) + { + child = sorting_node->children.front(); + break; + } + } /// sorting removed, so need to update sorting traits for upstream steps const DataStream * input_stream = &parent_node->children.front()->step->getOutputStream(); From 1f3be929f7a61d7c74f185e7bbc419a3ab5340af Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Tue, 21 Feb 2023 12:50:26 +0100 Subject: [PATCH 496/717] Update ReadFromMergeTree.cpp --- src/Processors/QueryPlan/ReadFromMergeTree.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index f90506b95a2..fd3645bf2f0 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -348,7 +348,7 @@ Pipe ReadFromMergeTree::readFromPool( if ((all_parts_are_remote && settings.allow_prefetched_read_pool_for_remote_filesystem && MergeTreePrefetchedReadPool::checkReadMethodAllowed(reader_settings.read_settings.remote_fs_method)) - || (!all_parts_are_local + || (all_parts_are_local && settings.allow_prefetched_read_pool_for_local_filesystem && MergeTreePrefetchedReadPool::checkReadMethodAllowed(reader_settings.read_settings.local_fs_method))) { From b5e6d74d48910212de98b05108ecc8cf95cfc44e Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 21 Feb 2023 15:36:23 +0300 Subject: [PATCH 497/717] More interesting settings for Stress Tests (#41534) * Update stress * fix --------- Co-authored-by: Alexey Milovidov --- docker/test/stress/stress | 6 +++++- src/Interpreters/IInterpreter.cpp | 9 ++++++++- src/Interpreters/IInterpreter.h | 2 +- src/Interpreters/InterpreterDropQuery.cpp | 4 ++-- src/Interpreters/InterpreterSelectQuery.cpp | 4 ++-- src/Interpreters/Session.cpp | 2 +- src/Server/HTTPHandler.cpp | 5 +++++ tests/queries/0_stateless/transactions.lib | 4 ++-- 8 files changed, 26 insertions(+), 10 deletions(-) diff --git a/docker/test/stress/stress b/docker/test/stress/stress index 667b5be90a5..310f8609d5a 100755 --- a/docker/test/stress/stress +++ b/docker/test/stress/stress @@ -40,12 +40,16 @@ def get_options(i, backward_compatibility_check): client_options.append("join_algorithm='auto'") client_options.append('max_rows_in_join=1000') - if i == 13: + if i % 5 == 1: client_options.append("memory_tracker_fault_probability=0.001") if i % 2 == 1 and not backward_compatibility_check: client_options.append("group_by_use_nulls=1") + if i == 12: # 12 % 3 == 0, so it's Atomic database + client_options.append("implicit_transaction=1") + client_options.append("throw_on_unsupported_query_inside_transaction=0") + if client_options: options.append(" --client-option " + " ".join(client_options)) diff --git a/src/Interpreters/IInterpreter.cpp b/src/Interpreters/IInterpreter.cpp index aff703f79af..148e73e43ce 100644 --- a/src/Interpreters/IInterpreter.cpp +++ b/src/Interpreters/IInterpreter.cpp @@ -30,7 +30,7 @@ void IInterpreter::extendQueryLogElem( extendQueryLogElemImpl(elem, ast, context); } -void IInterpreter::checkStorageSupportsTransactionsIfNeeded(const StoragePtr & storage, ContextPtr context) +void IInterpreter::checkStorageSupportsTransactionsIfNeeded(const StoragePtr & storage, ContextPtr context, bool is_readonly_query) { if (!context->getCurrentTransaction()) return; @@ -41,6 +41,13 @@ void IInterpreter::checkStorageSupportsTransactionsIfNeeded(const StoragePtr & s if (context->getSettingsRef().throw_on_unsupported_query_inside_transaction) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Storage {} (table {}) does not support transactions", storage->getName(), storage->getStorageID().getNameForLogs()); + + /// Do not allow transactions with ReplicatedMergeTree anyway (unless it's a readonly SELECT query) + /// because it may try to process transaction on MergeTreeData-level, + /// but then fail with a logical error or something on StorageReplicatedMergeTree-level. + if (!is_readonly_query && storage->supportsReplication()) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "ReplicatedMergeTree (table {}) does not support transactions", + storage->getStorageID().getNameForLogs()); } } diff --git a/src/Interpreters/IInterpreter.h b/src/Interpreters/IInterpreter.h index 74a568c5cba..5290c64387f 100644 --- a/src/Interpreters/IInterpreter.h +++ b/src/Interpreters/IInterpreter.h @@ -39,7 +39,7 @@ public: virtual bool supportsTransactions() const { return false; } /// Helper function for some Interpreters. - static void checkStorageSupportsTransactionsIfNeeded(const StoragePtr & storage, ContextPtr context); + static void checkStorageSupportsTransactionsIfNeeded(const StoragePtr & storage, ContextPtr context, bool is_readonly_query = false); virtual ~IInterpreter() = default; }; diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index e2484a48da4..f4507de5ac7 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -120,10 +120,10 @@ BlockIO InterpreterDropQuery::executeToTableImpl(ContextPtr context_, ASTDropQue auto [database, table] = query.if_exists ? DatabaseCatalog::instance().tryGetDatabaseAndTable(table_id, context_) : DatabaseCatalog::instance().getDatabaseAndTable(table_id, context_); - checkStorageSupportsTransactionsIfNeeded(table, context_); - if (database && table) { + checkStorageSupportsTransactionsIfNeeded(table, context_); + auto & ast_drop_query = query.as(); if (ast_drop_query.is_view && !table->isView()) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 2209635cebf..358bbc83599 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -436,7 +436,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( if (context->getCurrentTransaction() && context->getSettingsRef().throw_on_unsupported_query_inside_transaction) { if (storage) - checkStorageSupportsTransactionsIfNeeded(storage, context); + checkStorageSupportsTransactionsIfNeeded(storage, context, /* is_readonly_query */ true); for (const auto & table : joined_tables.tablesWithColumns()) { if (table.table.table.empty()) @@ -444,7 +444,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( auto maybe_storage = DatabaseCatalog::instance().tryGetTable({table.table.database, table.table.table}, context); if (!maybe_storage) continue; - checkStorageSupportsTransactionsIfNeeded(storage, context); + checkStorageSupportsTransactionsIfNeeded(storage, context, /* is_readonly_query */ true); } } diff --git a/src/Interpreters/Session.cpp b/src/Interpreters/Session.cpp index 5c72e24c577..7411050aa2d 100644 --- a/src/Interpreters/Session.cpp +++ b/src/Interpreters/Session.cpp @@ -453,7 +453,7 @@ std::shared_ptr Session::getSessionLog() const ContextMutablePtr Session::makeQueryContextImpl(const ClientInfo * client_info_to_copy, ClientInfo * client_info_to_move) const { if (!user_id && getClientInfo().interface != ClientInfo::Interface::TCP_INTERSERVER) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Session context must be created after authentication"); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Query context must be created after authentication"); /// We can create a query context either from a session context or from a global context. bool from_session_context = static_cast(session_context); diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp index ee8612974f0..bea2fe87e6d 100644 --- a/src/Server/HTTPHandler.cpp +++ b/src/Server/HTTPHandler.cpp @@ -554,6 +554,11 @@ void HTTPHandler::processQuery( std::string session_check = params.get("session_check", ""); session->makeSessionContext(session_id, session_timeout, session_check == "1"); } + else + { + /// We should create it even if we don't have a session_id + session->makeSessionContext(); + } auto client_info = session->getClientInfo(); auto context = session->makeQueryContext(std::move(client_info)); diff --git a/tests/queries/0_stateless/transactions.lib b/tests/queries/0_stateless/transactions.lib index 521c56754bc..6305caa4db1 100755 --- a/tests/queries/0_stateless/transactions.lib +++ b/tests/queries/0_stateless/transactions.lib @@ -13,7 +13,7 @@ function tx() url_without_session="https://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTPS}/?" url="${url_without_session}session_id=$session&query_id=$query_id&database=$CLICKHOUSE_DATABASE" - ${CLICKHOUSE_CURL} -m 60 -sSk "$url" --data "$query" | sed "s/^/tx$tx_num\t/" + ${CLICKHOUSE_CURL} -m 90 -sSk "$url" --data "$query" | sed "s/^/tx$tx_num\t/" } # Waits for the last query in session to finish @@ -63,7 +63,7 @@ function tx_async() tmp_file_name="${CLICKHOUSE_TMP}/tmp_tx_${CLICKHOUSE_TEST_ZOOKEEPER_PREFIX}" # run query asynchronously - ${CLICKHOUSE_CURL} -m 60 -sSk "$url" --data "$query" | sed "s/^/tx$tx_num\t/" & + ${CLICKHOUSE_CURL} -m 90 -sSk "$url" --data "$query" | sed "s/^/tx$tx_num\t/" & query_pid=$! echo -e "$query_id\t$query_pid" >> "$tmp_file_name" } From 6a1621fcad2f7810dcba9765cd4de7382d2e9ff0 Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Tue, 21 Feb 2023 14:15:01 +0100 Subject: [PATCH 498/717] Fix flaky test 01710_normal_projections --- tests/queries/0_stateless/01710_normal_projections.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01710_normal_projections.sh b/tests/queries/0_stateless/01710_normal_projections.sh index 70e38b3722a..8ee3f41ea28 100755 --- a/tests/queries/0_stateless/01710_normal_projections.sh +++ b/tests/queries/0_stateless/01710_normal_projections.sh @@ -4,7 +4,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -q "CREATE TABLE test_sort_proj (x UInt32, y UInt32, PROJECTION p (SELECT x, y ORDER BY y)) ENGINE = MergeTree ORDER BY x" +$CLICKHOUSE_CLIENT -q "CREATE TABLE test_sort_proj (x UInt32, y UInt32, PROJECTION p (SELECT x, y ORDER BY y)) ENGINE = MergeTree ORDER BY x SETTINGS index_granularity=8192" $CLICKHOUSE_CLIENT -q "insert into test_sort_proj select number, toUInt32(-number - 1) from numbers(100)" echo "select where x < 10" From 254cb1119b4d15a1ba2bb2e6d33a674de133c5b5 Mon Sep 17 00:00:00 2001 From: chen Date: Tue, 21 Feb 2023 21:52:17 +0800 Subject: [PATCH 499/717] Update s3Cluster.md --- docs/en/sql-reference/table-functions/s3Cluster.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/en/sql-reference/table-functions/s3Cluster.md b/docs/en/sql-reference/table-functions/s3Cluster.md index e77806a3665..9c1f7a66845 100644 --- a/docs/en/sql-reference/table-functions/s3Cluster.md +++ b/docs/en/sql-reference/table-functions/s3Cluster.md @@ -31,13 +31,13 @@ Select the data from all the files in the `/root/data/clickhouse` and `/root/dat ``` sql SELECT * FROM s3Cluster( - 'cluster_simple', - 'http://minio1:9001/root/data/{clickhouse,database}/*', - 'minio', - 'minio123', - 'CSV', - 'name String, value UInt32, polygon Array(Array(Tuple(Float64, Float64)))') ORDER BY (name, value, polygon -); + 'cluster_simple', + 'http://minio1:9001/root/data/{clickhouse,database}/*', + 'minio', + 'minio123', + 'CSV', + 'name String, value UInt32, polygon Array(Array(Tuple(Float64, Float64)))' +) ORDER BY (name, value, polygon); ``` Count the total amount of rows in all files in the cluster `cluster_simple`: From c1e611334a383fceaa70ab0267c9b1c9c8ce6274 Mon Sep 17 00:00:00 2001 From: chen Date: Tue, 21 Feb 2023 21:53:44 +0800 Subject: [PATCH 500/717] Update s3Cluster.md --- docs/en/sql-reference/table-functions/s3Cluster.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/table-functions/s3Cluster.md b/docs/en/sql-reference/table-functions/s3Cluster.md index 9c1f7a66845..f420a69596c 100644 --- a/docs/en/sql-reference/table-functions/s3Cluster.md +++ b/docs/en/sql-reference/table-functions/s3Cluster.md @@ -32,7 +32,7 @@ Select the data from all the files in the `/root/data/clickhouse` and `/root/dat ``` sql SELECT * FROM s3Cluster( 'cluster_simple', - 'http://minio1:9001/root/data/{clickhouse,database}/*', + 'http://minio1:9001/root/data/{clickhouse,database}/*', 'minio', 'minio123', 'CSV', From c83583e46b71b2dc4c5e5d17b033b5a3b3ddfb84 Mon Sep 17 00:00:00 2001 From: Salvatore Mesoraca Date: Tue, 21 Feb 2023 14:04:51 +0100 Subject: [PATCH 501/717] ActionsDAG: do not change result of and() during optimization Previously `and` results were inconsistent. Some optimizations can try to simplify and completely remove a single-argument `and`, replacing it with a cast to UInt8. This works well for all numeric types except floats. A float passed to `and` is normally interpreted as `bool` and values between 0 and 1 are treated as 1. The same value cast to an integer type will be truncated to 0. --- src/Interpreters/ActionsDAG.cpp | 20 +++- .../02667_and_consistency.reference | 23 ++++ .../0_stateless/02667_and_consistency.sql | 106 ++++++++++++++++++ 3 files changed, 148 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/02667_and_consistency.reference create mode 100644 tests/queries/0_stateless/02667_and_consistency.sql diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index 96e7daa306f..623ead29921 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -1943,6 +1943,10 @@ ActionsDAGPtr ActionsDAG::cloneActionsForFilterPushDown( } auto conjunction = getConjunctionNodes(predicate, allowed_nodes); + if (conjunction.allowed.size() == 1 && conjunction.rejected.size() == 1 + && WhichDataType{conjunction.rejected.front()->result_type}.isFloat()) + return nullptr; + auto actions = cloneActionsForConjunction(conjunction.allowed, all_inputs); if (!actions) return nullptr; @@ -2008,10 +2012,12 @@ ActionsDAGPtr ActionsDAG::cloneActionsForFilterPushDown( node.children.swap(new_children); *predicate = std::move(node); } - else + else if (!WhichDataType{new_children.front()->result_type}.isFloat()) { /// If type is different, cast column. /// This case is possible, cause AND can use any numeric type as argument. + /// But casting floats to UInt8 or Bool produces different results. + /// so we can't apply this optimization to them. Node node; node.type = ActionType::COLUMN; node.result_name = predicate->result_type->getName(); @@ -2033,8 +2039,20 @@ ActionsDAGPtr ActionsDAG::cloneActionsForFilterPushDown( else { /// Predicate is function AND, which still have more then one argument. + /// Or there is only one argument that is a float and we can't just + /// remove the AND. /// Just update children and rebuild it. predicate->children.swap(new_children); + if (WhichDataType{predicate->children.front()->result_type}.isFloat()) + { + Node node; + node.type = ActionType::COLUMN; + node.result_name = "1"; + node.column = DataTypeUInt8().createColumnConst(0, 1u); + node.result_type = std::make_shared(); + const auto * const_col = &nodes.emplace_back(std::move(node)); + predicate->children.emplace_back(const_col); + } auto arguments = prepareFunctionArguments(predicate->children); FunctionOverloadResolverPtr func_builder_and = std::make_unique(std::make_shared()); diff --git a/tests/queries/0_stateless/02667_and_consistency.reference b/tests/queries/0_stateless/02667_and_consistency.reference new file mode 100644 index 00000000000..bcb2b5aecfb --- /dev/null +++ b/tests/queries/0_stateless/02667_and_consistency.reference @@ -0,0 +1,23 @@ +true +===== +true +===== +true +===== +true +===== +===== +1 +===== +===== +allow_experimental_analyzer +true +#45440 +2086579505 0 1 0 0 +-542998757 -542998757 1 0 0 += +2086579505 0 1 0 0 +-542998757 -542998757 1 0 0 += +2086579505 0 1 0 0 +-542998757 -542998757 1 0 0 diff --git a/tests/queries/0_stateless/02667_and_consistency.sql b/tests/queries/0_stateless/02667_and_consistency.sql new file mode 100644 index 00000000000..f02185a1a52 --- /dev/null +++ b/tests/queries/0_stateless/02667_and_consistency.sql @@ -0,0 +1,106 @@ +SELECT toBool(sin(SUM(number))) AS x +FROM +( + SELECT 1 AS number +) +GROUP BY number +HAVING 1 AND sin(sum(number)) +SETTINGS enable_optimize_predicate_expression = 0; + +SELECT '====='; + +SELECT toBool(sin(SUM(number))) AS x +FROM +( + SELECT 1 AS number +) +GROUP BY number +HAVING 1 AND sin(1) +SETTINGS enable_optimize_predicate_expression = 0; + +SELECT '====='; + +SELECT toBool(sin(SUM(number))) AS x +FROM +( + SELECT 1 AS number +) +GROUP BY number +HAVING x AND sin(sum(number)) +SETTINGS enable_optimize_predicate_expression = 1; + +SELECT '====='; + +SELECT toBool(sin(SUM(number))) AS x +FROM +( + SELECT 1 AS number +) +GROUP BY number +HAVING 1 AND sin(sum(number)) +SETTINGS enable_optimize_predicate_expression = 0; + +SELECT '====='; + +SELECT toBool(sin(SUM(number))) AS x +FROM +( + SELECT 1 AS number +) +GROUP BY number +HAVING 1 AND sin(sum(number)) +SETTINGS enable_optimize_predicate_expression = 1; -- { serverError 59 } + +SELECT '====='; + +SELECT 1 and sin(1); + +SELECT '====='; + +SELECT toBool(sin(SUM(number))) AS x +FROM +( + SELECT 1 AS number +) +GROUP BY number +HAVING x AND sin(1) +SETTINGS enable_optimize_predicate_expression = 0; -- { serverError 59 } + +SELECT '====='; +SELECT 'allow_experimental_analyzer'; + +SET allow_experimental_analyzer = 1; + +SELECT toBool(sin(SUM(number))) AS x +FROM +( + SELECT 1 AS number +) +GROUP BY number +HAVING 1 AND sin(sum(number)) +SETTINGS enable_optimize_predicate_expression = 1; + +select '#45440'; + +DROP TABLE IF EXISTS t2; +CREATE TABLE t2(c0 Int32) ENGINE = MergeTree ORDER BY c0; +INSERT INTO t2 VALUES (928386547), (1541944097), (2086579505), (1990427322), (-542998757), (390253678), (554855248), (203290629), (1504693323); + +SELECT + MAX(left.c0), + min2(left.c0, -(-left.c0) * (radians(left.c0) - radians(left.c0))) AS g, + (((-1925024212 IS NOT NULL) IS NOT NULL) != radians(tan(1216286224))) AND cos(lcm(MAX(left.c0), -1966575216) OR (MAX(left.c0) * 1180517420)) AS h, + NOT h, + h IS NULL +FROM t2 AS left +GROUP BY g; +select '='; +SELECT MAX(left.c0), min2(left.c0, -(-left.c0) * (radians(left.c0) - radians(left.c0))) as g, (((-1925024212 IS NOT NULL) IS NOT NULL) != radians(tan(1216286224))) AND cos(lcm(MAX(left.c0), -1966575216) OR (MAX(left.c0) * 1180517420)) as h, not h, h is null + FROM t2 AS left + GROUP BY g HAVING h SETTINGS enable_optimize_predicate_expression = 0; +select '='; +SELECT MAX(left.c0), min2(left.c0, -(-left.c0) * (radians(left.c0) - radians(left.c0))) as g, (((-1925024212 IS NOT NULL) IS NOT NULL) != radians(tan(1216286224))) AND cos(lcm(MAX(left.c0), -1966575216) OR (MAX(left.c0) * 1180517420)) as h, not h, h is null + FROM t2 AS left + GROUP BY g HAVING h SETTINGS enable_optimize_predicate_expression = 1; + +DROP TABLE IF EXISTS t2; From 3df7a10ac7e13c8d7836583e9e0896a797183939 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Tue, 21 Feb 2023 16:25:11 +0100 Subject: [PATCH 502/717] Update postgres_utility.py --- tests/integration/helpers/postgres_utility.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/helpers/postgres_utility.py b/tests/integration/helpers/postgres_utility.py index 978b9a98fb4..838c22c8a7c 100644 --- a/tests/integration/helpers/postgres_utility.py +++ b/tests/integration/helpers/postgres_utility.py @@ -320,11 +320,11 @@ def check_tables_are_synchronized( ) result = instance.query(result_query) - for _ in range(30): + for _ in range(50): if result == expected: break else: - time.sleep(0.5) + time.sleep(1) result = instance.query(result_query) assert result == expected From 1d4352d82af2cc354890018d1e5d715d2968f7a9 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 21 Feb 2023 17:01:19 +0100 Subject: [PATCH 503/717] Fix integration test: terminate old version without wait --- .../test_backup_with_other_granularity/test.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_backup_with_other_granularity/test.py b/tests/integration/test_backup_with_other_granularity/test.py index d30c45c3691..f456fae23a8 100644 --- a/tests/integration/test_backup_with_other_granularity/test.py +++ b/tests/integration/test_backup_with_other_granularity/test.py @@ -54,7 +54,8 @@ def test_backup_from_old_version(started_cluster): node1.query("ALTER TABLE source_table FREEZE PARTITION tuple();") - node1.restart_with_latest_version(fix_metadata=True) + # We don't want to wait old outdated version to finish properly, just terminate it + node1.restart_with_latest_version(fix_metadata=True, signal=9) node1.query( "CREATE TABLE dest_table (A Int64, B String, Y String) ENGINE = ReplicatedMergeTree('/test/dest_table1', '1') ORDER BY tuple()" @@ -107,7 +108,8 @@ def test_backup_from_old_version_setting(started_cluster): node2.query("ALTER TABLE source_table FREEZE PARTITION tuple();") - node2.restart_with_latest_version(fix_metadata=True) + # We don't want to wait old outdated version to finish properly, just terminate it + node2.restart_with_latest_version(fix_metadata=True, signal=9) node2.query( "CREATE TABLE dest_table (A Int64, B String, Y String) ENGINE = ReplicatedMergeTree('/test/dest_table2', '1') ORDER BY tuple() SETTINGS enable_mixed_granularity_parts = 1" @@ -163,7 +165,8 @@ def test_backup_from_old_version_config(started_cluster): "1", ) - node3.restart_with_latest_version(callback_onstop=callback, fix_metadata=True) + # We don't want to wait old outdated version to finish properly, just terminate it + node3.restart_with_latest_version(callback_onstop=callback, fix_metadata=True, signal=9) node3.query( "CREATE TABLE dest_table (A Int64, B String, Y String) ENGINE = ReplicatedMergeTree('/test/dest_table3', '1') ORDER BY tuple() SETTINGS enable_mixed_granularity_parts = 1" From af677c7dcd6663cc89fa5756c2bff9c59bbb0d8d Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 21 Feb 2023 16:08:13 +0000 Subject: [PATCH 504/717] Automatic style fix --- tests/integration/test_backup_with_other_granularity/test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_backup_with_other_granularity/test.py b/tests/integration/test_backup_with_other_granularity/test.py index f456fae23a8..2a82fc71951 100644 --- a/tests/integration/test_backup_with_other_granularity/test.py +++ b/tests/integration/test_backup_with_other_granularity/test.py @@ -166,7 +166,9 @@ def test_backup_from_old_version_config(started_cluster): ) # We don't want to wait old outdated version to finish properly, just terminate it - node3.restart_with_latest_version(callback_onstop=callback, fix_metadata=True, signal=9) + node3.restart_with_latest_version( + callback_onstop=callback, fix_metadata=True, signal=9 + ) node3.query( "CREATE TABLE dest_table (A Int64, B String, Y String) ENGINE = ReplicatedMergeTree('/test/dest_table3', '1') ORDER BY tuple() SETTINGS enable_mixed_granularity_parts = 1" From 97e9df01488b68139d79a58e21a6f12c78cc3335 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 21 Feb 2023 19:10:44 +0300 Subject: [PATCH 505/717] Update stress --- docker/test/stress/stress | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/stress/stress b/docker/test/stress/stress index 310f8609d5a..e23d5988918 100755 --- a/docker/test/stress/stress +++ b/docker/test/stress/stress @@ -78,7 +78,7 @@ def run_func_test( pipes = [] for i in range(0, len(output_paths)): f = open(output_paths[i], "w") - full_command = "{} {} {} {} {} --stress".format( + full_command = "{} {} {} {} {}".format( cmd, get_options(i, backward_compatibility_check), global_time_limit_option, From b6612d2c18a7a45d2d4a36aa15e56a9933b3ff49 Mon Sep 17 00:00:00 2001 From: Dan Roscigno Date: Tue, 21 Feb 2023 11:24:39 -0500 Subject: [PATCH 506/717] fix anchor link --- docs/en/interfaces/formats.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index e94c6377ae9..992eaa8a49f 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -13,7 +13,7 @@ The supported formats are: | Format | Input | Output | |-------------------------------------------------------------------------------------------|------|--------| | [TabSeparated](#tabseparated) | ✔ | ✔ | -| [TabSeparatedRaw](#tabseparatedraw) | ✔ | ✔ | +| [TabSeparatedRaw](#tabseparatedraw) | ✔ | ✔ | | [TabSeparatedWithNames](#tabseparatedwithnames) | ✔ | ✔ | | [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes) | ✔ | ✔ | | [TabSeparatedRawWithNames](#tabseparatedrawwithnames) | ✔ | ✔ | @@ -33,7 +33,7 @@ The supported formats are: | [JSONAsString](#jsonasstring) | ✔ | ✗ | | [JSONStrings](#jsonstrings) | ✔ | ✔ | | [JSONColumns](#jsoncolumns) | ✔ | ✔ | -| [JSONColumnsWithMetadata](#jsoncolumnswithmetadata) | ✔ | ✔ | +| [JSONColumnsWithMetadata](#jsoncolumnsmonoblock)) | ✔ | ✔ | | [JSONCompact](#jsoncompact) | ✔ | ✔ | | [JSONCompactStrings](#jsoncompactstrings) | ✗ | ✔ | | [JSONCompactColumns](#jsoncompactcolumns) | ✔ | ✔ | From cd2ab02f22aefafa2e203230018bda12429a4827 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Tue, 21 Feb 2023 17:34:16 +0100 Subject: [PATCH 507/717] Get rid of legacy DocsReleaseChecks --- .github/workflows/docs_release.yml | 118 ---------------- docker/docs/release/Dockerfile | 44 ------ docker/docs/release/run.sh | 12 -- docker/images.json | 4 - docs/tools/.gitignore | 2 - docs/tools/README.md | 1 - docs/tools/build.py | 108 --------------- docs/tools/release.sh | 42 ------ docs/tools/requirements.txt | 1 - .../cancel_and_rerun_workflow_lambda/app.py | 1 - tests/ci/docs_release.py | 126 ------------------ tests/ci/workflow_approve_rerun_lambda/app.py | 1 - 12 files changed, 460 deletions(-) delete mode 100644 .github/workflows/docs_release.yml delete mode 100644 docker/docs/release/Dockerfile delete mode 100644 docker/docs/release/run.sh delete mode 100644 docs/tools/.gitignore delete mode 100644 docs/tools/README.md delete mode 100755 docs/tools/build.py delete mode 100755 docs/tools/release.sh delete mode 100644 docs/tools/requirements.txt delete mode 100644 tests/ci/docs_release.py diff --git a/.github/workflows/docs_release.yml b/.github/workflows/docs_release.yml deleted file mode 100644 index fc4b9d88c3e..00000000000 --- a/.github/workflows/docs_release.yml +++ /dev/null @@ -1,118 +0,0 @@ -name: DocsReleaseChecks - -env: - # Force the stdout and stderr streams to be unbuffered - PYTHONUNBUFFERED: 1 - -concurrency: - group: master-release - cancel-in-progress: true -'on': - push: - branches: - - master - paths: - - '.github/**' - - 'docker/docs/release/**' - - 'docs/**' - - 'utils/list-versions/version_date.tsv' - - 'website/**' - - 'utils/check-style/aspell-ignore/**' - workflow_dispatch: -jobs: - DockerHubPushAarch64: - runs-on: [self-hosted, style-checker-aarch64] - steps: - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Images check - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_images_check.py --suffix aarch64 - - name: Upload images files to artifacts - uses: actions/upload-artifact@v3 - with: - name: changed_images_aarch64 - path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json - DockerHubPushAmd64: - runs-on: [self-hosted, style-checker] - steps: - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Images check - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_images_check.py --suffix amd64 - - name: Upload images files to artifacts - uses: actions/upload-artifact@v3 - with: - name: changed_images_amd64 - path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json - DockerHubPush: - needs: [DockerHubPushAmd64, DockerHubPushAarch64] - runs-on: [self-hosted, style-checker] - steps: - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Download changed aarch64 images - uses: actions/download-artifact@v3 - with: - name: changed_images_aarch64 - path: ${{ runner.temp }} - - name: Download changed amd64 images - uses: actions/download-artifact@v3 - with: - name: changed_images_amd64 - path: ${{ runner.temp }} - - name: Images check - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 - - name: Upload images files to artifacts - uses: actions/upload-artifact@v3 - with: - name: changed_images - path: ${{ runner.temp }}/changed_images.json - DocsRelease: - needs: DockerHubPush - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - # https://docs.github.com/en/actions/learn-github-actions/workflow-commands-for-github-actions#multiline-strings - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/docs_release - REPO_COPY=${{runner.temp}}/docs_release/ClickHouse - CLOUDFLARE_TOKEN=${{secrets.CLOUDFLARE}} - ROBOT_CLICKHOUSE_SSH_KEY<> /etc/ssh/ssh_known_hosts - -COPY run.sh / - -ENV REPO_PATH=/repo_path -ENV OUTPUT_PATH=/output_path - -CMD ["/bin/bash", "/run.sh"] diff --git a/docker/docs/release/run.sh b/docker/docs/release/run.sh deleted file mode 100644 index e5a9f2101aa..00000000000 --- a/docker/docs/release/run.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -cd "$REPO_PATH/docs/tools" -if ! [ -d venv ]; then - mkdir -p venv - virtualenv -p "$(which python3)" venv - source venv/bin/activate - python3 -m pip install --ignore-installed -r requirements.txt -fi -source venv/bin/activate -./release.sh 2>&1 | tee "$OUTPUT_PATH/output.log" diff --git a/docker/images.json b/docker/images.json index bd63aea24ba..78e7729671e 100644 --- a/docker/images.json +++ b/docker/images.json @@ -146,9 +146,5 @@ "name": "clickhouse/docs-builder", "dependent": [ ] - }, - "docker/docs/release": { - "name": "clickhouse/docs-release", - "dependent": [] } } diff --git a/docs/tools/.gitignore b/docs/tools/.gitignore deleted file mode 100644 index 8d35cb3277f..00000000000 --- a/docs/tools/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -__pycache__ -*.pyc diff --git a/docs/tools/README.md b/docs/tools/README.md deleted file mode 100644 index c7147a0c850..00000000000 --- a/docs/tools/README.md +++ /dev/null @@ -1 +0,0 @@ -See https://github.com/ClickHouse/clickhouse-docs/blob/main/contrib-writing-guide.md diff --git a/docs/tools/build.py b/docs/tools/build.py deleted file mode 100755 index 5653a9b949d..00000000000 --- a/docs/tools/build.py +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env python3 - -from pathlib import Path -import argparse -import logging -import shutil -import sys - -import livereload - - -def write_redirect_html(output_path: Path, to_url: str) -> None: - output_dir = output_path.parent - output_dir.mkdir(parents=True, exist_ok=True) - output_path.write_text( - f""" - - - - - - - Page Redirection - - - If you are not redirected automatically, follow this link. - -""" - ) - - -def build_static_redirects(output_dir: Path): - for static_redirect in [ - ("benchmark.html", "/benchmark/dbms/"), - ("benchmark_hardware.html", "/benchmark/hardware/"), - ( - "tutorial.html", - "/docs/en/getting_started/tutorial/", - ), - ( - "reference_en.html", - "/docs/en/single/", - ), - ( - "reference_ru.html", - "/docs/ru/single/", - ), - ( - "docs/index.html", - "/docs/en/", - ), - ]: - write_redirect_html(output_dir / static_redirect[0], static_redirect[1]) - - -def build(root_dir: Path, output_dir: Path): - if output_dir.exists(): - shutil.rmtree(args.output_dir) - - (output_dir / "data").mkdir(parents=True) - - logging.info("Building website") - - # This file can be requested to check for available ClickHouse releases. - shutil.copy2( - root_dir / "utils" / "list-versions" / "version_date.tsv", - output_dir / "data" / "version_date.tsv", - ) - - # This file can be requested to install ClickHouse. - shutil.copy2( - root_dir / "docs" / "_includes" / "install" / "universal.sh", - output_dir / "data" / "install.sh", - ) - - build_static_redirects(output_dir) - - -if __name__ == "__main__": - root_dir = Path(__file__).parent.parent.parent - docs_dir = root_dir / "docs" - - arg_parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - arg_parser.add_argument( - "--output-dir", - type=Path, - default=docs_dir / "build", - help="path to the output dir", - ) - arg_parser.add_argument("--livereload", type=int, default="0") - arg_parser.add_argument("--verbose", action="store_true") - - args = arg_parser.parse_args() - - logging.basicConfig( - level=logging.DEBUG if args.verbose else logging.INFO, stream=sys.stderr - ) - - build(root_dir, args.output_dir) - - if args.livereload: - server = livereload.Server() - server.serve(root=args.output_dir, host="0.0.0.0", port=args.livereload) - sys.exit(0) diff --git a/docs/tools/release.sh b/docs/tools/release.sh deleted file mode 100755 index c198f488822..00000000000 --- a/docs/tools/release.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash -set -ex - -BASE_DIR=$(dirname "$(readlink -f "$0")") -BUILD_DIR="${BASE_DIR}/../build" -PUBLISH_DIR="${BASE_DIR}/../publish" -BASE_DOMAIN="${BASE_DOMAIN:-content.clickhouse.com}" -GIT_PROD_URI="${GIT_PROD_URI:-git@github.com:ClickHouse/clickhouse-com-content.git}" -EXTRA_BUILD_ARGS="${EXTRA_BUILD_ARGS:---verbose}" - -if [[ -z "$1" ]] -then - source "${BASE_DIR}/venv/bin/activate" - # shellcheck disable=2086 - python3 "${BASE_DIR}/build.py" ${EXTRA_BUILD_ARGS} - rm -rf "${PUBLISH_DIR}" - mkdir "${PUBLISH_DIR}" && cd "${PUBLISH_DIR}" - - # Will make a repository with website content as the only commit. - git init - git remote add origin "${GIT_PROD_URI}" - git config user.email "robot-clickhouse@users.noreply.github.com" - git config user.name "robot-clickhouse" - - # Add files. - cp -R "${BUILD_DIR}"/* . - echo -n "${BASE_DOMAIN}" > CNAME - cat > README.md << 'EOF' -## This repo is the source for https://content.clickhouse.com -It's built in [the action](https://github.com/ClickHouse/ClickHouse/blob/master/.github/workflows/docs_release.yml) in the DocsRelease job. -EOF - echo -n "" > ".nojekyll" - cp "${BASE_DIR}/../../LICENSE" . - git add ./* - git add ".nojekyll" - - git commit --quiet -m "Add new release at $(date)" - - # Push to GitHub rewriting the existing contents. - # Sometimes it does not work with error message "! [remote rejected] master -> master (cannot lock ref 'refs/heads/master': is at 42a0f6b6b6c7be56a469441b4bf29685c1cebac3 but expected 520e9b02c0d4678a2a5f41d2f561e6532fb98cc1)" - for _ in {1..10}; do git push --force origin master && break; sleep 5; done -fi diff --git a/docs/tools/requirements.txt b/docs/tools/requirements.txt deleted file mode 100644 index 0e0f7c6d044..00000000000 --- a/docs/tools/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -livereload==2.6.3 diff --git a/tests/ci/cancel_and_rerun_workflow_lambda/app.py b/tests/ci/cancel_and_rerun_workflow_lambda/app.py index 550ab45da55..047b630e241 100644 --- a/tests/ci/cancel_and_rerun_workflow_lambda/app.py +++ b/tests/ci/cancel_and_rerun_workflow_lambda/app.py @@ -20,7 +20,6 @@ NEED_RERUN_ON_EDITED = { } NEED_RERUN_OR_CANCELL_WORKFLOWS = { - "DocsReleaseChecks", "BackportPR", }.union(NEED_RERUN_ON_EDITED) diff --git a/tests/ci/docs_release.py b/tests/ci/docs_release.py deleted file mode 100644 index 1b93aba99ba..00000000000 --- a/tests/ci/docs_release.py +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env python3 -import argparse -import logging -import subprocess -import os -import sys - -from github import Github - -from commit_status_helper import get_commit -from docker_pull_helper import get_image_with_version -from env_helper import TEMP_PATH, REPO_COPY, CLOUDFLARE_TOKEN -from get_robot_token import get_best_robot_token -from pr_info import PRInfo -from report import TestResults, TestResult -from rerun_helper import RerunHelper -from s3_helper import S3Helper -from ssh import SSHKey -from tee_popen import TeePopen -from upload_result_helper import upload_results - -NAME = "Docs Release" - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - description="ClickHouse building script using prebuilt Docker image", - ) - parser.add_argument( - "--as-root", action="store_true", help="if the container should run as root" - ) - return parser.parse_args() - - -def main(): - logging.basicConfig(level=logging.INFO) - args = parse_args() - - temp_path = TEMP_PATH - repo_path = REPO_COPY - - gh = Github(get_best_robot_token(), per_page=100) - pr_info = PRInfo() - rerun_helper = RerunHelper(gh, pr_info, NAME) - if rerun_helper.is_already_finished_by_status(): - logging.info("Check is already finished according to github status, exiting") - sys.exit(0) - - if not os.path.exists(temp_path): - os.makedirs(temp_path) - - docker_image = get_image_with_version(temp_path, "clickhouse/docs-release") - - test_output = os.path.join(temp_path, "docs_release_log") - if not os.path.exists(test_output): - os.makedirs(test_output) - - if args.as_root: - user = "0:0" - else: - user = f"{os.geteuid()}:{os.getegid()}" - - run_log_path = os.path.join(test_output, "run.log") - - with SSHKey("ROBOT_CLICKHOUSE_SSH_KEY"): - cmd = ( - f"docker run --cap-add=SYS_PTRACE --user={user} " - f"--volume='{os.getenv('SSH_AUTH_SOCK', '')}:/ssh-agent' " - f"--volume={repo_path}:/repo_path --volume={test_output}:/output_path " - f"-e SSH_AUTH_SOCK=/ssh-agent -e EXTRA_BUILD_ARGS='--verbose' " - f"-e CLOUDFLARE_TOKEN={CLOUDFLARE_TOKEN} {docker_image}" - ) - logging.info("Running command: %s", cmd) - with TeePopen(cmd, run_log_path) as process: - retcode = process.wait() - if retcode == 0: - logging.info("Run successfully") - status = "success" - description = "Released successfuly" - else: - description = "Release failed (non zero exit code)" - status = "failure" - logging.info("Run failed") - - subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {temp_path}", shell=True) - files = os.listdir(test_output) - test_results = [] # type: TestResults - additional_files = [] - if not files: - logging.error("No output files after docs release") - description = "No output files after docs release" - status = "failure" - else: - for f in files: - path = os.path.join(test_output, f) - additional_files.append(path) - with open(path, "r", encoding="utf-8") as check_file: - for line in check_file: - if "ERROR" in line: - test_results.append(TestResult(line.split(":")[-1], "FAIL")) - if test_results: - status = "failure" - description = "Found errors in docs" - elif status != "failure": - test_results.append(TestResult("No errors found", "OK")) - else: - test_results.append(TestResult("Non zero exit code", "FAIL")) - - s3_helper = S3Helper() - - report_url = upload_results( - s3_helper, pr_info.number, pr_info.sha, test_results, additional_files, NAME - ) - print("::notice ::Report url: {report_url}") - commit = get_commit(gh, pr_info.sha) - commit.create_status( - context=NAME, description=description, state=status, target_url=report_url - ) - - if status == "failure": - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/tests/ci/workflow_approve_rerun_lambda/app.py b/tests/ci/workflow_approve_rerun_lambda/app.py index fb14dfd2258..a4a5a013c36 100644 --- a/tests/ci/workflow_approve_rerun_lambda/app.py +++ b/tests/ci/workflow_approve_rerun_lambda/app.py @@ -62,7 +62,6 @@ TRUSTED_WORKFLOW_IDS = { NEED_RERUN_WORKFLOWS = { "BackportPR", "DocsCheck", - "DocsReleaseChecks", "MasterCI", "NightlyBuilds", "PullRequestCI", From eeac1abaa5532f12e04e3c5cfb15149fb5318663 Mon Sep 17 00:00:00 2001 From: Han Fei Date: Tue, 21 Feb 2023 17:49:52 +0100 Subject: [PATCH 508/717] add docs for setting async_insert_max_query_number --- docs/en/operations/settings/settings.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 1d1d8b32d1d..a983970de34 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -1561,6 +1561,17 @@ Possible values: Default value: `100000`. +### async_insert_max_query_number {#async-insert-max-query-number} + +The maximum number of insert query in per block before being inserted. It takes effect only if [async_insert_deduplicate](#settings-async-insert-deduplicate) is enabled. + +Possible values: + +- Positive integer. +- 0 — Asynchronous insertions are disabled. + +Default value: `450`. + ### async_insert_busy_timeout_ms {#async-insert-busy-timeout-ms} The maximum timeout in milliseconds since the first `INSERT` query before inserting collected data. From 45b1b66fd8d420db449eedb87b0271dcd53e0b08 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 21 Feb 2023 16:58:45 +0000 Subject: [PATCH 509/717] Remove unnecessary comment --- .../QueryPlan/Optimizations/removeRedundantSorting.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Processors/QueryPlan/Optimizations/removeRedundantSorting.cpp b/src/Processors/QueryPlan/Optimizations/removeRedundantSorting.cpp index c7b945b755c..41e30dee83e 100644 --- a/src/Processors/QueryPlan/Optimizations/removeRedundantSorting.cpp +++ b/src/Processors/QueryPlan/Optimizations/removeRedundantSorting.cpp @@ -188,7 +188,6 @@ private: return false; /// remove sorting - // parent_node->children.front() = sorting_node->children.front(); for (auto & child : parent_node->children) { if (child == sorting_node) From 6822ad67e2346f6eb21e9dadfaab56299edbab4c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 21 Feb 2023 20:02:12 +0300 Subject: [PATCH 510/717] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5b5dc4ae528..d2809c1b141 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ClickHouse® is an open-source column-oriented database management system that allows generating analytical data reports in real-time. -## How To Install +## How To Install (Linux, macOS, FreeBSD) ``` curl https://clickhouse.com/ | sh ``` From 9a7c71b78e24e0f3b38a2a3d49a1f446b4a74e7e Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 21 Feb 2023 18:07:57 +0100 Subject: [PATCH 511/717] Allow to hide only values from system.named_collections --- src/Access/Common/AccessType.h | 1 + src/Access/UsersConfigAccessStorage.cpp | 6 ++++++ .../System/StorageSystemNamedCollections.cpp | 7 ++++++- .../configs/users.d/users.xml | 1 + .../integration/test_named_collections/test.py | 18 +++++++++++++++++- 5 files changed, 31 insertions(+), 2 deletions(-) diff --git a/src/Access/Common/AccessType.h b/src/Access/Common/AccessType.h index 497327c1bad..f57cc2886e3 100644 --- a/src/Access/Common/AccessType.h +++ b/src/Access/Common/AccessType.h @@ -135,6 +135,7 @@ enum class AccessType M(SHOW_SETTINGS_PROFILES, "SHOW PROFILES, SHOW CREATE SETTINGS PROFILE, SHOW CREATE PROFILE", GLOBAL, SHOW_ACCESS) \ M(SHOW_ACCESS, "", GROUP, ACCESS_MANAGEMENT) \ M(SHOW_NAMED_COLLECTIONS, "SHOW NAMED COLLECTIONS", GLOBAL, ACCESS_MANAGEMENT) \ + M(SHOW_NAMED_COLLECTIONS_SECRETS, "SHOW NAMED COLLECTIONS SECRETS", GLOBAL, ACCESS_MANAGEMENT) \ M(ACCESS_MANAGEMENT, "", GROUP, ALL) \ \ M(SYSTEM_SHUTDOWN, "SYSTEM KILL, SHUTDOWN", GLOBAL, SYSTEM) \ diff --git a/src/Access/UsersConfigAccessStorage.cpp b/src/Access/UsersConfigAccessStorage.cpp index 58edff039ca..b893554cb8a 100644 --- a/src/Access/UsersConfigAccessStorage.cpp +++ b/src/Access/UsersConfigAccessStorage.cpp @@ -239,6 +239,12 @@ namespace user->access.revoke(AccessType::SHOW_NAMED_COLLECTIONS); } + bool show_named_collections_secrets = config.getBool(user_config + ".show_named_collections_secrets", false); + if (!show_named_collections_secrets) + { + user->access.revoke(AccessType::SHOW_NAMED_COLLECTIONS_SECRETS); + } + String default_database = config.getString(user_config + ".default_database", ""); user->default_database = default_database; diff --git a/src/Storages/System/StorageSystemNamedCollections.cpp b/src/Storages/System/StorageSystemNamedCollections.cpp index bc1e3a45e6b..621799c37f2 100644 --- a/src/Storages/System/StorageSystemNamedCollections.cpp +++ b/src/Storages/System/StorageSystemNamedCollections.cpp @@ -9,6 +9,7 @@ #include #include #include +#include namespace DB @@ -30,6 +31,7 @@ StorageSystemNamedCollections::StorageSystemNamedCollections(const StorageID & t void StorageSystemNamedCollections::fillData(MutableColumns & res_columns, ContextPtr context, const SelectQueryInfo &) const { context->checkAccess(AccessType::SHOW_NAMED_COLLECTIONS); + const auto & access = context->getAccess(); auto collections = NamedCollectionFactory::instance().getAll(); for (const auto & [name, collection] : collections) @@ -47,7 +49,10 @@ void StorageSystemNamedCollections::fillData(MutableColumns & res_columns, Conte for (const auto & key : collection->getKeys()) { key_column.insertData(key.data(), key.size()); - value_column.insert(collection->get(key)); + if (access->isGranted(AccessType::SHOW_NAMED_COLLECTIONS_SECRETS)) + value_column.insert(collection->get(key)); + else + value_column.insert("[HIDDEN]"); size++; } diff --git a/tests/integration/test_named_collections/configs/users.d/users.xml b/tests/integration/test_named_collections/configs/users.d/users.xml index fb5e2028d6e..8556e73c82f 100644 --- a/tests/integration/test_named_collections/configs/users.d/users.xml +++ b/tests/integration/test_named_collections/configs/users.d/users.xml @@ -5,6 +5,7 @@ default default 1 + 1 diff --git a/tests/integration/test_named_collections/test.py b/tests/integration/test_named_collections/test.py index 3b102f1aa70..612b894461b 100644 --- a/tests/integration/test_named_collections/test.py +++ b/tests/integration/test_named_collections/test.py @@ -102,7 +102,23 @@ def test_access(cluster): ["bash", "-c", f"cat /etc/clickhouse-server/users.d/users.xml"] ) node.restart_clickhouse() - assert int(node.query("select count() from system.named_collections")) > 0 + assert node.query("select collection['key1'] from system.named_collections").strip() == "value1" + replace_in_users_config( + node, "show_named_collections_secrets>1", "show_named_collections_secrets>0" + ) + assert "show_named_collections_secrets>0" in node.exec_in_container( + ["bash", "-c", f"cat /etc/clickhouse-server/users.d/users.xml"] + ) + node.restart_clickhouse() + assert node.query("select collection['key1'] from system.named_collections").strip() == "[HIDDEN]" + replace_in_users_config( + node, "show_named_collections_secrets>0", "show_named_collections_secrets>1" + ) + assert "show_named_collections_secrets>1" in node.exec_in_container( + ["bash", "-c", f"cat /etc/clickhouse-server/users.d/users.xml"] + ) + node.restart_clickhouse() + assert node.query("select collection['key1'] from system.named_collections").strip() == "value1" def test_config_reload(cluster): From efea3cbc5c6f37bc269f70fb35fb1ce4843ac6a6 Mon Sep 17 00:00:00 2001 From: Dan Roscigno Date: Tue, 21 Feb 2023 12:19:56 -0500 Subject: [PATCH 512/717] Update docs/en/operations/settings/settings.md --- docs/en/operations/settings/settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index a983970de34..1060eae1b0e 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -1563,7 +1563,7 @@ Default value: `100000`. ### async_insert_max_query_number {#async-insert-max-query-number} -The maximum number of insert query in per block before being inserted. It takes effect only if [async_insert_deduplicate](#settings-async-insert-deduplicate) is enabled. +The maximum number of insert queries per block before being inserted. This setting takes effect only if [async_insert_deduplicate](#settings-async-insert-deduplicate) is enabled. Possible values: From 07158e625302e596cd7b5280e71712bd79fd285b Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 13 Feb 2023 17:41:39 +0000 Subject: [PATCH 513/717] Add CrossToInnerJoinPass --- src/Analyzer/JoinNode.cpp | 18 ++ src/Analyzer/JoinNode.h | 2 + src/Analyzer/Passes/CrossToInnerJoinPass.cpp | 253 +++++++++++++++++++ src/Analyzer/Passes/CrossToInnerJoinPass.h | 24 ++ src/Analyzer/QueryTreePassManager.cpp | 2 + 5 files changed, 299 insertions(+) create mode 100644 src/Analyzer/Passes/CrossToInnerJoinPass.cpp create mode 100644 src/Analyzer/Passes/CrossToInnerJoinPass.h diff --git a/src/Analyzer/JoinNode.cpp b/src/Analyzer/JoinNode.cpp index 28a0c4ad7e0..86cf7e90c3f 100644 --- a/src/Analyzer/JoinNode.cpp +++ b/src/Analyzer/JoinNode.cpp @@ -15,6 +15,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + JoinNode::JoinNode(QueryTreeNodePtr left_table_expression_, QueryTreeNodePtr right_table_expression_, QueryTreeNodePtr join_expression_, @@ -113,4 +118,17 @@ ASTPtr JoinNode::toASTImpl() const return tables_in_select_query_ast; } +void JoinNode::crossToInner(const QueryTreeNodePtr & join_expression_) +{ + if (kind != JoinKind::Cross && kind != JoinKind::Comma) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot rewrite join {} to inner join, expected cross", toString(kind)); + + if (children[join_expression_child_index]) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Join expression is not empty: '{}'", children[join_expression_child_index]->formatConvertedASTForErrorMessage()); + + kind = JoinKind::Inner; + strictness = JoinStrictness::All; + children[join_expression_child_index] = std::move(join_expression_); +} + } diff --git a/src/Analyzer/JoinNode.h b/src/Analyzer/JoinNode.h index 15ba11a0122..d1cd64e67b7 100644 --- a/src/Analyzer/JoinNode.h +++ b/src/Analyzer/JoinNode.h @@ -126,6 +126,8 @@ public: return QueryTreeNodeType::JOIN; } + void crossToInner(const QueryTreeNodePtr & join_expression_); + void dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const override; protected: diff --git a/src/Analyzer/Passes/CrossToInnerJoinPass.cpp b/src/Analyzer/Passes/CrossToInnerJoinPass.cpp new file mode 100644 index 00000000000..dc03e340b15 --- /dev/null +++ b/src/Analyzer/Passes/CrossToInnerJoinPass.cpp @@ -0,0 +1,253 @@ +#include + +#include + +#include +#include +#include +#include +#include + +#include +#include + +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int INCORRECT_QUERY; +} + +namespace +{ + +using EquiCondition = std::tuple; + +void exctractJoinConditions(const QueryTreeNodePtr & node, QueryTreeNodes & equi_conditions, QueryTreeNodes & other) +{ + if (auto * func = node->as()) + { + const auto & args = func->getArguments().getNodes(); + + if (args.size() == 2 && func->getFunctionName() == "equals") + { + equi_conditions.push_back(node); + return; + } + + if (func->getFunctionName() == "and") + { + for (auto & arg : args) + exctractJoinConditions(arg, equi_conditions, other); + return; + } + } + + other.push_back(node); +} + +const QueryTreeNodePtr & getEquiArgument(const QueryTreeNodePtr & cond, size_t index) +{ + const auto * func = cond->as(); + chassert(func && func->getFunctionName() == "equals" && func->getArguments().getNodes().size() == 2); + return func->getArguments().getNodes()[index]; +} + + +/// Check that node has only one source and return it. +/// {_, false} - multiple sources +/// {nullptr, true} - no sources +/// {source, true} - single source +std::pair getExpressionSource(const QueryTreeNodePtr & node) +{ + if (const auto * column = node->as()) + { + auto source = column->getColumnSourceOrNull(); + if (!source) + return {nullptr, false}; + return {source.get(), true}; + } + + if (const auto * func = node->as()) + { + const IQueryTreeNode * source = nullptr; + const auto & args = func->getArguments().getNodes(); + for (auto & arg : args) + { + auto [arg_source, is_ok] = getExpressionSource(arg); + if (!is_ok) + return {nullptr, false}; + + if (!source) + source = arg_source; + else if (arg_source && !source->isEqual(*arg_source)) + return {nullptr, false}; + } + return {source, true}; + + } + + if (node->as()) + return {nullptr, true}; + + return {nullptr, false}; +} + +bool findInTableExpression(const IQueryTreeNode * source, const QueryTreeNodePtr & table_expression) +{ + if (!source) + return true; + + if (source->isEqual(*table_expression)) + return true; + + if (const auto * join_node = table_expression->as()) + { + return findInTableExpression(source, join_node->getLeftTableExpression()) + || findInTableExpression(source, join_node->getRightTableExpression()); + } + + if (const auto * query_node = table_expression->as()) + { + return findInTableExpression(source, query_node->getJoinTree()); + } + + return false; +} + +class CrossToInnerJoinVisitor : public InDepthQueryTreeVisitorWithContext +{ +public: + using Base = InDepthQueryTreeVisitorWithContext; + using Base::Base; + + /// Returns false if can't rewrite cross to inner join + bool tryRewrite(JoinNode * join_node) + { + if (!isCrossOrComma(join_node->getKind())) + return true; + + if (where_stack.empty()) + return false; + + auto & where_condition = *where_stack.back(); + if (!where_condition) + return false; + + const auto & left_table = join_node->getLeftTableExpression(); + const auto & right_table = join_node->getRightTableExpression(); + + QueryTreeNodes equi_conditions; + QueryTreeNodes other_conditions; + exctractJoinConditions(where_condition, equi_conditions, other_conditions); + bool can_join_on_anything = false; + for (auto & cond : equi_conditions) + { + auto left_src = getExpressionSource(getEquiArgument(cond, 0)); + auto right_src = getExpressionSource(getEquiArgument(cond, 1)); + if (left_src.second && right_src.second && left_src.first && right_src.first) + { + bool can_join_on = (findInTableExpression(left_src.first, left_table) && findInTableExpression(right_src.first, right_table)) + || (findInTableExpression(left_src.first, right_table) && findInTableExpression(right_src.first, left_table)); + + if (can_join_on) + { + can_join_on_anything = true; + continue; + } + } + + /// Can't join on this condition, move it to other conditions + other_conditions.push_back(cond); + cond = nullptr; + } + + if (!can_join_on_anything) + return false; + + equi_conditions.erase(std::remove(equi_conditions.begin(), equi_conditions.end(), nullptr), equi_conditions.end()); + join_node->crossToInner(makeConjunction(equi_conditions)); + where_condition = makeConjunction(other_conditions); + return true; + } + + void visitImpl(QueryTreeNodePtr & node) + { + if (!isEnabled()) + return; + + if (auto * query_node = node->as()) + { + /// We are entering the subtree and can use WHERE condition from this subtree + if (auto & where_node = query_node->getWhere()) + where_stack.push_back(&where_node); + } + + if (auto * join_node = node->as()) + { + bool is_rewritten = tryRewrite(join_node); + if (!is_rewritten && forceRewrite(join_node->getKind())) + { + throw Exception(ErrorCodes::INCORRECT_QUERY, + "Failed to rewrite '{}' to INNER JOIN: " + "no equi-join conditions found in WHERE clause. " + "You may set setting `cross_to_inner_join_rewrite` to `1` to allow slow CROSS JOIN for this case", + join_node->formatASTForErrorMessage()); + } + } + + if (!where_stack.empty() && where_stack.back()->get() == node.get()) + { + /// We are visiting the WHERE clause. + /// It means that we have visited current subtree and will go out of WHERE scope. + where_stack.pop_back(); + } + } + +private: + bool isEnabled() const + { + return getSettings().cross_to_inner_join_rewrite; + } + + bool forceRewrite(JoinKind kind) const + { + if (kind == JoinKind::Cross) + return false; + /// Comma join can be forced to rewrite + return getSettings().cross_to_inner_join_rewrite >= 2; + } + + QueryTreeNodePtr makeConjunction(const QueryTreeNodes & nodes) + { + if (nodes.empty()) + return nullptr; + + if (nodes.size() == 1) + return nodes.front(); + + auto function_node = std::make_shared("and"); + for (auto & node : nodes) + function_node->getArguments().getNodes().push_back(node); + + const auto & function = FunctionFactory::instance().get("and", getContext()); + function_node->resolveAsFunction(function->build(function_node->getArgumentColumns())); + return function_node; + } + + std::deque where_stack; +}; + +} + +void CrossToInnerJoinPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context) +{ + CrossToInnerJoinVisitor visitor(std::move(context)); + visitor.visit(query_tree_node); +} + +} diff --git a/src/Analyzer/Passes/CrossToInnerJoinPass.h b/src/Analyzer/Passes/CrossToInnerJoinPass.h new file mode 100644 index 00000000000..8f50f72e3d1 --- /dev/null +++ b/src/Analyzer/Passes/CrossToInnerJoinPass.h @@ -0,0 +1,24 @@ +#pragma once + +#include + +namespace DB +{ + + +/** Replace CROSS JOIN with INNER JOIN. + */ +class CrossToInnerJoinPass final : public IQueryTreePass +{ +public: + String getName() override { return "CrossToInnerJoin"; } + + String getDescription() override + { + return "Replace CROSS JOIN with INNER JOIN"; + } + + void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override; +}; + +} diff --git a/src/Analyzer/QueryTreePassManager.cpp b/src/Analyzer/QueryTreePassManager.cpp index 218e47d973f..9ba18e27f73 100644 --- a/src/Analyzer/QueryTreePassManager.cpp +++ b/src/Analyzer/QueryTreePassManager.cpp @@ -39,6 +39,7 @@ #include #include #include +#include namespace DB @@ -268,6 +269,7 @@ void addQueryTreePasses(QueryTreePassManager & manager) manager.addPass(std::make_unique()); + manager.addPass(std::make_unique()); } } From 6296109be0901691bcd970bc7038fd6d0116f589 Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 14 Feb 2023 13:12:10 +0000 Subject: [PATCH 514/717] Tests for CrossToInnerJoinPass --- .../00849_multiple_comma_join_2.reference | 232 +++--------------- .../00849_multiple_comma_join_2.sql | 108 ++++++-- .../02364_setting_cross_to_inner_rewrite.sql | 2 - .../02564_analyzer_cross_to_inner.reference | 10 + .../02564_analyzer_cross_to_inner.sql | 62 +++++ 5 files changed, 194 insertions(+), 220 deletions(-) create mode 100644 tests/queries/0_stateless/02564_analyzer_cross_to_inner.reference create mode 100644 tests/queries/0_stateless/02564_analyzer_cross_to_inner.sql diff --git a/tests/queries/0_stateless/00849_multiple_comma_join_2.reference b/tests/queries/0_stateless/00849_multiple_comma_join_2.reference index 2652a82ab54..16f228a5569 100644 --- a/tests/queries/0_stateless/00849_multiple_comma_join_2.reference +++ b/tests/queries/0_stateless/00849_multiple_comma_join_2.reference @@ -1,205 +1,33 @@ -SELECT a -FROM t1 -CROSS JOIN t2 -SELECT a -FROM t1 -ALL INNER JOIN t2 ON a = t2.a -WHERE a = t2.a -SELECT a -FROM t1 -ALL INNER JOIN t2 ON b = t2.b -WHERE b = t2.b -SELECT `--t1.a` AS `t1.a` -FROM -( - SELECT - a AS `--t1.a`, - t2.a AS `--t2.a` - FROM t1 - ALL INNER JOIN t2 ON `--t1.a` = `--t2.a` -) AS `--.s` -ALL INNER JOIN t3 ON `--t1.a` = a -WHERE (`--t1.a` = `--t2.a`) AND (`--t1.a` = a) -SELECT `--t1.a` AS `t1.a` -FROM -( - SELECT - b AS `--t1.b`, - a AS `--t1.a`, - t2.b AS `--t2.b` - FROM t1 - ALL INNER JOIN t2 ON `--t1.b` = `--t2.b` -) AS `--.s` -ALL INNER JOIN t3 ON `--t1.b` = b -WHERE (`--t1.b` = `--t2.b`) AND (`--t1.b` = b) -SELECT `--t1.a` AS `t1.a` -FROM -( - SELECT - `--t1.a`, - `--t2.a`, - a AS `--t3.a` - FROM - ( - SELECT - a AS `--t1.a`, - t2.a AS `--t2.a` - FROM t1 - ALL INNER JOIN t2 ON `--t1.a` = `--t2.a` - ) AS `--.s` - ALL INNER JOIN t3 ON `--t1.a` = `--t3.a` -) AS `--.s` -ALL INNER JOIN t4 ON `--t1.a` = a -WHERE (`--t1.a` = `--t2.a`) AND (`--t1.a` = `--t3.a`) AND (`--t1.a` = a) -SELECT `--t1.a` AS `t1.a` -FROM -( - SELECT - `--t1.b`, - `--t1.a`, - `--t2.b`, - b AS `--t3.b` - FROM - ( - SELECT - b AS `--t1.b`, - a AS `--t1.a`, - t2.b AS `--t2.b` - FROM t1 - ALL INNER JOIN t2 ON `--t1.b` = `--t2.b` - ) AS `--.s` - ALL INNER JOIN t3 ON `--t1.b` = `--t3.b` -) AS `--.s` -ALL INNER JOIN t4 ON `--t1.b` = b -WHERE (`--t1.b` = `--t2.b`) AND (`--t1.b` = `--t3.b`) AND (`--t1.b` = b) -SELECT `--t1.a` AS `t1.a` -FROM -( - SELECT - `--t1.a`, - `--t2.a`, - a AS `--t3.a` - FROM - ( - SELECT - a AS `--t1.a`, - t2.a AS `--t2.a` - FROM t1 - ALL INNER JOIN t2 ON `--t2.a` = `--t1.a` - ) AS `--.s` - ALL INNER JOIN t3 ON `--t2.a` = `--t3.a` -) AS `--.s` -ALL INNER JOIN t4 ON `--t2.a` = a -WHERE (`--t2.a` = `--t1.a`) AND (`--t2.a` = `--t3.a`) AND (`--t2.a` = a) -SELECT `--t1.a` AS `t1.a` -FROM -( - SELECT - `--t1.a`, - `--t2.a`, - a AS `--t3.a` - FROM - ( - SELECT - a AS `--t1.a`, - t2.a AS `--t2.a` - FROM t1 - CROSS JOIN t2 - ) AS `--.s` - ALL INNER JOIN t3 ON (`--t3.a` = `--t1.a`) AND (`--t3.a` = `--t2.a`) -) AS `--.s` -ALL INNER JOIN t4 ON `--t3.a` = a -WHERE (`--t3.a` = `--t1.a`) AND (`--t3.a` = `--t2.a`) AND (`--t3.a` = a) -SELECT `--t1.a` AS `t1.a` -FROM -( - SELECT - `--t1.a`, - `--t2.a`, - a AS `--t3.a` - FROM - ( - SELECT - a AS `--t1.a`, - t2.a AS `--t2.a` - FROM t1 - CROSS JOIN t2 - ) AS `--.s` - CROSS JOIN t3 -) AS `--.s` -ALL INNER JOIN t4 ON (a = `--t1.a`) AND (a = `--t2.a`) AND (a = `--t3.a`) -WHERE (a = `--t1.a`) AND (a = `--t2.a`) AND (a = `--t3.a`) -SELECT `--t1.a` AS `t1.a` -FROM -( - SELECT - `--t1.a`, - `--t2.a`, - a AS `--t3.a` - FROM - ( - SELECT - a AS `--t1.a`, - t2.a AS `--t2.a` - FROM t1 - ALL INNER JOIN t2 ON `--t1.a` = `--t2.a` - ) AS `--.s` - ALL INNER JOIN t3 ON `--t2.a` = `--t3.a` -) AS `--.s` -ALL INNER JOIN t4 ON `--t3.a` = a -WHERE (`--t1.a` = `--t2.a`) AND (`--t2.a` = `--t3.a`) AND (`--t3.a` = a) -SELECT `--t1.a` AS `t1.a` -FROM -( - SELECT `--t1.a` - FROM - ( - SELECT a AS `--t1.a` - FROM t1 - CROSS JOIN t2 - ) AS `--.s` - CROSS JOIN t3 -) AS `--.s` -CROSS JOIN t4 -SELECT `--t1.a` AS `t1.a` -FROM -( - SELECT `--t1.a` - FROM - ( - SELECT a AS `--t1.a` - FROM t1 - CROSS JOIN t2 - ) AS `--.s` - CROSS JOIN t3 -) AS `--.s` -CROSS JOIN t4 -SELECT `--t1.a` AS `t1.a` -FROM -( - SELECT a AS `--t1.a` - FROM t1 - CROSS JOIN t2 -) AS `--.s` -CROSS JOIN t3 -SELECT `--t1.a` AS `t1.a` -FROM -( - SELECT a AS `--t1.a` - FROM t1 - ALL INNER JOIN t2 USING (a) -) AS `--.s` -CROSS JOIN t3 -SELECT `--t1.a` AS `t1.a` -FROM -( - SELECT - a AS `--t1.a`, - t2.a AS `--t2.a` - FROM t1 - ALL INNER JOIN t2 ON `--t1.a` = `--t2.a` -) AS `--.s` -CROSS JOIN t3 +0 1 +0 1 +0 2 +0 2 +0 3 +0 3 +0 3 +1 2 +2 1 +0 3 +3 0 +3 0 +2 0 +1 1 +1 1 +0 1 +0 1 +0 2 +0 2 +0 3 +0 3 +0 3 +1 2 +2 1 +0 3 +3 0 +3 0 +2 0 +1 1 +1 1 SELECT * FROM t1, t2 1 1 1 1 1 1 1 \N diff --git a/tests/queries/0_stateless/00849_multiple_comma_join_2.sql b/tests/queries/0_stateless/00849_multiple_comma_join_2.sql index eb803450ff7..db8b27c4d4d 100644 --- a/tests/queries/0_stateless/00849_multiple_comma_join_2.sql +++ b/tests/queries/0_stateless/00849_multiple_comma_join_2.sql @@ -12,31 +12,107 @@ CREATE TABLE t2 (a UInt32, b Nullable(Int32)) ENGINE = Memory; CREATE TABLE t3 (a UInt32, b Nullable(Int32)) ENGINE = Memory; CREATE TABLE t4 (a UInt32, b Nullable(Int32)) ENGINE = Memory; -EXPLAIN SYNTAX SELECT t1.a FROM t1, t2; -EXPLAIN SYNTAX SELECT t1.a FROM t1, t2 WHERE t1.a = t2.a; -EXPLAIN SYNTAX SELECT t1.a FROM t1, t2 WHERE t1.b = t2.b; -EXPLAIN SYNTAX SELECT t1.a FROM t1, t2, t3 WHERE t1.a = t2.a AND t1.a = t3.a; -EXPLAIN SYNTAX SELECT t1.a FROM t1, t2, t3 WHERE t1.b = t2.b AND t1.b = t3.b; -EXPLAIN SYNTAX SELECT t1.a FROM t1, t2, t3, t4 WHERE t1.a = t2.a AND t1.a = t3.a AND t1.a = t4.a; -EXPLAIN SYNTAX SELECT t1.a FROM t1, t2, t3, t4 WHERE t1.b = t2.b AND t1.b = t3.b AND t1.b = t4.b; +SET allow_experimental_analyzer = 0; -EXPLAIN SYNTAX SELECT t1.a FROM t1, t2, t3, t4 WHERE t2.a = t1.a AND t2.a = t3.a AND t2.a = t4.a; -EXPLAIN SYNTAX SELECT t1.a FROM t1, t2, t3, t4 WHERE t3.a = t1.a AND t3.a = t2.a AND t3.a = t4.a; -EXPLAIN SYNTAX SELECT t1.a FROM t1, t2, t3, t4 WHERE t4.a = t1.a AND t4.a = t2.a AND t4.a = t3.a; -EXPLAIN SYNTAX SELECT t1.a FROM t1, t2, t3, t4 WHERE t1.a = t2.a AND t2.a = t3.a AND t3.a = t4.a; +--- EXPLAIN SYNTAX (old AST based optimization) +SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM ( + EXPLAIN SYNTAX SELECT t1.a FROM t1, t2 WHERE t1.a = t2.a); -EXPLAIN SYNTAX SELECT t1.a FROM t1, t2, t3, t4; -EXPLAIN SYNTAX SELECT t1.a FROM t1 CROSS JOIN t2 CROSS JOIN t3 CROSS JOIN t4; +SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM ( + EXPLAIN SYNTAX SELECT t1.a FROM t1, t2 WHERE t1.b = t2.b); -EXPLAIN SYNTAX SELECT t1.a FROM t1, t2 CROSS JOIN t3; -EXPLAIN SYNTAX SELECT t1.a FROM t1 JOIN t2 USING a CROSS JOIN t3; -EXPLAIN SYNTAX SELECT t1.a FROM t1 JOIN t2 ON t1.a = t2.a CROSS JOIN t3; +SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM ( + EXPLAIN SYNTAX SELECT t1.a FROM t1, t2, t3 WHERE t1.a = t2.a AND t1.a = t3.a); + +SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM ( + EXPLAIN SYNTAX SELECT t1.a FROM t1, t2, t3 WHERE t1.b = t2.b AND t1.b = t3.b); + +SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM ( + EXPLAIN SYNTAX SELECT t1.a FROM t1, t2, t3, t4 WHERE t1.a = t2.a AND t1.a = t3.a AND t1.a = t4.a); + +SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM ( + EXPLAIN SYNTAX SELECT t1.a FROM t1, t2, t3, t4 WHERE t1.b = t2.b AND t1.b = t3.b AND t1.b = t4.b); + +SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM ( + EXPLAIN SYNTAX SELECT t1.a FROM t1, t2, t3, t4 WHERE t2.a = t1.a AND t2.a = t3.a AND t2.a = t4.a); + +SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM ( + EXPLAIN SYNTAX SELECT t1.a FROM t1, t2, t3, t4 WHERE t3.a = t1.a AND t3.a = t2.a AND t3.a = t4.a); + +SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM ( + EXPLAIN SYNTAX SELECT t1.a FROM t1, t2, t3, t4 WHERE t4.a = t1.a AND t4.a = t2.a AND t4.a = t3.a); + +SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM ( + EXPLAIN SYNTAX SELECT t1.a FROM t1, t2, t3, t4 WHERE t1.a = t2.a AND t2.a = t3.a AND t3.a = t4.a); + +SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM ( + EXPLAIN SYNTAX SELECT t1.a FROM t1, t2, t3, t4); + +SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM ( + EXPLAIN SYNTAX SELECT t1.a FROM t1 CROSS JOIN t2 CROSS JOIN t3 CROSS JOIN t4); + +SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM ( + EXPLAIN SYNTAX SELECT t1.a FROM t1, t2 CROSS JOIN t3); + +SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM ( + EXPLAIN SYNTAX SELECT t1.a FROM t1 JOIN t2 USING a CROSS JOIN t3); + +SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM ( + EXPLAIN SYNTAX SELECT t1.a FROM t1 JOIN t2 ON t1.a = t2.a CROSS JOIN t3); + +--- EXPLAIN QUERY TREE +SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM ( + EXPLAIN QUERY TREE SELECT t1.a FROM t1, t2 WHERE t1.a = t2.a); + +SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM ( + EXPLAIN QUERY TREE SELECT t1.a FROM t1, t2 WHERE t1.b = t2.b); + +SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM ( + EXPLAIN QUERY TREE SELECT t1.a FROM t1, t2, t3 WHERE t1.a = t2.a AND t1.a = t3.a); + +SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM ( + EXPLAIN QUERY TREE SELECT t1.a FROM t1, t2, t3 WHERE t1.b = t2.b AND t1.b = t3.b); + +SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM ( + EXPLAIN QUERY TREE SELECT t1.a FROM t1, t2, t3, t4 WHERE t1.a = t2.a AND t1.a = t3.a AND t1.a = t4.a); + +SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM ( + EXPLAIN QUERY TREE SELECT t1.a FROM t1, t2, t3, t4 WHERE t1.b = t2.b AND t1.b = t3.b AND t1.b = t4.b); + +SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM ( + EXPLAIN QUERY TREE SELECT t1.a FROM t1, t2, t3, t4 WHERE t2.a = t1.a AND t2.a = t3.a AND t2.a = t4.a); + +SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM ( + EXPLAIN QUERY TREE SELECT t1.a FROM t1, t2, t3, t4 WHERE t3.a = t1.a AND t3.a = t2.a AND t3.a = t4.a); + +SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM ( + EXPLAIN QUERY TREE SELECT t1.a FROM t1, t2, t3, t4 WHERE t4.a = t1.a AND t4.a = t2.a AND t4.a = t3.a); + +SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM ( + EXPLAIN QUERY TREE SELECT t1.a FROM t1, t2, t3, t4 WHERE t1.a = t2.a AND t2.a = t3.a AND t3.a = t4.a); + +SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM ( + EXPLAIN QUERY TREE SELECT t1.a FROM t1, t2, t3, t4); + +SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM ( + EXPLAIN QUERY TREE SELECT t1.a FROM t1 CROSS JOIN t2 CROSS JOIN t3 CROSS JOIN t4); + +SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM ( + EXPLAIN QUERY TREE SELECT t1.a FROM t1, t2 CROSS JOIN t3); + +SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM ( + EXPLAIN QUERY TREE SELECT t1.a FROM t1 JOIN t2 USING a CROSS JOIN t3); + +SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM ( + EXPLAIN QUERY TREE SELECT t1.a FROM t1 JOIN t2 ON t1.a = t2.a CROSS JOIN t3); INSERT INTO t1 values (1,1), (2,2), (3,3), (4,4); INSERT INTO t2 values (1,1), (1, Null); INSERT INTO t3 values (1,1), (1, Null); INSERT INTO t4 values (1,1), (1, Null); +SET allow_experimental_analyzer = 1; + SELECT 'SELECT * FROM t1, t2'; SELECT * FROM t1, t2 ORDER BY t1.a, t2.b; diff --git a/tests/queries/0_stateless/02364_setting_cross_to_inner_rewrite.sql b/tests/queries/0_stateless/02364_setting_cross_to_inner_rewrite.sql index cdbac93937e..86a8414e799 100644 --- a/tests/queries/0_stateless/02364_setting_cross_to_inner_rewrite.sql +++ b/tests/queries/0_stateless/02364_setting_cross_to_inner_rewrite.sql @@ -1,5 +1,3 @@ - - DROP TABLE IF EXISTS t1; DROP TABLE IF EXISTS t2; diff --git a/tests/queries/0_stateless/02564_analyzer_cross_to_inner.reference b/tests/queries/0_stateless/02564_analyzer_cross_to_inner.reference new file mode 100644 index 00000000000..63d28b57a01 --- /dev/null +++ b/tests/queries/0_stateless/02564_analyzer_cross_to_inner.reference @@ -0,0 +1,10 @@ +5 6 5 6 5 +3 4 3 4 5 +3 4 3 4 7 +3 4 3 4 9 +5 6 5 6 5 +5 6 5 6 7 +5 6 5 6 9 +2 0 +0 2 +1 1 diff --git a/tests/queries/0_stateless/02564_analyzer_cross_to_inner.sql b/tests/queries/0_stateless/02564_analyzer_cross_to_inner.sql new file mode 100644 index 00000000000..42446252ea7 --- /dev/null +++ b/tests/queries/0_stateless/02564_analyzer_cross_to_inner.sql @@ -0,0 +1,62 @@ +SET allow_experimental_analyzer = 1; + +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; +DROP TABLE IF EXISTS t3; + +CREATE TABLE t1 (a UInt64, b UInt64) ENGINE = Memory; +INSERT INTO t1 VALUES (1, 2), (3, 4), (5, 6); + +CREATE TABLE t2 (a UInt64, b UInt64) ENGINE = Memory; +INSERT INTO t2 VALUES (3, 4), (5, 6), (7, 8); + +CREATE TABLE t3 (a UInt64, b UInt64) ENGINE = Memory; +INSERT INTO t3 VALUES (5, 6), (7, 8), (9, 10); + +SET cross_to_inner_join_rewrite = 1; + +SELECT * FROM t1, t2, (SELECT a as x from t3 where a + 1 = b ) as t3 +WHERE t1.a = if(t2.b > 0, t2.a, 0) AND t2.a = t3.x AND 1 +; + +SELECT * FROM t1, t2, (SELECT a as x from t3 where a + 1 = b ) as t3 +WHERE t1.a = if(t2.b > 0, t2.a, 0) +ORDER BY t1.a, t2.a, t3.x +; + +-- rewrite two joins +SELECT countIf(explain like '%strictness: ALL, %kind: INNER%'), countIf(explain like '%kind: COMMA%') FROM ( + EXPLAIN QUERY TREE + SELECT * FROM t1, t2, (SELECT a as x from t3 where a + 1 = b ) as t3 + WHERE t1.a = if(t2.b > 0, t2.a, 0) AND t2.a = t3.x AND 1 +) WHERE explain like '% JOIN % kind: %' +SETTINGS allow_experimental_analyzer = 0 -- workaround for viewExplain +; + +-- setting is disabled +SELECT countIf(explain like '%strictness: ALL, %kind: INNER%'), countIf(explain like '%kind: COMMA%') FROM ( + EXPLAIN QUERY TREE + SELECT * FROM t1, t2, (SELECT a as x from t3 where a + 1 = b ) as t3 + WHERE t1.a = if(t2.b > 0, t2.a, 0) AND t2.a = t3.x AND 1 + SETTINGS cross_to_inner_join_rewrite = 0 +) WHERE explain like '% JOIN % kind: %' +SETTINGS allow_experimental_analyzer = 0 -- workaround for viewExplain +; + +-- only one join can be rewritten +SELECT countIf(explain like '%strictness: ALL, %kind: INNER%'), countIf(explain like '%kind: COMMA%') FROM ( + EXPLAIN QUERY TREE + SELECT * FROM t1, t2, (SELECT a as x from t3 where a + 1 = b ) as t3 + WHERE t1.a = if(t2.b > 0, t2.a, 0) +) WHERE explain like '% JOIN % kind: %' +SETTINGS allow_experimental_analyzer = 0 -- workaround for viewExplain +; + +-- throw in force mode +SELECT * FROM t1, t2, (SELECT a as x from t3 where a + 1 = b ) as t3 +WHERE t1.a = if(t2.b > 0, t2.a, 0) +SETTINGS cross_to_inner_join_rewrite = 2; -- { serverError INCORRECT_QUERY } + +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; +DROP TABLE IF EXISTS t3; From ba04a3cc1f10df1dad9e40890d0de55d97830390 Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 20 Feb 2023 11:49:57 +0000 Subject: [PATCH 515/717] Fixes for CrossToInnerPass --- src/Analyzer/JoinNode.cpp | 7 ++-- src/Analyzer/JoinNode.h | 5 +++ src/Analyzer/Passes/CrossToInnerJoinPass.cpp | 37 ++++++++++---------- src/Analyzer/Passes/CrossToInnerJoinPass.h | 4 +++ 4 files changed, 32 insertions(+), 21 deletions(-) diff --git a/src/Analyzer/JoinNode.cpp b/src/Analyzer/JoinNode.cpp index 86cf7e90c3f..fe4dd2c5016 100644 --- a/src/Analyzer/JoinNode.cpp +++ b/src/Analyzer/JoinNode.cpp @@ -121,14 +121,15 @@ ASTPtr JoinNode::toASTImpl() const void JoinNode::crossToInner(const QueryTreeNodePtr & join_expression_) { if (kind != JoinKind::Cross && kind != JoinKind::Comma) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot rewrite join {} to inner join, expected cross", toString(kind)); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot rewrite {} to INNER JOIN, expected CROSS", toString(kind)); if (children[join_expression_child_index]) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Join expression is not empty: '{}'", children[join_expression_child_index]->formatConvertedASTForErrorMessage()); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Join expression is expected to be empty for CROSS JOIN, got '{}'", + children[join_expression_child_index]->formatConvertedASTForErrorMessage()); kind = JoinKind::Inner; strictness = JoinStrictness::All; - children[join_expression_child_index] = std::move(join_expression_); + children[join_expression_child_index] = join_expression_; } } diff --git a/src/Analyzer/JoinNode.h b/src/Analyzer/JoinNode.h index d1cd64e67b7..0d856985794 100644 --- a/src/Analyzer/JoinNode.h +++ b/src/Analyzer/JoinNode.h @@ -126,6 +126,11 @@ public: return QueryTreeNodeType::JOIN; } + /* + * Convert CROSS to INNER JOIN - changes JOIN kind and sets a new join expression + * (that was moved from WHERE clause). + * Expects the current kind to be CROSS (and join expression to be null because of that). + */ void crossToInner(const QueryTreeNodePtr & join_expression_); void dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const override; diff --git a/src/Analyzer/Passes/CrossToInnerJoinPass.cpp b/src/Analyzer/Passes/CrossToInnerJoinPass.cpp index dc03e340b15..400c760cd20 100644 --- a/src/Analyzer/Passes/CrossToInnerJoinPass.cpp +++ b/src/Analyzer/Passes/CrossToInnerJoinPass.cpp @@ -25,29 +25,30 @@ namespace ErrorCodes namespace { -using EquiCondition = std::tuple; - void exctractJoinConditions(const QueryTreeNodePtr & node, QueryTreeNodes & equi_conditions, QueryTreeNodes & other) { - if (auto * func = node->as()) + auto * func = node->as(); + if (!func) { - const auto & args = func->getArguments().getNodes(); - - if (args.size() == 2 && func->getFunctionName() == "equals") - { - equi_conditions.push_back(node); - return; - } - - if (func->getFunctionName() == "and") - { - for (auto & arg : args) - exctractJoinConditions(arg, equi_conditions, other); - return; - } + other.push_back(node); + return; } - other.push_back(node); + const auto & args = func->getArguments().getNodes(); + + if (args.size() == 2 && func->getFunctionName() == "equals") + { + equi_conditions.push_back(node); + } + else if (func->getFunctionName() == "and") + { + for (auto & arg : args) + exctractJoinConditions(arg, equi_conditions, other); + } + else + { + other.push_back(node); + } } const QueryTreeNodePtr & getEquiArgument(const QueryTreeNodePtr & cond, size_t index) diff --git a/src/Analyzer/Passes/CrossToInnerJoinPass.h b/src/Analyzer/Passes/CrossToInnerJoinPass.h index 8f50f72e3d1..127d26dc41d 100644 --- a/src/Analyzer/Passes/CrossToInnerJoinPass.h +++ b/src/Analyzer/Passes/CrossToInnerJoinPass.h @@ -7,6 +7,10 @@ namespace DB /** Replace CROSS JOIN with INNER JOIN. + * Example: + * SELECT * FROM t1 CROSS JOIN t2 WHERE t1.a = t2.a AND t1.b > 10 AND t2.b = t2.c + * We can move equality condition to ON section of INNER JOIN: + * SELECT * FROM t1 INNER JOIN t2 ON t1.a = t2.a WHERE t1.b > 10 AND t2.b = t2.c */ class CrossToInnerJoinPass final : public IQueryTreePass { From 7cf7d31b4b4b24c98d0f5a16ee940ca7b90483d4 Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 20 Feb 2023 11:50:28 +0000 Subject: [PATCH 516/717] Update 02564_analyzer_cross_to_inner --- .../02564_analyzer_cross_to_inner.reference | 201 +++++++++++++++++- .../02564_analyzer_cross_to_inner.sql | 40 ++-- 2 files changed, 212 insertions(+), 29 deletions(-) diff --git a/tests/queries/0_stateless/02564_analyzer_cross_to_inner.reference b/tests/queries/0_stateless/02564_analyzer_cross_to_inner.reference index 63d28b57a01..e4d7ff55b86 100644 --- a/tests/queries/0_stateless/02564_analyzer_cross_to_inner.reference +++ b/tests/queries/0_stateless/02564_analyzer_cross_to_inner.reference @@ -5,6 +5,201 @@ 5 6 5 6 5 5 6 5 6 7 5 6 5 6 9 -2 0 -0 2 -1 1 +-- { echoOn } + +EXPLAIN QUERY TREE +SELECT * FROM t1, t2, (SELECT a as x from t3 where a + 1 = b ) as t3 +WHERE t1.a = if(t2.b > 0, t2.a, 0) AND t2.a = t3.x AND 1; +QUERY id: 0 + PROJECTION COLUMNS + t1.a UInt64 + t1.b UInt64 + t2.a UInt64 + t2.b UInt64 + x UInt64 + PROJECTION + LIST id: 1, nodes: 5 + COLUMN id: 2, column_name: a, result_type: UInt64, source_id: 3 + COLUMN id: 4, column_name: b, result_type: UInt64, source_id: 3 + COLUMN id: 5, column_name: a, result_type: UInt64, source_id: 6 + COLUMN id: 7, column_name: b, result_type: UInt64, source_id: 6 + COLUMN id: 8, column_name: x, result_type: UInt64, source_id: 9 + JOIN TREE + JOIN id: 10, strictness: ALL, kind: INNER + LEFT TABLE EXPRESSION + JOIN id: 11, strictness: ALL, kind: INNER + LEFT TABLE EXPRESSION + TABLE id: 3, table_name: default.t1 + RIGHT TABLE EXPRESSION + TABLE id: 6, table_name: default.t2 + JOIN EXPRESSION + FUNCTION id: 12, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 13, nodes: 2 + COLUMN id: 14, column_name: a, result_type: UInt64, source_id: 3 + FUNCTION id: 15, function_name: if, function_type: ordinary, result_type: UInt64 + ARGUMENTS + LIST id: 16, nodes: 3 + FUNCTION id: 17, function_name: greater, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 18, nodes: 2 + COLUMN id: 19, column_name: b, result_type: UInt64, source_id: 6 + CONSTANT id: 20, constant_value: UInt64_0, constant_value_type: UInt8 + COLUMN id: 21, column_name: a, result_type: UInt64, source_id: 6 + CONSTANT id: 22, constant_value: UInt64_0, constant_value_type: UInt8 + RIGHT TABLE EXPRESSION + QUERY id: 9, alias: t3, is_subquery: 1 + PROJECTION COLUMNS + x UInt64 + PROJECTION + LIST id: 23, nodes: 1 + COLUMN id: 24, column_name: a, result_type: UInt64, source_id: 25 + JOIN TREE + TABLE id: 25, table_name: default.t3 + WHERE + FUNCTION id: 26, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 27, nodes: 2 + FUNCTION id: 28, function_name: plus, function_type: ordinary, result_type: UInt64 + ARGUMENTS + LIST id: 29, nodes: 2 + COLUMN id: 24, column_name: a, result_type: UInt64, source_id: 25 + CONSTANT id: 30, constant_value: UInt64_1, constant_value_type: UInt8 + COLUMN id: 31, column_name: b, result_type: UInt64, source_id: 25 + JOIN EXPRESSION + FUNCTION id: 32, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 33, nodes: 2 + COLUMN id: 21, column_name: a, result_type: UInt64, source_id: 6 + COLUMN id: 34, column_name: x, result_type: UInt64, source_id: 9 + WHERE + CONSTANT id: 35, constant_value: UInt64_1, constant_value_type: UInt8 +EXPLAIN QUERY TREE +SELECT * FROM t1, t2, (SELECT a as x from t3 where a + 1 = b ) as t3 +WHERE t1.a = if(t2.b > 0, t2.a, 0) AND t2.a = t3.x AND 1 +SETTINGS cross_to_inner_join_rewrite = 0; +QUERY id: 0 + PROJECTION COLUMNS + t1.a UInt64 + t1.b UInt64 + t2.a UInt64 + t2.b UInt64 + x UInt64 + PROJECTION + LIST id: 1, nodes: 5 + COLUMN id: 2, column_name: a, result_type: UInt64, source_id: 3 + COLUMN id: 4, column_name: b, result_type: UInt64, source_id: 3 + COLUMN id: 5, column_name: a, result_type: UInt64, source_id: 6 + COLUMN id: 7, column_name: b, result_type: UInt64, source_id: 6 + COLUMN id: 8, column_name: x, result_type: UInt64, source_id: 9 + JOIN TREE + JOIN id: 10, kind: COMMA + LEFT TABLE EXPRESSION + JOIN id: 11, kind: COMMA + LEFT TABLE EXPRESSION + TABLE id: 3, table_name: default.t1 + RIGHT TABLE EXPRESSION + TABLE id: 6, table_name: default.t2 + RIGHT TABLE EXPRESSION + QUERY id: 9, alias: t3, is_subquery: 1 + PROJECTION COLUMNS + x UInt64 + PROJECTION + LIST id: 12, nodes: 1 + COLUMN id: 13, column_name: a, result_type: UInt64, source_id: 14 + JOIN TREE + TABLE id: 14, table_name: default.t3 + WHERE + FUNCTION id: 15, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 16, nodes: 2 + FUNCTION id: 17, function_name: plus, function_type: ordinary, result_type: UInt64 + ARGUMENTS + LIST id: 18, nodes: 2 + COLUMN id: 13, column_name: a, result_type: UInt64, source_id: 14 + CONSTANT id: 19, constant_value: UInt64_1, constant_value_type: UInt8 + COLUMN id: 20, column_name: b, result_type: UInt64, source_id: 14 + WHERE + FUNCTION id: 21, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 22, nodes: 3 + FUNCTION id: 23, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 24, nodes: 2 + COLUMN id: 25, column_name: a, result_type: UInt64, source_id: 3 + FUNCTION id: 26, function_name: if, function_type: ordinary, result_type: UInt64 + ARGUMENTS + LIST id: 27, nodes: 3 + FUNCTION id: 28, function_name: greater, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 29, nodes: 2 + COLUMN id: 30, column_name: b, result_type: UInt64, source_id: 6 + CONSTANT id: 31, constant_value: UInt64_0, constant_value_type: UInt8 + COLUMN id: 32, column_name: a, result_type: UInt64, source_id: 6 + CONSTANT id: 33, constant_value: UInt64_0, constant_value_type: UInt8 + FUNCTION id: 34, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 35, nodes: 2 + COLUMN id: 32, column_name: a, result_type: UInt64, source_id: 6 + COLUMN id: 36, column_name: x, result_type: UInt64, source_id: 9 + CONSTANT id: 37, constant_value: UInt64_1, constant_value_type: UInt8 + SETTINGS cross_to_inner_join_rewrite=0 +EXPLAIN QUERY TREE +SELECT * FROM t1, t2, (SELECT a as x from t3 where a + 1 = b ) as t3 +WHERE t1.a = if(t2.b > 0, t2.a, 0); +QUERY id: 0 + PROJECTION COLUMNS + t1.a UInt64 + t1.b UInt64 + t2.a UInt64 + t2.b UInt64 + x UInt64 + PROJECTION + LIST id: 1, nodes: 5 + COLUMN id: 2, column_name: a, result_type: UInt64, source_id: 3 + COLUMN id: 4, column_name: b, result_type: UInt64, source_id: 3 + COLUMN id: 5, column_name: a, result_type: UInt64, source_id: 6 + COLUMN id: 7, column_name: b, result_type: UInt64, source_id: 6 + COLUMN id: 8, column_name: x, result_type: UInt64, source_id: 9 + JOIN TREE + JOIN id: 10, kind: COMMA + LEFT TABLE EXPRESSION + JOIN id: 11, strictness: ALL, kind: INNER + LEFT TABLE EXPRESSION + TABLE id: 3, table_name: default.t1 + RIGHT TABLE EXPRESSION + TABLE id: 6, table_name: default.t2 + JOIN EXPRESSION + FUNCTION id: 12, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 13, nodes: 2 + COLUMN id: 14, column_name: a, result_type: UInt64, source_id: 3 + FUNCTION id: 15, function_name: if, function_type: ordinary, result_type: UInt64 + ARGUMENTS + LIST id: 16, nodes: 3 + FUNCTION id: 17, function_name: greater, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 18, nodes: 2 + COLUMN id: 19, column_name: b, result_type: UInt64, source_id: 6 + CONSTANT id: 20, constant_value: UInt64_0, constant_value_type: UInt8 + COLUMN id: 21, column_name: a, result_type: UInt64, source_id: 6 + CONSTANT id: 22, constant_value: UInt64_0, constant_value_type: UInt8 + RIGHT TABLE EXPRESSION + QUERY id: 9, alias: t3, is_subquery: 1 + PROJECTION COLUMNS + x UInt64 + PROJECTION + LIST id: 23, nodes: 1 + COLUMN id: 24, column_name: a, result_type: UInt64, source_id: 25 + JOIN TREE + TABLE id: 25, table_name: default.t3 + WHERE + FUNCTION id: 26, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 27, nodes: 2 + FUNCTION id: 28, function_name: plus, function_type: ordinary, result_type: UInt64 + ARGUMENTS + LIST id: 29, nodes: 2 + COLUMN id: 24, column_name: a, result_type: UInt64, source_id: 25 + CONSTANT id: 30, constant_value: UInt64_1, constant_value_type: UInt8 + COLUMN id: 31, column_name: b, result_type: UInt64, source_id: 25 diff --git a/tests/queries/0_stateless/02564_analyzer_cross_to_inner.sql b/tests/queries/0_stateless/02564_analyzer_cross_to_inner.sql index 42446252ea7..a83cd238982 100644 --- a/tests/queries/0_stateless/02564_analyzer_cross_to_inner.sql +++ b/tests/queries/0_stateless/02564_analyzer_cross_to_inner.sql @@ -24,35 +24,23 @@ WHERE t1.a = if(t2.b > 0, t2.a, 0) ORDER BY t1.a, t2.a, t3.x ; --- rewrite two joins -SELECT countIf(explain like '%strictness: ALL, %kind: INNER%'), countIf(explain like '%kind: COMMA%') FROM ( - EXPLAIN QUERY TREE - SELECT * FROM t1, t2, (SELECT a as x from t3 where a + 1 = b ) as t3 - WHERE t1.a = if(t2.b > 0, t2.a, 0) AND t2.a = t3.x AND 1 -) WHERE explain like '% JOIN % kind: %' -SETTINGS allow_experimental_analyzer = 0 -- workaround for viewExplain -; +-- { echoOn } --- setting is disabled -SELECT countIf(explain like '%strictness: ALL, %kind: INNER%'), countIf(explain like '%kind: COMMA%') FROM ( - EXPLAIN QUERY TREE - SELECT * FROM t1, t2, (SELECT a as x from t3 where a + 1 = b ) as t3 - WHERE t1.a = if(t2.b > 0, t2.a, 0) AND t2.a = t3.x AND 1 - SETTINGS cross_to_inner_join_rewrite = 0 -) WHERE explain like '% JOIN % kind: %' -SETTINGS allow_experimental_analyzer = 0 -- workaround for viewExplain -; +EXPLAIN QUERY TREE +SELECT * FROM t1, t2, (SELECT a as x from t3 where a + 1 = b ) as t3 +WHERE t1.a = if(t2.b > 0, t2.a, 0) AND t2.a = t3.x AND 1; --- only one join can be rewritten -SELECT countIf(explain like '%strictness: ALL, %kind: INNER%'), countIf(explain like '%kind: COMMA%') FROM ( - EXPLAIN QUERY TREE - SELECT * FROM t1, t2, (SELECT a as x from t3 where a + 1 = b ) as t3 - WHERE t1.a = if(t2.b > 0, t2.a, 0) -) WHERE explain like '% JOIN % kind: %' -SETTINGS allow_experimental_analyzer = 0 -- workaround for viewExplain -; +EXPLAIN QUERY TREE +SELECT * FROM t1, t2, (SELECT a as x from t3 where a + 1 = b ) as t3 +WHERE t1.a = if(t2.b > 0, t2.a, 0) AND t2.a = t3.x AND 1 +SETTINGS cross_to_inner_join_rewrite = 0; + +EXPLAIN QUERY TREE +SELECT * FROM t1, t2, (SELECT a as x from t3 where a + 1 = b ) as t3 +WHERE t1.a = if(t2.b > 0, t2.a, 0); + +-- { echoOff } --- throw in force mode SELECT * FROM t1, t2, (SELECT a as x from t3 where a + 1 = b ) as t3 WHERE t1.a = if(t2.b > 0, t2.a, 0) SETTINGS cross_to_inner_join_rewrite = 2; -- { serverError INCORRECT_QUERY } From 08b0e3c6309f520cdf1dcc97fd205ba5d4ffbd19 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 21 Feb 2023 18:27:37 +0100 Subject: [PATCH 517/717] Fix style check --- tests/integration/test_named_collections/test.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_named_collections/test.py b/tests/integration/test_named_collections/test.py index 612b894461b..ba62880e9de 100644 --- a/tests/integration/test_named_collections/test.py +++ b/tests/integration/test_named_collections/test.py @@ -102,7 +102,10 @@ def test_access(cluster): ["bash", "-c", f"cat /etc/clickhouse-server/users.d/users.xml"] ) node.restart_clickhouse() - assert node.query("select collection['key1'] from system.named_collections").strip() == "value1" + assert ( + node.query("select collection['key1'] from system.named_collections").strip() + == "value1" + ) replace_in_users_config( node, "show_named_collections_secrets>1", "show_named_collections_secrets>0" ) @@ -110,7 +113,10 @@ def test_access(cluster): ["bash", "-c", f"cat /etc/clickhouse-server/users.d/users.xml"] ) node.restart_clickhouse() - assert node.query("select collection['key1'] from system.named_collections").strip() == "[HIDDEN]" + assert ( + node.query("select collection['key1'] from system.named_collections").strip() + == "[HIDDEN]" + ) replace_in_users_config( node, "show_named_collections_secrets>0", "show_named_collections_secrets>1" ) @@ -118,7 +124,10 @@ def test_access(cluster): ["bash", "-c", f"cat /etc/clickhouse-server/users.d/users.xml"] ) node.restart_clickhouse() - assert node.query("select collection['key1'] from system.named_collections").strip() == "value1" + assert ( + node.query("select collection['key1'] from system.named_collections").strip() + == "value1" + ) def test_config_reload(cluster): From bf9f1663bb73bd969fcb35242b060c7e55a61024 Mon Sep 17 00:00:00 2001 From: avogar Date: Tue, 21 Feb 2023 18:15:16 +0000 Subject: [PATCH 518/717] Fix totals and extremes with constants in clickhouse-local --- src/Client/ClientBase.cpp | 4 ++-- .../02556_local_with_totals_and_extremes.reference | 6 ++++++ .../0_stateless/02556_local_with_totals_and_extremes.sh | 8 ++++++++ 3 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/02556_local_with_totals_and_extremes.reference create mode 100755 tests/queries/0_stateless/02556_local_with_totals_and_extremes.sh diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index bc8c43af8c6..96aff9aa304 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -481,14 +481,14 @@ void ClientBase::onLogData(Block & block) void ClientBase::onTotals(Block & block, ASTPtr parsed_query) { initOutputFormat(block, parsed_query); - output_format->setTotals(block); + output_format->setTotals(materializeBlock(block)); } void ClientBase::onExtremes(Block & block, ASTPtr parsed_query) { initOutputFormat(block, parsed_query); - output_format->setExtremes(block); + output_format->setExtremes(materializeBlock(block)); } diff --git a/tests/queries/0_stateless/02556_local_with_totals_and_extremes.reference b/tests/queries/0_stateless/02556_local_with_totals_and_extremes.reference new file mode 100644 index 00000000000..0b9836e530b --- /dev/null +++ b/tests/queries/0_stateless/02556_local_with_totals_and_extremes.reference @@ -0,0 +1,6 @@ +1,1 + +1,1 + +1,1 +1,1 diff --git a/tests/queries/0_stateless/02556_local_with_totals_and_extremes.sh b/tests/queries/0_stateless/02556_local_with_totals_and_extremes.sh new file mode 100755 index 00000000000..ef31b3855cd --- /dev/null +++ b/tests/queries/0_stateless/02556_local_with_totals_and_extremes.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +$CLICKHOUSE_LOCAL -q "SELECT 1, sum(1) with totals format CSV settings extremes=1" + From 1f0ab8d427b7819baf13176234ae94acbd9addd7 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 21 Feb 2023 19:24:01 +0100 Subject: [PATCH 519/717] Hide disk setting arguemtns --- src/Core/Field.h | 6 +- src/Parsers/ASTSetQuery.cpp | 6 +- src/Parsers/FieldFromAST.cpp | 55 +++++++++++++++++++ src/Parsers/FieldFromAST.h | 4 +- ...54_create_table_with_custom_disk.reference | 2 +- 5 files changed, 67 insertions(+), 6 deletions(-) diff --git a/src/Core/Field.h b/src/Core/Field.h index 95ce43ccd44..2e772a64afc 100644 --- a/src/Core/Field.h +++ b/src/Core/Field.h @@ -108,7 +108,8 @@ struct CustomType { virtual ~CustomTypeImpl() = default; virtual const char * getTypeName() const = 0; - virtual String toString() const = 0; + virtual String toString(bool show_secrets) const = 0; + virtual bool isSecret() const = 0; virtual bool operator < (const CustomTypeImpl &) const = 0; virtual bool operator <= (const CustomTypeImpl &) const = 0; @@ -120,8 +121,9 @@ struct CustomType CustomType() = default; explicit CustomType(std::shared_ptr impl_) : impl(impl_) {} + bool isSecret() const { return impl->isSecret(); } const char * getTypeName() const { return impl->getTypeName(); } - String toString() const { return impl->toString(); } + String toString(bool show_secrets = true) const { return impl->toString(show_secrets); } const CustomTypeImpl & getImpl() { return *impl; } bool operator < (const CustomType & rhs) const { return *impl < *rhs.impl; } diff --git a/src/Parsers/ASTSetQuery.cpp b/src/Parsers/ASTSetQuery.cpp index 26420f4988c..0b8d76dbb89 100644 --- a/src/Parsers/ASTSetQuery.cpp +++ b/src/Parsers/ASTSetQuery.cpp @@ -34,7 +34,11 @@ void ASTSetQuery::formatImpl(const FormatSettings & format, FormatState &, Forma first = false; formatSettingName(change.name, format.ostr); - format.ostr << " = " << applyVisitor(FieldVisitorToString(), change.value); + CustomType custom; + if (!format.show_secrets && change.value.tryGet(custom) && custom.isSecret()) + format.ostr << " = " << custom.toString(false); + else + format.ostr << " = " << applyVisitor(FieldVisitorToString(), change.value); } for (const auto & setting_name : default_settings) diff --git a/src/Parsers/FieldFromAST.cpp b/src/Parsers/FieldFromAST.cpp index 7b7302696ed..5889699c081 100644 --- a/src/Parsers/FieldFromAST.cpp +++ b/src/Parsers/FieldFromAST.cpp @@ -1,10 +1,18 @@ #include +#include +#include +#include +#include +#include +#include + namespace DB { namespace ErrorCodes { extern const int LOGICAL_ERROR; + extern const int BAD_ARGUMENTS; } Field createFieldFromAST(ASTPtr ast) @@ -17,4 +25,51 @@ Field createFieldFromAST(ASTPtr ast) throw Exception(ErrorCodes::LOGICAL_ERROR, "Method {} not implemented for {}", method, getTypeName()); } +bool FieldFromASTImpl::isSecret() const +{ + return isDiskFunction(ast); +} + +String FieldFromASTImpl::toString(bool show_secrets) const +{ + if (!show_secrets && isDiskFunction(ast)) + { + auto hidden = ast->clone(); + auto & disk_function = assert_cast(*hidden); + auto * disk_function_args_expr = assert_cast(disk_function.arguments.get()); + auto & disk_function_args = disk_function_args_expr->children; + + auto is_secret_arg = [](const std::string & arg_name) + { + return arg_name != "type"; + }; + + for (auto & arg : disk_function_args) + { + auto * setting_function = arg->as(); + if (!setting_function || setting_function->name != "equals") + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad format: expected equals function"); + + auto * function_args_expr = assert_cast(setting_function->arguments.get()); + if (!function_args_expr) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad format: expected arguments"); + + auto & function_args = function_args_expr->children; + if (function_args.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad format: expected non zero number of arguments"); + + auto * key_identifier = function_args[0]->as(); + if (!key_identifier) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad format: expected Identifier"); + + const std::string & key = key_identifier->name(); + if (is_secret_arg(key)) + function_args[1] = std::make_shared("[HIDDEN]"); + } + return serializeAST(*hidden); + } + + return serializeAST(*ast); +} + } diff --git a/src/Parsers/FieldFromAST.h b/src/Parsers/FieldFromAST.h index 132f7e3e705..a69c086a170 100644 --- a/src/Parsers/FieldFromAST.h +++ b/src/Parsers/FieldFromAST.h @@ -1,7 +1,6 @@ #pragma once #include #include -#include namespace DB { @@ -13,7 +12,8 @@ struct FieldFromASTImpl : public CustomType::CustomTypeImpl explicit FieldFromASTImpl(ASTPtr ast_) : ast(ast_) {} const char * getTypeName() const override { return name; } - String toString() const override { return serializeAST(*ast); } + String toString(bool show_secrets) const override; + bool isSecret() const override; [[noreturn]] void throwNotImplemented(std::string_view method) const; diff --git a/tests/queries/0_stateless/02454_create_table_with_custom_disk.reference b/tests/queries/0_stateless/02454_create_table_with_custom_disk.reference index 378722b5166..1d8610c59c9 100644 --- a/tests/queries/0_stateless/02454_create_table_with_custom_disk.reference +++ b/tests/queries/0_stateless/02454_create_table_with_custom_disk.reference @@ -6,6 +6,6 @@ ENGINE = MergeTree ORDER BY tuple() SETTINGS disk = disk(type = local, path = \'/var/lib/clickhouse/disks/local/\') 100 -CREATE TABLE default.test\n(\n `a` Int32\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS disk = disk(type = local, path = \'/var/lib/clickhouse/disks/local/\'), index_granularity = 8192 +CREATE TABLE default.test\n(\n `a` Int32\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS disk = disk(type = local, path = \'[HIDDEN]\'), index_granularity = 8192 a Int32 200 From 3ab54ac0adb6c77f0ca6e04dc7f4f0f746b0c337 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 21 Feb 2023 19:24:10 +0100 Subject: [PATCH 520/717] Fix test output --- .../00980_merge_alter_settings.reference | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/queries/0_stateless/00980_merge_alter_settings.reference b/tests/queries/0_stateless/00980_merge_alter_settings.reference index 706b64184ca..a29577f362c 100644 --- a/tests/queries/0_stateless/00980_merge_alter_settings.reference +++ b/tests/queries/0_stateless/00980_merge_alter_settings.reference @@ -1,12 +1,12 @@ -CREATE TABLE default.table_for_alter\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096, index_granularity_bytes = '10Mi' -CREATE TABLE default.table_for_alter\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096, index_granularity_bytes = '10Mi', parts_to_throw_insert = 1, parts_to_delay_insert = 1 -CREATE TABLE default.table_for_alter\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096, index_granularity_bytes = '10Mi', parts_to_throw_insert = 100, parts_to_delay_insert = 100 +CREATE TABLE default.table_for_alter\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096, index_granularity_bytes = \'10Mi\' +CREATE TABLE default.table_for_alter\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096, index_granularity_bytes = \'10Mi\', parts_to_throw_insert = 1, parts_to_delay_insert = 1 +CREATE TABLE default.table_for_alter\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096, index_granularity_bytes = \'10Mi\', parts_to_throw_insert = 100, parts_to_delay_insert = 100 2 -CREATE TABLE default.table_for_alter\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096, index_granularity_bytes = '10Mi', parts_to_throw_insert = 100, parts_to_delay_insert = 100, check_delay_period = 30 -CREATE TABLE default.table_for_alter\n(\n `id` UInt64,\n `Data` String,\n `Data2` UInt64\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096, index_granularity_bytes = '10Mi', parts_to_throw_insert = 100, parts_to_delay_insert = 100, check_delay_period = 15 -CREATE TABLE default.table_for_reset_setting\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096, index_granularity_bytes = '10Mi' -CREATE TABLE default.table_for_reset_setting\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096, parts_to_throw_insert = 1, parts_to_delay_insert = 1 -CREATE TABLE default.table_for_reset_setting\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096, index_granularity_bytes = '10Mi' -CREATE TABLE default.table_for_reset_setting\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096, index_granularity_bytes = '10Mi' -CREATE TABLE default.table_for_reset_setting\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096, index_granularity_bytes = '10Mi', merge_with_ttl_timeout = 300, max_concurrent_queries = 1 -CREATE TABLE default.table_for_reset_setting\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096, index_granularity_bytes = '10Mi' +CREATE TABLE default.table_for_alter\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096, index_granularity_bytes = \'10Mi\', parts_to_throw_insert = 100, parts_to_delay_insert = 100, check_delay_period = 30 +CREATE TABLE default.table_for_alter\n(\n `id` UInt64,\n `Data` String,\n `Data2` UInt64\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096, index_granularity_bytes = \'10Mi\', parts_to_throw_insert = 100, parts_to_delay_insert = 100, check_delay_period = 15 +CREATE TABLE default.table_for_reset_setting\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096, index_granularity_bytes = \'10Mi\' +CREATE TABLE default.table_for_reset_setting\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096, index_granularity_bytes = \'10Mi\', parts_to_throw_insert = 1, parts_to_delay_insert = 1 +CREATE TABLE default.table_for_reset_setting\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096, index_granularity_bytes = \'10Mi\' +CREATE TABLE default.table_for_reset_setting\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096, index_granularity_bytes = \'10Mi\' +CREATE TABLE default.table_for_reset_setting\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096, index_granularity_bytes = \'10Mi\', merge_with_ttl_timeout = 300, max_concurrent_queries = 1 +CREATE TABLE default.table_for_reset_setting\n(\n `id` UInt64,\n `Data` String\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 4096, index_granularity_bytes = \'10Mi\' From d9cff3a5e8f19b58379830c435ceb04916fd15e4 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 21 Feb 2023 13:52:28 -0500 Subject: [PATCH 521/717] fix layout issues in dashboard.html --- programs/server/dashboard.html | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/programs/server/dashboard.html b/programs/server/dashboard.html index a91040b2701..fa940e01ad5 100644 --- a/programs/server/dashboard.html +++ b/programs/server/dashboard.html @@ -116,21 +116,29 @@ width: 50%; display: flex; - flex-flow: row wrap; + flex-flow: column nowrap; } .unconnected #url { width: 100%; } - .unconnected #user { + #user { + margin-right: 0.25rem; width: 50%; } - .unconnected #password { + #password { width: 49.5%; } .unconnected input { margin-bottom: 5px; } + #username-password { + width: 100%; + + display: flex; + flex-flow: row nowrap; + } + .inputs #chart-params { display: block; } @@ -142,7 +150,7 @@ #connection-params { margin-bottom: 0.5rem; display: grid; - grid-template-columns: auto 15% 15%; + grid-template-columns: 69.77% 30%; column-gap: 0.25rem; } @@ -339,8 +347,10 @@
- - +
+ + +
From dcf8aeab8fa221484836fa017b9d93e30277544b Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 21 Feb 2023 22:16:05 +0300 Subject: [PATCH 522/717] Update stress --- docker/test/stress/stress | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/test/stress/stress b/docker/test/stress/stress index e23d5988918..c142da1be9e 100755 --- a/docker/test/stress/stress +++ b/docker/test/stress/stress @@ -46,7 +46,8 @@ def get_options(i, backward_compatibility_check): if i % 2 == 1 and not backward_compatibility_check: client_options.append("group_by_use_nulls=1") - if i == 12: # 12 % 3 == 0, so it's Atomic database + # 12 % 3 == 0, so it's Atomic database + if i == 12 and not backward_compatibility_check: client_options.append("implicit_transaction=1") client_options.append("throw_on_unsupported_query_inside_transaction=0") From cfef911f0d1be82e9a36edce0234e78438fd33c0 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 22 Feb 2023 00:32:55 +0300 Subject: [PATCH 523/717] Update 01710_normal_projections.sh --- tests/queries/0_stateless/01710_normal_projections.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01710_normal_projections.sh b/tests/queries/0_stateless/01710_normal_projections.sh index 8ee3f41ea28..3f2114b9a2b 100755 --- a/tests/queries/0_stateless/01710_normal_projections.sh +++ b/tests/queries/0_stateless/01710_normal_projections.sh @@ -4,7 +4,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -q "CREATE TABLE test_sort_proj (x UInt32, y UInt32, PROJECTION p (SELECT x, y ORDER BY y)) ENGINE = MergeTree ORDER BY x SETTINGS index_granularity=8192" +$CLICKHOUSE_CLIENT -q "CREATE TABLE test_sort_proj (x UInt32, y UInt32, PROJECTION p (SELECT x, y ORDER BY y)) ENGINE = MergeTree ORDER BY x SETTINGS index_granularity=8192, index_granularity_bytes='10Mi'" $CLICKHOUSE_CLIENT -q "insert into test_sort_proj select number, toUInt32(-number - 1) from numbers(100)" echo "select where x < 10" From c009c2f4cbf7333407734f92780d16c8a0aed908 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 22 Feb 2023 00:50:21 +0300 Subject: [PATCH 524/717] Update test_ttl_move_memory_usage.py --- .../test_s3_zero_copy_ttl/test_ttl_move_memory_usage.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/integration/test_s3_zero_copy_ttl/test_ttl_move_memory_usage.py b/tests/integration/test_s3_zero_copy_ttl/test_ttl_move_memory_usage.py index 29177b6a67b..5fbe426074f 100644 --- a/tests/integration/test_s3_zero_copy_ttl/test_ttl_move_memory_usage.py +++ b/tests/integration/test_s3_zero_copy_ttl/test_ttl_move_memory_usage.py @@ -2,6 +2,12 @@ import time import pytest + +# FIXME This test is too flaky +# https://github.com/ClickHouse/ClickHouse/issues/45887 + +pytestmark = pytest.mark.skip + from helpers.cluster import ClickHouseCluster From 8232966b9e923f12e711ac4bb036949c61eb87db Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Tue, 21 Feb 2023 22:02:23 +0000 Subject: [PATCH 525/717] Add a comment --- src/Interpreters/AsynchronousInsertQueue.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Interpreters/AsynchronousInsertQueue.h b/src/Interpreters/AsynchronousInsertQueue.h index b8c7d9d285b..4cda5078801 100644 --- a/src/Interpreters/AsynchronousInsertQueue.h +++ b/src/Interpreters/AsynchronousInsertQueue.h @@ -91,6 +91,9 @@ private: ~InsertData() { auto it = entries.begin(); + // Entries must be destroyed in context of user who runs async insert. + // Each entry in the list may correspond to a different user, + // so we need to switch current thread's MemoryTracker parent on each iteration. while (it != entries.end()) { UserMemoryTrackerSwitcher switcher((*it)->user_memory_tracker); From ad1e5f391888cc5b1ac02b89fbdb19ecc201859d Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Tue, 21 Feb 2023 22:06:17 +0000 Subject: [PATCH 526/717] Review fixes --- src/Common/CurrentThread.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Common/CurrentThread.cpp b/src/Common/CurrentThread.cpp index 95b316671a4..f5cc9658f7a 100644 --- a/src/Common/CurrentThread.cpp +++ b/src/Common/CurrentThread.cpp @@ -115,10 +115,11 @@ MemoryTracker * CurrentThread::getUserMemoryTracker() if (unlikely(!current_thread)) return nullptr; - if (auto group = current_thread->getThreadGroup()) - return group->memory_tracker.getParent(); + auto * tracker = current_thread->memory_tracker.getParent(); + while (tracker && tracker->level != VariableContext::User) + tracker = tracker->getParent(); - return nullptr; + return tracker; } void CurrentThread::flushUntrackedMemory() From 0bf0fe488eb4b6637e863c95c2ed8f07c4509ec8 Mon Sep 17 00:00:00 2001 From: AVMusorin Date: Sun, 19 Feb 2023 21:09:40 +0100 Subject: [PATCH 527/717] added last_exception_time column into distribution_queue table --- src/Storages/Distributed/DirectoryMonitor.cpp | 1 + src/Storages/Distributed/DirectoryMonitor.h | 1 + src/Storages/System/StorageSystemDistributionQueue.cpp | 2 ++ .../0_stateless/01555_system_distribution_queue_mask.reference | 2 +- .../0_stateless/01555_system_distribution_queue_mask.sql | 2 +- .../0_stateless/02117_show_create_table_system.reference | 3 ++- 6 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/Storages/Distributed/DirectoryMonitor.cpp b/src/Storages/Distributed/DirectoryMonitor.cpp index 7aa7aac2ef3..cb6659e59ce 100644 --- a/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/src/Storages/Distributed/DirectoryMonitor.cpp @@ -465,6 +465,7 @@ void StorageDistributedDirectoryMonitor::run() tryLogCurrentException(getLoggerName().data()); status.last_exception = std::current_exception(); + status.last_exception_time = std::chrono::system_clock::now(); } } else diff --git a/src/Storages/Distributed/DirectoryMonitor.h b/src/Storages/Distributed/DirectoryMonitor.h index 7015fca0311..030d6acf6e2 100644 --- a/src/Storages/Distributed/DirectoryMonitor.h +++ b/src/Storages/Distributed/DirectoryMonitor.h @@ -58,6 +58,7 @@ public: struct InternalStatus { std::exception_ptr last_exception; + std::chrono::system_clock::time_point last_exception_time; size_t error_count = 0; diff --git a/src/Storages/System/StorageSystemDistributionQueue.cpp b/src/Storages/System/StorageSystemDistributionQueue.cpp index 5297c4eb93c..34cff7df65d 100644 --- a/src/Storages/System/StorageSystemDistributionQueue.cpp +++ b/src/Storages/System/StorageSystemDistributionQueue.cpp @@ -101,6 +101,7 @@ NamesAndTypesList StorageSystemDistributionQueue::getNamesAndTypes() { "broken_data_files", std::make_shared() }, { "broken_data_compressed_bytes", std::make_shared() }, { "last_exception", std::make_shared() }, + { "last_exception_time", std::make_shared() }, }; } @@ -190,6 +191,7 @@ void StorageSystemDistributionQueue::fillData(MutableColumns & res_columns, Cont res_columns[col_num++]->insert(getExceptionMessage(status.last_exception, false)); else res_columns[col_num++]->insertDefault(); + res_columns[col_num++]->insert(static_cast(std::chrono::system_clock::to_time_t(status.last_exception_time))); } } } diff --git a/tests/queries/0_stateless/01555_system_distribution_queue_mask.reference b/tests/queries/0_stateless/01555_system_distribution_queue_mask.reference index bd0eac10816..745160a517e 100644 --- a/tests/queries/0_stateless/01555_system_distribution_queue_mask.reference +++ b/tests/queries/0_stateless/01555_system_distribution_queue_mask.reference @@ -1,4 +1,4 @@ masked -3,"default:*@127%2E0%2E0%2E1:9000,default:*@127%2E0%2E0%2E2:9000" +3,"default:*@127%2E0%2E0%2E1:9000,default:*@127%2E0%2E0%2E2:9000","AUTHENTICATION_FAILED",1 no masking 1,"default@localhost:9000" diff --git a/tests/queries/0_stateless/01555_system_distribution_queue_mask.sql b/tests/queries/0_stateless/01555_system_distribution_queue_mask.sql index bdcde1adbad..285e93a4f90 100644 --- a/tests/queries/0_stateless/01555_system_distribution_queue_mask.sql +++ b/tests/queries/0_stateless/01555_system_distribution_queue_mask.sql @@ -18,7 +18,7 @@ create table dist_01555 (key Int) Engine=Distributed(test_cluster_with_incorrect insert into dist_01555 values (1)(2); -- since test_cluster_with_incorrect_pw contains incorrect password ignore error system flush distributed dist_01555; -- { serverError 516; } -select length(splitByChar('*', data_path)), replaceRegexpOne(data_path, '^.*/([^/]*)/' , '\\1') from system.distribution_queue where database = currentDatabase() and table = 'dist_01555' format CSV; +select length(splitByChar('*', data_path)), replaceRegexpOne(data_path, '^.*/([^/]*)/' , '\\1'), extract(last_exception, 'AUTHENTICATION_FAILED'), dateDiff('s', last_exception_time, now()) < 5 from system.distribution_queue where database = currentDatabase() and table = 'dist_01555' format CSV; drop table dist_01555; diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference index aabe05ea5e2..1840c5aa5a3 100644 --- a/tests/queries/0_stateless/02117_show_create_table_system.reference +++ b/tests/queries/0_stateless/02117_show_create_table_system.reference @@ -229,7 +229,8 @@ CREATE TABLE system.distribution_queue `data_compressed_bytes` UInt64, `broken_data_files` UInt64, `broken_data_compressed_bytes` UInt64, - `last_exception` String + `last_exception` String, + `last_exception_time` DateTime ) ENGINE = SystemDistributionQueue COMMENT 'SYSTEM TABLE is built on the fly.' From ef33d11e3fbc9c8eebf6ed1b2dfd40eac9a32a75 Mon Sep 17 00:00:00 2001 From: HarryLeeIBM Date: Tue, 21 Feb 2023 18:40:11 -0800 Subject: [PATCH 528/717] Refactor code according to code review --- src/Functions/FunctionsHashing.h | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/src/Functions/FunctionsHashing.h b/src/Functions/FunctionsHashing.h index 6bf1a2db3ac..59d573df3d1 100644 --- a/src/Functions/FunctionsHashing.h +++ b/src/Functions/FunctionsHashing.h @@ -1025,27 +1025,19 @@ private: if constexpr (Impl::use_int_hash_for_pods) { - if constexpr (std::endian::native == std::endian::little) + if constexpr (std::is_same_v) { - if constexpr (std::is_same_v) - h = IntHash64Impl::apply(bit_cast(vec_from[i])); - else - h = IntHash32Impl::apply(bit_cast(vec_from[i])); + UInt64 v = bit_cast(vec_from[i]); + if constexpr (std::endian::native == std::endian::big) + v = __builtin_bswap64(v); + h = IntHash64Impl::apply(v); } else { - if constexpr (std::is_same_v) - { - UInt64 v = bit_cast(vec_from[i]); - v = __builtin_bswap64(v); - h = IntHash64Impl::apply(v); - } - else - { - UInt32 v = bit_cast(vec_from[i]); + UInt32 v = bit_cast(vec_from[i]); + if constexpr (std::endian::native == std::endian::big) v = __builtin_bswap32(v); - h = IntHash32Impl::apply(v); - } + h = IntHash32Impl::apply(v); } } else From 2ca47a6eb60ba886f689122ab6e8b22b2d2bbb84 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 22 Feb 2023 10:41:57 +0100 Subject: [PATCH 529/717] BackgroundSchedulePool should not have any query context BackgroundSchedulePool is used for some peridic jobs, not from the query context, i.e. flush of Buffer table. And for such jobs there cannot be any query context, and more importantly it will not work correctly since that query_context will eventually expires. And this is the reason of this failures [1]. [1]: https://s3.amazonaws.com/clickhouse-test-reports/46668/015991bc5e20c787851050c2eaa13f0fef3aac00/stateless_tests_flaky_check__asan_.html Signed-off-by: Azat Khuzhin --- src/Core/BackgroundSchedulePool.cpp | 28 ------------------- src/Core/BackgroundSchedulePool.h | 5 ---- src/Interpreters/ConcurrentHashJoin.h | 1 - .../MergeTree/MergeTreePrefetchedReadPool.h | 1 - 4 files changed, 35 deletions(-) diff --git a/src/Core/BackgroundSchedulePool.cpp b/src/Core/BackgroundSchedulePool.cpp index 165d8902e85..993cfb6ef04 100644 --- a/src/Core/BackgroundSchedulePool.cpp +++ b/src/Core/BackgroundSchedulePool.cpp @@ -252,36 +252,10 @@ void BackgroundSchedulePool::cancelDelayedTask(const TaskInfoPtr & task, std::lo } -scope_guard BackgroundSchedulePool::attachToThreadGroup() -{ - scope_guard guard = [&]() - { - if (thread_group) - CurrentThread::detachQueryIfNotDetached(); - }; - - std::lock_guard lock(delayed_tasks_mutex); - - if (thread_group) - { - /// Put all threads to one thread pool - CurrentThread::attachTo(thread_group); - } - else - { - CurrentThread::initializeQuery(); - thread_group = CurrentThread::getGroup(); - } - return guard; -} - - void BackgroundSchedulePool::threadFunction() { setThreadName(thread_name.c_str()); - auto detach_thread_guard = attachToThreadGroup(); - while (!shutdown) { TaskInfoPtr task; @@ -311,8 +285,6 @@ void BackgroundSchedulePool::delayExecutionThreadFunction() { setThreadName((thread_name + "/D").c_str()); - auto detach_thread_guard = attachToThreadGroup(); - while (!shutdown) { TaskInfoPtr task; diff --git a/src/Core/BackgroundSchedulePool.h b/src/Core/BackgroundSchedulePool.h index ba1be312f27..0fb70b1f715 100644 --- a/src/Core/BackgroundSchedulePool.h +++ b/src/Core/BackgroundSchedulePool.h @@ -90,13 +90,8 @@ private: /// Tasks ordered by scheduled time. DelayedTasks delayed_tasks; - /// Thread group used for profiling purposes - ThreadGroupStatusPtr thread_group; - CurrentMetrics::Metric tasks_metric; std::string thread_name; - - [[nodiscard]] scope_guard attachToThreadGroup(); }; diff --git a/src/Interpreters/ConcurrentHashJoin.h b/src/Interpreters/ConcurrentHashJoin.h index a00c3ed1326..5e53f9845aa 100644 --- a/src/Interpreters/ConcurrentHashJoin.h +++ b/src/Interpreters/ConcurrentHashJoin.h @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include diff --git a/src/Storages/MergeTree/MergeTreePrefetchedReadPool.h b/src/Storages/MergeTree/MergeTreePrefetchedReadPool.h index bad158cd7a7..98cfe28c563 100644 --- a/src/Storages/MergeTree/MergeTreePrefetchedReadPool.h +++ b/src/Storages/MergeTree/MergeTreePrefetchedReadPool.h @@ -4,7 +4,6 @@ #include #include #include -#include #include #include From d4bb84e68b6b91e9168df1555ecd08ecf53fb547 Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 22 Feb 2023 09:56:10 +0000 Subject: [PATCH 530/717] make clang-tidy happy about CrossToInnerJoinPass --- src/Analyzer/Passes/CrossToInnerJoinPass.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Analyzer/Passes/CrossToInnerJoinPass.cpp b/src/Analyzer/Passes/CrossToInnerJoinPass.cpp index 400c760cd20..0c4fa0ed82f 100644 --- a/src/Analyzer/Passes/CrossToInnerJoinPass.cpp +++ b/src/Analyzer/Passes/CrossToInnerJoinPass.cpp @@ -42,7 +42,7 @@ void exctractJoinConditions(const QueryTreeNodePtr & node, QueryTreeNodes & equi } else if (func->getFunctionName() == "and") { - for (auto & arg : args) + for (const auto & arg : args) exctractJoinConditions(arg, equi_conditions, other); } else @@ -77,7 +77,7 @@ std::pair getExpressionSource(const QueryTreeNodeP { const IQueryTreeNode * source = nullptr; const auto & args = func->getArguments().getNodes(); - for (auto & arg : args) + for (const auto & arg : args) { auto [arg_source, is_ok] = getExpressionSource(arg); if (!is_ok) @@ -232,7 +232,7 @@ private: return nodes.front(); auto function_node = std::make_shared("and"); - for (auto & node : nodes) + for (const auto & node : nodes) function_node->getArguments().getNodes().push_back(node); const auto & function = FunctionFactory::instance().get("and", getContext()); From 16d61832fbb032d42ebbd869488ae2ea32fec9a0 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 22 Feb 2023 10:03:08 +0000 Subject: [PATCH 531/717] Bump minimum required Clang from 12 to 15 Needed due to https://github.com/ClickHouse/ClickHouse/pull/46247#discussion_r1109855435 --- cmake/tools.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/tools.cmake b/cmake/tools.cmake index 84376d13d9b..4d4d741cc3a 100644 --- a/cmake/tools.cmake +++ b/cmake/tools.cmake @@ -15,7 +15,7 @@ execute_process(COMMAND ${CMAKE_CXX_COMPILER} --version OUTPUT_VARIABLE COMPILER message (STATUS "Using compiler:\n${COMPILER_SELF_IDENTIFICATION}") # Require minimum compiler versions -set (CLANG_MINIMUM_VERSION 12) +set (CLANG_MINIMUM_VERSION 15) set (XCODE_MINIMUM_VERSION 12.0) set (APPLE_CLANG_MINIMUM_VERSION 12.0.0) set (GCC_MINIMUM_VERSION 11) From a4919ce3a20f93404512561e7e8655ff412df346 Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 22 Feb 2023 10:13:39 +0000 Subject: [PATCH 532/717] Add doc for temporary_data_in_cache --- .../settings.md | 90 ++++++++++++++++--- 1 file changed, 80 insertions(+), 10 deletions(-) diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index da42b31b78a..75ae6f3d2bc 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -1539,33 +1539,103 @@ Example 9005 ``` + ## tmp_path {#tmp-path} -Path to temporary data for processing large queries. +Path on the local filesystem to store temporary data for processing large queries. :::note -The trailing slash is mandatory. +- Only one option can be used to configure temporary data storage: `tmp_path` ,`tmp_policy`, `temporary_data_in_cache`. +- The trailing slash is mandatory. ::: **Example** -``` xml +```xml /var/lib/clickhouse/tmp/ ``` ## tmp_policy {#tmp-policy} -Policy from [storage_configuration](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes) to store temporary files. - -If not set, [tmp_path](#tmp-path) is used, otherwise it is ignored. +Alternatively, a policy from [storage_configuration](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes) can be used to store temporary files. :::note -- `move_factor` is ignored. -- `keep_free_space_bytes` is ignored. -- `max_data_part_size_bytes` is ignored. -- Policy should have exactly one volume with local disks. +- Only one option can be used to configure temporary data storage: `tmp_path` ,`tmp_policy`, `temporary_data_in_cache`. +- `move_factor`, `keep_free_space_bytes`,`max_data_part_size_bytes` and are ignored. +- Policy should have exactly *one volume* with *local* disks. ::: +**Example** + +```xml + + + + /disk1/ + + + /disk2/ + + + + + + +
+ disk1 + disk2 +
+ + + + + + tmp_two_disks + + +``` + +When `/disk1` is full, temporary data will be stored on `/disk2`. + +## temporary_data_in_cache {#temporary-data-in-cache} + +With this option, temporary data will be stored in the cache for the particular disk. +In this section, you should specify the disk name with the type `cache`. +In that case, the cache and temporary data will share the same space, and the disk cache can be evicted to create temporary data. + +:::note +- Only one option can be used to configure temporary data storage: `tmp_path` ,`tmp_policy`, `temporary_data_in_cache`. +::: + +**Example** + +```xml + + + + + local + /local_disk/ + + + + cache + local_disk + /tiny_local_cache/ + 10M + 1M + 1 + 0 + + + + + tiny_local_cache + +``` + +Cache for `local_disk` and temporary data will be stored in `/tiny_local_cache` on the filesystem, managed by `tiny_local_cache`. + ## max_temporary_data_on_disk_size {#max_temporary_data_on_disk_size} Limit the amount of disk space consumed by temporary files in `tmp_path` for the server. From 678e4250cd40c8849a0c01e1de70545455cf0c06 Mon Sep 17 00:00:00 2001 From: flynn Date: Wed, 22 Feb 2023 18:54:19 +0800 Subject: [PATCH 533/717] Fix incorrect predicate push down with grouping sets (#46151) --- .../PredicateExpressionsOptimizer.cpp | 3 +- src/Processors/QueryPlan/AggregatingStep.h | 2 + .../Optimizations/filterPushDown.cpp | 61 ++++++++++ ...rouping_sets_predicate_push_down.reference | 62 ++++++++++ ..._fix_grouping_sets_predicate_push_down.sql | 109 ++++++++++++++++++ 5 files changed, 236 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/02554_fix_grouping_sets_predicate_push_down.reference create mode 100644 tests/queries/0_stateless/02554_fix_grouping_sets_predicate_push_down.sql diff --git a/src/Interpreters/PredicateExpressionsOptimizer.cpp b/src/Interpreters/PredicateExpressionsOptimizer.cpp index d9ea29fe1d8..6606e64f689 100644 --- a/src/Interpreters/PredicateExpressionsOptimizer.cpp +++ b/src/Interpreters/PredicateExpressionsOptimizer.cpp @@ -35,7 +35,8 @@ bool PredicateExpressionsOptimizer::optimize(ASTSelectQuery & select_query) if (!enable_optimize_predicate_expression) return false; - if (select_query.having() && (!select_query.group_by_with_cube && !select_query.group_by_with_rollup && !select_query.group_by_with_totals)) + const bool has_incompatible_constructs = select_query.group_by_with_cube || select_query.group_by_with_rollup || select_query.group_by_with_totals || select_query.group_by_with_grouping_sets; + if (select_query.having() && !has_incompatible_constructs) tryMovePredicatesFromHavingToWhere(select_query); if (!select_query.tables() || select_query.tables()->children.empty()) diff --git a/src/Processors/QueryPlan/AggregatingStep.h b/src/Processors/QueryPlan/AggregatingStep.h index d395e94c58b..5f5557fb204 100644 --- a/src/Processors/QueryPlan/AggregatingStep.h +++ b/src/Processors/QueryPlan/AggregatingStep.h @@ -56,6 +56,8 @@ public: const Aggregator::Params & getParams() const { return params; } + const auto & getGroupingSetsParamsList() const { return grouping_sets_params; } + bool inOrder() const { return !sort_description_for_merging.empty(); } bool explicitSortingRequired() const { return explicit_sorting_required_for_aggregation_in_order; } bool isGroupingSets() const { return !grouping_sets_params.empty(); } diff --git a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp index 46fe3055e32..d466c52725f 100644 --- a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp +++ b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp @@ -53,6 +53,53 @@ static void checkChildrenSize(QueryPlan::Node * node, size_t child_num) child_num, child->getInputStreams().size(), node->children.size()); } +static bool identifiersIsAmongAllGroupingSets(const GroupingSetsParamsList & grouping_sets_params, const NameSet & identifiers_in_predicate) +{ + for (const auto & grouping_set : grouping_sets_params) + { + for (const auto & identifier : identifiers_in_predicate) + { + if (std::find(grouping_set.used_keys.begin(), grouping_set.used_keys.end(), identifier) == grouping_set.used_keys.end()) + return false; + } + } + return true; +} + +static NameSet findIdentifiersOfNode(const ActionsDAG::Node * node) +{ + NameSet res; + + /// We treat all INPUT as identifier + if (node->type == ActionsDAG::ActionType::INPUT) + { + res.emplace(node->result_name); + return res; + } + + std::queue queue; + queue.push(node); + + while (!queue.empty()) + { + const auto * top = queue.front(); + for (const auto * child : top->children) + { + if (child->type == ActionsDAG::ActionType::INPUT) + { + res.emplace(child->result_name); + } + else + { + /// Only push non INPUT child into the queue + queue.push(child); + } + } + queue.pop(); + } + return res; +} + static ActionsDAGPtr splitFilter(QueryPlan::Node * parent_node, const Names & allowed_inputs, size_t child_idx = 0) { QueryPlan::Node * child_node = parent_node->children.front(); @@ -176,6 +223,20 @@ size_t tryPushDownFilter(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes if (auto * aggregating = typeid_cast(child.get())) { + /// If aggregating is GROUPING SETS, and not all the identifiers exist in all + /// of the grouping sets, we could not push the filter down. + if (aggregating->isGroupingSets()) + { + + const auto & actions = filter->getExpression(); + const auto & filter_node = actions->findInOutputs(filter->getFilterColumnName()); + + auto identifiers_in_predicate = findIdentifiersOfNode(&filter_node); + + if (!identifiersIsAmongAllGroupingSets(aggregating->getGroupingSetsParamsList(), identifiers_in_predicate)) + return 0; + } + const auto & params = aggregating->getParams(); const auto & keys = params.keys; diff --git a/tests/queries/0_stateless/02554_fix_grouping_sets_predicate_push_down.reference b/tests/queries/0_stateless/02554_fix_grouping_sets_predicate_push_down.reference new file mode 100644 index 00000000000..440f668c614 --- /dev/null +++ b/tests/queries/0_stateless/02554_fix_grouping_sets_predicate_push_down.reference @@ -0,0 +1,62 @@ +---Explain Syntax--- +SELECT + day_, + type_1 +FROM +( + SELECT + day_, + if(type_1 = \'\', \'all\', type_1) AS type_1 + FROM + ( + SELECT + day_, + type_1 + FROM test_grouping_sets_predicate + PREWHERE day_ = \'2023-01-05\' + GROUP BY + GROUPING SETS ( + (day_, type_1), + (day_)) + HAVING if(type_1 = \'\', \'all\', type_1) = \'all\' + ) AS t + WHERE type_1 = \'all\' +) +WHERE type_1 = \'all\' + +---Explain Pipeline--- +(Expression) +ExpressionTransform × 2 + (Filter) + FilterTransform × 2 + (Filter) + FilterTransform × 2 + (Filter) + FilterTransform × 2 + (Aggregating) + ExpressionTransform × 2 + AggregatingTransform × 2 + Copy 1 → 2 + (Expression) + ExpressionTransform + (ReadFromMergeTree) + MergeTreeInOrder 0 → 1 + +---Result--- +2023-01-05 all + +---Explain Pipeline--- +(Expression) +ExpressionTransform × 2 + (Aggregating) + ExpressionTransform × 2 + AggregatingTransform × 2 + Copy 1 → 2 + (Filter) + FilterTransform + (Filter) + FilterTransform + (Filter) + FilterTransform + (ReadFromMergeTree) + MergeTreeInOrder 0 → 1 diff --git a/tests/queries/0_stateless/02554_fix_grouping_sets_predicate_push_down.sql b/tests/queries/0_stateless/02554_fix_grouping_sets_predicate_push_down.sql new file mode 100644 index 00000000000..9a970674890 --- /dev/null +++ b/tests/queries/0_stateless/02554_fix_grouping_sets_predicate_push_down.sql @@ -0,0 +1,109 @@ +DROP TABLE IF EXISTS test_grouping_sets_predicate; + +CREATE TABLE test_grouping_sets_predicate +( + day_ Date, + type_1 String +) +ENGINE=MergeTree +ORDER BY day_; + +INSERT INTO test_grouping_sets_predicate SELECT + toDate('2023-01-05') AS day_, + 'hello, world' +FROM numbers (10); + +SELECT '---Explain Syntax---'; +EXPLAIN SYNTAX +SELECT * +FROM +( + SELECT + day_, + if(type_1 = '', 'all', type_1) AS type_1 + FROM + ( + SELECT + day_, + type_1 + FROM test_grouping_sets_predicate + WHERE day_ = '2023-01-05' + GROUP BY + GROUPING SETS ( + (day_, type_1), + (day_)) + ) AS t +) +WHERE type_1 = 'all'; + +SELECT ''; +SELECT '---Explain Pipeline---'; +EXPLAIN PIPELINE +SELECT * +FROM +( + SELECT + day_, + if(type_1 = '', 'all', type_1) AS type_1 + FROM + ( + SELECT + day_, + type_1 + FROM test_grouping_sets_predicate + WHERE day_ = '2023-01-05' + GROUP BY + GROUPING SETS ( + (day_, type_1), + (day_)) + ) AS t +) +WHERE type_1 = 'all'; + +SELECT ''; +SELECT '---Result---'; +SELECT * +FROM +( + SELECT + day_, + if(type_1 = '', 'all', type_1) AS type_1 + FROM + ( + SELECT + day_, + type_1 + FROM test_grouping_sets_predicate + WHERE day_ = '2023-01-05' + GROUP BY + GROUPING SETS ( + (day_, type_1), + (day_)) + ) AS t +) +WHERE type_1 = 'all'; + +SELECT ''; +SELECT '---Explain Pipeline---'; +EXPLAIN PIPELINE +SELECT * +FROM +( + SELECT + day_, + if(type_1 = '', 'all', type_1) AS type_1 + FROM + ( + SELECT + day_, + type_1 + FROM test_grouping_sets_predicate + GROUP BY + GROUPING SETS ( + (day_, type_1), + (day_)) + ) AS t +) +WHERE day_ = '2023-01-05'; + +DROP TABLE test_grouping_sets_predicate; From 21fcc3b69c6355945617e1a5d77aa57de4694e91 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 22 Feb 2023 12:01:18 +0100 Subject: [PATCH 534/717] Add iceberg doc --- .../table-engines/integrations/deltalake.md | 23 +++++++- .../table-engines/integrations/hudi.md | 23 +++++++- .../table-engines/integrations/iceberg.md | 52 +++++++++++++++++ .../sql-reference/table-functions/iceberg.md | 58 +++++++++++++++++++ 4 files changed, 152 insertions(+), 4 deletions(-) create mode 100644 docs/en/engines/table-engines/integrations/iceberg.md create mode 100644 docs/en/sql-reference/table-functions/iceberg.md diff --git a/docs/en/engines/table-engines/integrations/deltalake.md b/docs/en/engines/table-engines/integrations/deltalake.md index 83526ac944d..64ef7ec4dfc 100644 --- a/docs/en/engines/table-engines/integrations/deltalake.md +++ b/docs/en/engines/table-engines/integrations/deltalake.md @@ -19,7 +19,9 @@ CREATE TABLE deltalake **Engine parameters** - `url` — Bucket url with path to the existing Delta Lake table. -- `aws_access_key_id`, `aws_secret_access_key` - Long-term credentials for the [AWS](https://aws.amazon.com/) account user. You can use these to authenticate your requests. Parameter is optional. If credentials are not specified, they are used from the configuration file. For more information see [Using S3 for Data Storage](../mergetree-family/mergetree.md#table_engine-mergetree-s3). +- `aws_access_key_id`, `aws_secret_access_key` - Long-term credentials for the [AWS](https://aws.amazon.com/) account user. You can use these to authenticate your requests. Parameter is optional. If credentials are not specified, they are used from the configuration file. + +Engine parameters can be specified using [Named Collections](../operations/settings/named-collections.md) **Example** @@ -27,7 +29,24 @@ CREATE TABLE deltalake CREATE TABLE deltalake ENGINE=DeltaLake('http://mars-doc-test.s3.amazonaws.com/clickhouse-bucket-3/test_table/', 'ABC123', 'Abc+123') ``` +Using named collections: + +``` xml + + + + http://mars-doc-test.s3.amazonaws.com/clickhouse-bucket-3/ + ABC123 + Abc+123 + + + +``` + +```sql +CREATE TABLE iceberg_table ENGINE=DeltaLake(deltalake_conf, filename = 'test_table') +``` + ## See also - [deltaLake table function](../../../sql-reference/table-functions/deltalake.md) - diff --git a/docs/en/engines/table-engines/integrations/hudi.md b/docs/en/engines/table-engines/integrations/hudi.md index 4e335e6c075..eb916b17cf9 100644 --- a/docs/en/engines/table-engines/integrations/hudi.md +++ b/docs/en/engines/table-engines/integrations/hudi.md @@ -19,7 +19,9 @@ CREATE TABLE hudi_table **Engine parameters** - `url` — Bucket url with the path to an existing Hudi table. -- `aws_access_key_id`, `aws_secret_access_key` - Long-term credentials for the [AWS](https://aws.amazon.com/) account user. You can use these to authenticate your requests. Parameter is optional. If credentials are not specified, they are used from the configuration file. For more information see [Using S3 for Data Storage](../mergetree-family/mergetree.md#table_engine-mergetree-s3). +- `aws_access_key_id`, `aws_secret_access_key` - Long-term credentials for the [AWS](https://aws.amazon.com/) account user. You can use these to authenticate your requests. Parameter is optional. If credentials are not specified, they are used from the configuration file. + +Engine parameters can be specified using [Named Collections](../operations/settings/named-collections.md) **Example** @@ -27,7 +29,24 @@ CREATE TABLE hudi_table CREATE TABLE hudi_table ENGINE=Hudi('http://mars-doc-test.s3.amazonaws.com/clickhouse-bucket-3/test_table/', 'ABC123', 'Abc+123') ``` +Using named collections: + +``` xml + + + + http://mars-doc-test.s3.amazonaws.com/clickhouse-bucket-3/ + ABC123 + Abc+123 + + + +``` + +```sql +CREATE TABLE iceberg_table ENGINE=Hudi(hudi_conf, filename = 'test_table') +``` + ## See also - [hudi table function](/docs/en/sql-reference/table-functions/hudi.md) - diff --git a/docs/en/engines/table-engines/integrations/iceberg.md b/docs/en/engines/table-engines/integrations/iceberg.md new file mode 100644 index 00000000000..33ec5f877bf --- /dev/null +++ b/docs/en/engines/table-engines/integrations/iceberg.md @@ -0,0 +1,52 @@ +--- +slug: /en/engines/table-engines/integrations/iceberg +sidebar_label: Iceberg +--- + +# Iceberg Table Engine + +This engine provides a read-only integration with existing Apache [Iceberg](https://iceberg.apache.org/) tables in Amazon S3. + +## Create Table + +Note that the Iceberg table must already exist in S3, this command does not take DDL parameters to create a new table. + +``` sql +CREATE TABLE iceberg_table + ENGINE = Iceberg(url, [aws_access_key_id, aws_secret_access_key,]) +``` + +**Engine parameters** + +- `url` — url with the path to an existing Iceberg table. +- `aws_access_key_id`, `aws_secret_access_key` - Long-term credentials for the [AWS](https://aws.amazon.com/) account user. You can use these to authenticate your requests. Parameter is optional. If credentials are not specified, they are used from the configuration file. + +Engine parameters can be specified using [Named Collections](../operations/settings/named-collections.md) + +**Example** + +```sql +CREATE TABLE iceberg_table ENGINE=Iceberg('http://test.s3.amazonaws.com/clickhouse-bucket/test_table', 'test', 'test') +``` + +Using named collections: + +``` xml + + + + http://test.s3.amazonaws.com/clickhouse-bucket/ + test + test + + + +``` + +```sql +CREATE TABLE iceberg_table ENGINE=Iceberg(iceberg_conf, filename = 'test_table') +``` + +## See also + +- [iceberg table function](/docs/en/sql-reference/table-functions/iceberg.md) diff --git a/docs/en/sql-reference/table-functions/iceberg.md b/docs/en/sql-reference/table-functions/iceberg.md new file mode 100644 index 00000000000..036c1379847 --- /dev/null +++ b/docs/en/sql-reference/table-functions/iceberg.md @@ -0,0 +1,58 @@ +--- +slug: /en/sql-reference/table-functions/iceberg +sidebar_label: Iceberg +--- + +# iceberg Table Function + +Provides a read-only table-like interface to Apache [Iceberg](https://iceberg.apache.org/) tables in Amazon S3. + +## Syntax + +``` sql +iceberg(url [,aws_access_key_id, aws_secret_access_key] [,format] [,structure]) +``` + +## Arguments + +- `url` — Bucket url with the path to an existing Iceberg table in S3. +- `aws_access_key_id`, `aws_secret_access_key` - Long-term credentials for the [AWS](https://aws.amazon.com/) account user. You can use these to authenticate your requests. These parameters are optional. If credentials are not specified, they are used from the ClickHouse configuration. For more information see [Using S3 for Data Storage](/docs/en/engines/table-engines/mergetree-family/mergetree.md/#table_engine-mergetree-s3). +- `format` — The [format](/docs/en/interfaces/formats.md/#formats) of the file. By default `Parquet` is used. +- `structure` — Structure of the table. Format `'column1_name column1_type, column2_name column2_type, ...'`. + +Engine parameters can be specified using [Named Collections](../operations/settings/named-collections.md) + +**Returned value** + +A table with the specified structure for reading data in the specified Iceberg table in S3. + +**Example** + +```sql +SELECT * FROM iceberg('http://test.s3.amazonaws.com/clickhouse-bucket/test_table', 'test', 'test') +``` + +Using named collections: + +```xml + + + + http://test.s3.amazonaws.com/clickhouse-bucket/ + test + test + auto + auto + + + +``` + +```sql +SELECT * FROM iceberg(iceberg_conf, filename = 'test_table') +DESCRIBE iceberg(iceberg_conf, filename = 'test_table') +``` + +**See Also** + +- [Iceberg engine](/docs/en/engines/table-engines/integrations/iceberg.md) From ef15d6489565a8486815cfd43500ebf6fa0729ed Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Wed, 22 Feb 2023 12:11:23 +0100 Subject: [PATCH 535/717] Update docs/en/engines/table-engines/integrations/deltalake.md Co-authored-by: flynn --- docs/en/engines/table-engines/integrations/deltalake.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/engines/table-engines/integrations/deltalake.md b/docs/en/engines/table-engines/integrations/deltalake.md index 64ef7ec4dfc..a2816c7ff57 100644 --- a/docs/en/engines/table-engines/integrations/deltalake.md +++ b/docs/en/engines/table-engines/integrations/deltalake.md @@ -44,7 +44,7 @@ Using named collections: ``` ```sql -CREATE TABLE iceberg_table ENGINE=DeltaLake(deltalake_conf, filename = 'test_table') +CREATE TABLE deltalake ENGINE=DeltaLake(deltalake_conf, filename = 'test_table') ``` ## See also From c242fe3e5e80c2bcc3259c47b8e896be3aa13952 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Wed, 22 Feb 2023 12:11:42 +0100 Subject: [PATCH 536/717] Update docs/en/engines/table-engines/integrations/hudi.md Co-authored-by: flynn --- docs/en/engines/table-engines/integrations/hudi.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/engines/table-engines/integrations/hudi.md b/docs/en/engines/table-engines/integrations/hudi.md index eb916b17cf9..6ff998d86d9 100644 --- a/docs/en/engines/table-engines/integrations/hudi.md +++ b/docs/en/engines/table-engines/integrations/hudi.md @@ -44,7 +44,7 @@ Using named collections: ``` ```sql -CREATE TABLE iceberg_table ENGINE=Hudi(hudi_conf, filename = 'test_table') +CREATE TABLE hudi_table ENGINE=Hudi(hudi_conf, filename = 'test_table') ``` ## See also From 98c10ff6e5751404c48106293befb212b126e7d1 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 22 Feb 2023 12:16:09 +0100 Subject: [PATCH 537/717] Update docs/en/operations/system-tables/processors_profile_log.md Co-authored-by: Nikita Taranov --- docs/en/operations/system-tables/processors_profile_log.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/en/operations/system-tables/processors_profile_log.md b/docs/en/operations/system-tables/processors_profile_log.md index 269385deab6..cc917138741 100644 --- a/docs/en/operations/system-tables/processors_profile_log.md +++ b/docs/en/operations/system-tables/processors_profile_log.md @@ -33,6 +33,7 @@ SELECT sleep(1) │ (ReadFromStorage) │ │ SourceFromSingleChunk 0 → 1 │ └─────────────────────────────────┘ + SELECT sleep(1) SETTINGS log_processors_profiles = 1 Query id: feb5ed16-1c24-4227-aa54-78c02b3b27d4 From ab94d6dc1831de374f1530c9c5e78bcc06227133 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 22 Feb 2023 12:16:19 +0100 Subject: [PATCH 538/717] Update docs/en/operations/system-tables/processors_profile_log.md Co-authored-by: Nikita Taranov --- docs/en/operations/system-tables/processors_profile_log.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/en/operations/system-tables/processors_profile_log.md b/docs/en/operations/system-tables/processors_profile_log.md index cc917138741..a2e7a9ebabd 100644 --- a/docs/en/operations/system-tables/processors_profile_log.md +++ b/docs/en/operations/system-tables/processors_profile_log.md @@ -41,6 +41,7 @@ Query id: feb5ed16-1c24-4227-aa54-78c02b3b27d4 │ 0 │ └──────────┘ 1 rows in set. Elapsed: 1.018 sec. + SELECT name, elapsed_us, From ceff5f41d14653a3c95208bcef963a18902db64d Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 22 Feb 2023 12:23:14 +0100 Subject: [PATCH 539/717] Fix tests --- tests/config/users.d/access_management.xml | 1 + .../configs/users.d/users.xml | 1 + .../test_create_query_constraints/configs/users.xml | 1 + .../test_global_overcommit_tracker/configs/users.xml | 1 + .../test_grant_and_revoke/configs/users.d/users.xml | 1 + .../test_overcommit_tracker/configs/users.d/users.xml | 1 + .../configs/users.d/users.xml | 1 + tests/queries/0_stateless/01271_show_privileges.reference | 1 + .../0_stateless/02117_show_create_table_system.reference | 6 +++--- 9 files changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/config/users.d/access_management.xml b/tests/config/users.d/access_management.xml index 8f4d82805be..f7963cdb7f2 100644 --- a/tests/config/users.d/access_management.xml +++ b/tests/config/users.d/access_management.xml @@ -3,6 +3,7 @@ 1 1 + 1 diff --git a/tests/integration/test_access_control_on_cluster/configs/users.d/users.xml b/tests/integration/test_access_control_on_cluster/configs/users.d/users.xml index fb5e2028d6e..8556e73c82f 100644 --- a/tests/integration/test_access_control_on_cluster/configs/users.d/users.xml +++ b/tests/integration/test_access_control_on_cluster/configs/users.d/users.xml @@ -5,6 +5,7 @@ default default 1 + 1 diff --git a/tests/integration/test_create_query_constraints/configs/users.xml b/tests/integration/test_create_query_constraints/configs/users.xml index fb5e2028d6e..8556e73c82f 100644 --- a/tests/integration/test_create_query_constraints/configs/users.xml +++ b/tests/integration/test_create_query_constraints/configs/users.xml @@ -5,6 +5,7 @@ default default 1 + 1 diff --git a/tests/integration/test_global_overcommit_tracker/configs/users.xml b/tests/integration/test_global_overcommit_tracker/configs/users.xml index fb5e2028d6e..8556e73c82f 100644 --- a/tests/integration/test_global_overcommit_tracker/configs/users.xml +++ b/tests/integration/test_global_overcommit_tracker/configs/users.xml @@ -5,6 +5,7 @@ default default 1 + 1 diff --git a/tests/integration/test_grant_and_revoke/configs/users.d/users.xml b/tests/integration/test_grant_and_revoke/configs/users.d/users.xml index fb5e2028d6e..8556e73c82f 100644 --- a/tests/integration/test_grant_and_revoke/configs/users.d/users.xml +++ b/tests/integration/test_grant_and_revoke/configs/users.d/users.xml @@ -5,6 +5,7 @@ default default 1 + 1 diff --git a/tests/integration/test_overcommit_tracker/configs/users.d/users.xml b/tests/integration/test_overcommit_tracker/configs/users.d/users.xml index fb5e2028d6e..8556e73c82f 100644 --- a/tests/integration/test_overcommit_tracker/configs/users.d/users.xml +++ b/tests/integration/test_overcommit_tracker/configs/users.d/users.xml @@ -5,6 +5,7 @@ default default 1 + 1 diff --git a/tests/integration/test_settings_constraints_distributed/configs/users.d/users.xml b/tests/integration/test_settings_constraints_distributed/configs/users.d/users.xml index fb5e2028d6e..8556e73c82f 100644 --- a/tests/integration/test_settings_constraints_distributed/configs/users.d/users.xml +++ b/tests/integration/test_settings_constraints_distributed/configs/users.d/users.xml @@ -5,6 +5,7 @@ default default 1 + 1 diff --git a/tests/queries/0_stateless/01271_show_privileges.reference b/tests/queries/0_stateless/01271_show_privileges.reference index 58b1cab6e20..c061eb95a65 100644 --- a/tests/queries/0_stateless/01271_show_privileges.reference +++ b/tests/queries/0_stateless/01271_show_privileges.reference @@ -90,6 +90,7 @@ SHOW QUOTAS ['SHOW CREATE QUOTA'] GLOBAL SHOW ACCESS SHOW SETTINGS PROFILES ['SHOW PROFILES','SHOW CREATE SETTINGS PROFILE','SHOW CREATE PROFILE'] GLOBAL SHOW ACCESS SHOW ACCESS [] \N ACCESS MANAGEMENT SHOW NAMED COLLECTIONS ['SHOW NAMED COLLECTIONS'] GLOBAL ACCESS MANAGEMENT +SHOW NAMED COLLECTIONS SECRETS ['SHOW NAMED COLLECTIONS SECRETS'] GLOBAL ACCESS MANAGEMENT ACCESS MANAGEMENT [] \N ALL SYSTEM SHUTDOWN ['SYSTEM KILL','SHUTDOWN'] GLOBAL SYSTEM SYSTEM DROP DNS CACHE ['SYSTEM DROP DNS','DROP DNS CACHE','DROP DNS'] GLOBAL SYSTEM DROP CACHE diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference index aabe05ea5e2..fe93418aa6d 100644 --- a/tests/queries/0_stateless/02117_show_create_table_system.reference +++ b/tests/queries/0_stateless/02117_show_create_table_system.reference @@ -288,7 +288,7 @@ CREATE TABLE system.grants ( `user_name` Nullable(String), `role_name` Nullable(String), - `access_type` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'ACCESS MANAGEMENT' = 92, 'SYSTEM SHUTDOWN' = 93, 'SYSTEM DROP DNS CACHE' = 94, 'SYSTEM DROP MARK CACHE' = 95, 'SYSTEM DROP UNCOMPRESSED CACHE' = 96, 'SYSTEM DROP MMAP CACHE' = 97, 'SYSTEM DROP QUERY CACHE' = 98, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 99, 'SYSTEM DROP FILESYSTEM CACHE' = 100, 'SYSTEM DROP SCHEMA CACHE' = 101, 'SYSTEM DROP S3 CLIENT CACHE' = 102, 'SYSTEM DROP CACHE' = 103, 'SYSTEM RELOAD CONFIG' = 104, 'SYSTEM RELOAD USERS' = 105, 'SYSTEM RELOAD SYMBOLS' = 106, 'SYSTEM RELOAD DICTIONARY' = 107, 'SYSTEM RELOAD MODEL' = 108, 'SYSTEM RELOAD FUNCTION' = 109, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 110, 'SYSTEM RELOAD' = 111, 'SYSTEM RESTART DISK' = 112, 'SYSTEM MERGES' = 113, 'SYSTEM TTL MERGES' = 114, 'SYSTEM FETCHES' = 115, 'SYSTEM MOVES' = 116, 'SYSTEM DISTRIBUTED SENDS' = 117, 'SYSTEM REPLICATED SENDS' = 118, 'SYSTEM SENDS' = 119, 'SYSTEM REPLICATION QUEUES' = 120, 'SYSTEM DROP REPLICA' = 121, 'SYSTEM SYNC REPLICA' = 122, 'SYSTEM RESTART REPLICA' = 123, 'SYSTEM RESTORE REPLICA' = 124, 'SYSTEM WAIT LOADING PARTS' = 125, 'SYSTEM SYNC DATABASE REPLICA' = 126, 'SYSTEM SYNC TRANSACTION LOG' = 127, 'SYSTEM SYNC FILE CACHE' = 128, 'SYSTEM FLUSH DISTRIBUTED' = 129, 'SYSTEM FLUSH LOGS' = 130, 'SYSTEM FLUSH' = 131, 'SYSTEM THREAD FUZZER' = 132, 'SYSTEM UNFREEZE' = 133, 'SYSTEM' = 134, 'dictGet' = 135, 'addressToLine' = 136, 'addressToLineWithInlines' = 137, 'addressToSymbol' = 138, 'demangle' = 139, 'INTROSPECTION' = 140, 'FILE' = 141, 'URL' = 142, 'REMOTE' = 143, 'MONGO' = 144, 'MEILISEARCH' = 145, 'MYSQL' = 146, 'POSTGRES' = 147, 'SQLITE' = 148, 'ODBC' = 149, 'JDBC' = 150, 'HDFS' = 151, 'S3' = 152, 'HIVE' = 153, 'SOURCES' = 154, 'CLUSTER' = 155, 'ALL' = 156, 'NONE' = 157), + `access_type` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'SHOW NAMED COLLECTIONS SECRETS' = 92, 'ACCESS MANAGEMENT' = 93, 'SYSTEM SHUTDOWN' = 94, 'SYSTEM DROP DNS CACHE' = 95, 'SYSTEM DROP MARK CACHE' = 96, 'SYSTEM DROP UNCOMPRESSED CACHE' = 97, 'SYSTEM DROP MMAP CACHE' = 98, 'SYSTEM DROP QUERY CACHE' = 99, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 100, 'SYSTEM DROP FILESYSTEM CACHE' = 101, 'SYSTEM DROP SCHEMA CACHE' = 102, 'SYSTEM DROP S3 CLIENT CACHE' = 103, 'SYSTEM DROP CACHE' = 104, 'SYSTEM RELOAD CONFIG' = 105, 'SYSTEM RELOAD USERS' = 106, 'SYSTEM RELOAD SYMBOLS' = 107, 'SYSTEM RELOAD DICTIONARY' = 108, 'SYSTEM RELOAD MODEL' = 109, 'SYSTEM RELOAD FUNCTION' = 110, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 111, 'SYSTEM RELOAD' = 112, 'SYSTEM RESTART DISK' = 113, 'SYSTEM MERGES' = 114, 'SYSTEM TTL MERGES' = 115, 'SYSTEM FETCHES' = 116, 'SYSTEM MOVES' = 117, 'SYSTEM DISTRIBUTED SENDS' = 118, 'SYSTEM REPLICATED SENDS' = 119, 'SYSTEM SENDS' = 120, 'SYSTEM REPLICATION QUEUES' = 121, 'SYSTEM DROP REPLICA' = 122, 'SYSTEM SYNC REPLICA' = 123, 'SYSTEM RESTART REPLICA' = 124, 'SYSTEM RESTORE REPLICA' = 125, 'SYSTEM WAIT LOADING PARTS' = 126, 'SYSTEM SYNC DATABASE REPLICA' = 127, 'SYSTEM SYNC TRANSACTION LOG' = 128, 'SYSTEM SYNC FILE CACHE' = 129, 'SYSTEM FLUSH DISTRIBUTED' = 130, 'SYSTEM FLUSH LOGS' = 131, 'SYSTEM FLUSH' = 132, 'SYSTEM THREAD FUZZER' = 133, 'SYSTEM UNFREEZE' = 134, 'SYSTEM' = 135, 'dictGet' = 136, 'addressToLine' = 137, 'addressToLineWithInlines' = 138, 'addressToSymbol' = 139, 'demangle' = 140, 'INTROSPECTION' = 141, 'FILE' = 142, 'URL' = 143, 'REMOTE' = 144, 'MONGO' = 145, 'MEILISEARCH' = 146, 'MYSQL' = 147, 'POSTGRES' = 148, 'SQLITE' = 149, 'ODBC' = 150, 'JDBC' = 151, 'HDFS' = 152, 'S3' = 153, 'HIVE' = 154, 'SOURCES' = 155, 'CLUSTER' = 156, 'ALL' = 157, 'NONE' = 158), `database` Nullable(String), `table` Nullable(String), `column` Nullable(String), @@ -569,10 +569,10 @@ ENGINE = SystemPartsColumns COMMENT 'SYSTEM TABLE is built on the fly.' CREATE TABLE system.privileges ( - `privilege` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'ACCESS MANAGEMENT' = 92, 'SYSTEM SHUTDOWN' = 93, 'SYSTEM DROP DNS CACHE' = 94, 'SYSTEM DROP MARK CACHE' = 95, 'SYSTEM DROP UNCOMPRESSED CACHE' = 96, 'SYSTEM DROP MMAP CACHE' = 97, 'SYSTEM DROP QUERY CACHE' = 98, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 99, 'SYSTEM DROP FILESYSTEM CACHE' = 100, 'SYSTEM DROP SCHEMA CACHE' = 101, 'SYSTEM DROP S3 CLIENT CACHE' = 102, 'SYSTEM DROP CACHE' = 103, 'SYSTEM RELOAD CONFIG' = 104, 'SYSTEM RELOAD USERS' = 105, 'SYSTEM RELOAD SYMBOLS' = 106, 'SYSTEM RELOAD DICTIONARY' = 107, 'SYSTEM RELOAD MODEL' = 108, 'SYSTEM RELOAD FUNCTION' = 109, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 110, 'SYSTEM RELOAD' = 111, 'SYSTEM RESTART DISK' = 112, 'SYSTEM MERGES' = 113, 'SYSTEM TTL MERGES' = 114, 'SYSTEM FETCHES' = 115, 'SYSTEM MOVES' = 116, 'SYSTEM DISTRIBUTED SENDS' = 117, 'SYSTEM REPLICATED SENDS' = 118, 'SYSTEM SENDS' = 119, 'SYSTEM REPLICATION QUEUES' = 120, 'SYSTEM DROP REPLICA' = 121, 'SYSTEM SYNC REPLICA' = 122, 'SYSTEM RESTART REPLICA' = 123, 'SYSTEM RESTORE REPLICA' = 124, 'SYSTEM WAIT LOADING PARTS' = 125, 'SYSTEM SYNC DATABASE REPLICA' = 126, 'SYSTEM SYNC TRANSACTION LOG' = 127, 'SYSTEM SYNC FILE CACHE' = 128, 'SYSTEM FLUSH DISTRIBUTED' = 129, 'SYSTEM FLUSH LOGS' = 130, 'SYSTEM FLUSH' = 131, 'SYSTEM THREAD FUZZER' = 132, 'SYSTEM UNFREEZE' = 133, 'SYSTEM' = 134, 'dictGet' = 135, 'addressToLine' = 136, 'addressToLineWithInlines' = 137, 'addressToSymbol' = 138, 'demangle' = 139, 'INTROSPECTION' = 140, 'FILE' = 141, 'URL' = 142, 'REMOTE' = 143, 'MONGO' = 144, 'MEILISEARCH' = 145, 'MYSQL' = 146, 'POSTGRES' = 147, 'SQLITE' = 148, 'ODBC' = 149, 'JDBC' = 150, 'HDFS' = 151, 'S3' = 152, 'HIVE' = 153, 'SOURCES' = 154, 'CLUSTER' = 155, 'ALL' = 156, 'NONE' = 157), + `privilege` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'SHOW NAMED COLLECTIONS SECRETS' = 92, 'ACCESS MANAGEMENT' = 93, 'SYSTEM SHUTDOWN' = 94, 'SYSTEM DROP DNS CACHE' = 95, 'SYSTEM DROP MARK CACHE' = 96, 'SYSTEM DROP UNCOMPRESSED CACHE' = 97, 'SYSTEM DROP MMAP CACHE' = 98, 'SYSTEM DROP QUERY CACHE' = 99, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 100, 'SYSTEM DROP FILESYSTEM CACHE' = 101, 'SYSTEM DROP SCHEMA CACHE' = 102, 'SYSTEM DROP S3 CLIENT CACHE' = 103, 'SYSTEM DROP CACHE' = 104, 'SYSTEM RELOAD CONFIG' = 105, 'SYSTEM RELOAD USERS' = 106, 'SYSTEM RELOAD SYMBOLS' = 107, 'SYSTEM RELOAD DICTIONARY' = 108, 'SYSTEM RELOAD MODEL' = 109, 'SYSTEM RELOAD FUNCTION' = 110, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 111, 'SYSTEM RELOAD' = 112, 'SYSTEM RESTART DISK' = 113, 'SYSTEM MERGES' = 114, 'SYSTEM TTL MERGES' = 115, 'SYSTEM FETCHES' = 116, 'SYSTEM MOVES' = 117, 'SYSTEM DISTRIBUTED SENDS' = 118, 'SYSTEM REPLICATED SENDS' = 119, 'SYSTEM SENDS' = 120, 'SYSTEM REPLICATION QUEUES' = 121, 'SYSTEM DROP REPLICA' = 122, 'SYSTEM SYNC REPLICA' = 123, 'SYSTEM RESTART REPLICA' = 124, 'SYSTEM RESTORE REPLICA' = 125, 'SYSTEM WAIT LOADING PARTS' = 126, 'SYSTEM SYNC DATABASE REPLICA' = 127, 'SYSTEM SYNC TRANSACTION LOG' = 128, 'SYSTEM SYNC FILE CACHE' = 129, 'SYSTEM FLUSH DISTRIBUTED' = 130, 'SYSTEM FLUSH LOGS' = 131, 'SYSTEM FLUSH' = 132, 'SYSTEM THREAD FUZZER' = 133, 'SYSTEM UNFREEZE' = 134, 'SYSTEM' = 135, 'dictGet' = 136, 'addressToLine' = 137, 'addressToLineWithInlines' = 138, 'addressToSymbol' = 139, 'demangle' = 140, 'INTROSPECTION' = 141, 'FILE' = 142, 'URL' = 143, 'REMOTE' = 144, 'MONGO' = 145, 'MEILISEARCH' = 146, 'MYSQL' = 147, 'POSTGRES' = 148, 'SQLITE' = 149, 'ODBC' = 150, 'JDBC' = 151, 'HDFS' = 152, 'S3' = 153, 'HIVE' = 154, 'SOURCES' = 155, 'CLUSTER' = 156, 'ALL' = 157, 'NONE' = 158), `aliases` Array(String), `level` Nullable(Enum8('GLOBAL' = 0, 'DATABASE' = 1, 'TABLE' = 2, 'DICTIONARY' = 3, 'VIEW' = 4, 'COLUMN' = 5)), - `parent_group` Nullable(Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'ACCESS MANAGEMENT' = 92, 'SYSTEM SHUTDOWN' = 93, 'SYSTEM DROP DNS CACHE' = 94, 'SYSTEM DROP MARK CACHE' = 95, 'SYSTEM DROP UNCOMPRESSED CACHE' = 96, 'SYSTEM DROP MMAP CACHE' = 97, 'SYSTEM DROP QUERY CACHE' = 98, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 99, 'SYSTEM DROP FILESYSTEM CACHE' = 100, 'SYSTEM DROP SCHEMA CACHE' = 101, 'SYSTEM DROP S3 CLIENT CACHE' = 102, 'SYSTEM DROP CACHE' = 103, 'SYSTEM RELOAD CONFIG' = 104, 'SYSTEM RELOAD USERS' = 105, 'SYSTEM RELOAD SYMBOLS' = 106, 'SYSTEM RELOAD DICTIONARY' = 107, 'SYSTEM RELOAD MODEL' = 108, 'SYSTEM RELOAD FUNCTION' = 109, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 110, 'SYSTEM RELOAD' = 111, 'SYSTEM RESTART DISK' = 112, 'SYSTEM MERGES' = 113, 'SYSTEM TTL MERGES' = 114, 'SYSTEM FETCHES' = 115, 'SYSTEM MOVES' = 116, 'SYSTEM DISTRIBUTED SENDS' = 117, 'SYSTEM REPLICATED SENDS' = 118, 'SYSTEM SENDS' = 119, 'SYSTEM REPLICATION QUEUES' = 120, 'SYSTEM DROP REPLICA' = 121, 'SYSTEM SYNC REPLICA' = 122, 'SYSTEM RESTART REPLICA' = 123, 'SYSTEM RESTORE REPLICA' = 124, 'SYSTEM WAIT LOADING PARTS' = 125, 'SYSTEM SYNC DATABASE REPLICA' = 126, 'SYSTEM SYNC TRANSACTION LOG' = 127, 'SYSTEM SYNC FILE CACHE' = 128, 'SYSTEM FLUSH DISTRIBUTED' = 129, 'SYSTEM FLUSH LOGS' = 130, 'SYSTEM FLUSH' = 131, 'SYSTEM THREAD FUZZER' = 132, 'SYSTEM UNFREEZE' = 133, 'SYSTEM' = 134, 'dictGet' = 135, 'addressToLine' = 136, 'addressToLineWithInlines' = 137, 'addressToSymbol' = 138, 'demangle' = 139, 'INTROSPECTION' = 140, 'FILE' = 141, 'URL' = 142, 'REMOTE' = 143, 'MONGO' = 144, 'MEILISEARCH' = 145, 'MYSQL' = 146, 'POSTGRES' = 147, 'SQLITE' = 148, 'ODBC' = 149, 'JDBC' = 150, 'HDFS' = 151, 'S3' = 152, 'HIVE' = 153, 'SOURCES' = 154, 'CLUSTER' = 155, 'ALL' = 156, 'NONE' = 157)) + `parent_group` Nullable(Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'SHOW NAMED COLLECTIONS SECRETS' = 92, 'ACCESS MANAGEMENT' = 93, 'SYSTEM SHUTDOWN' = 94, 'SYSTEM DROP DNS CACHE' = 95, 'SYSTEM DROP MARK CACHE' = 96, 'SYSTEM DROP UNCOMPRESSED CACHE' = 97, 'SYSTEM DROP MMAP CACHE' = 98, 'SYSTEM DROP QUERY CACHE' = 99, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 100, 'SYSTEM DROP FILESYSTEM CACHE' = 101, 'SYSTEM DROP SCHEMA CACHE' = 102, 'SYSTEM DROP S3 CLIENT CACHE' = 103, 'SYSTEM DROP CACHE' = 104, 'SYSTEM RELOAD CONFIG' = 105, 'SYSTEM RELOAD USERS' = 106, 'SYSTEM RELOAD SYMBOLS' = 107, 'SYSTEM RELOAD DICTIONARY' = 108, 'SYSTEM RELOAD MODEL' = 109, 'SYSTEM RELOAD FUNCTION' = 110, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 111, 'SYSTEM RELOAD' = 112, 'SYSTEM RESTART DISK' = 113, 'SYSTEM MERGES' = 114, 'SYSTEM TTL MERGES' = 115, 'SYSTEM FETCHES' = 116, 'SYSTEM MOVES' = 117, 'SYSTEM DISTRIBUTED SENDS' = 118, 'SYSTEM REPLICATED SENDS' = 119, 'SYSTEM SENDS' = 120, 'SYSTEM REPLICATION QUEUES' = 121, 'SYSTEM DROP REPLICA' = 122, 'SYSTEM SYNC REPLICA' = 123, 'SYSTEM RESTART REPLICA' = 124, 'SYSTEM RESTORE REPLICA' = 125, 'SYSTEM WAIT LOADING PARTS' = 126, 'SYSTEM SYNC DATABASE REPLICA' = 127, 'SYSTEM SYNC TRANSACTION LOG' = 128, 'SYSTEM SYNC FILE CACHE' = 129, 'SYSTEM FLUSH DISTRIBUTED' = 130, 'SYSTEM FLUSH LOGS' = 131, 'SYSTEM FLUSH' = 132, 'SYSTEM THREAD FUZZER' = 133, 'SYSTEM UNFREEZE' = 134, 'SYSTEM' = 135, 'dictGet' = 136, 'addressToLine' = 137, 'addressToLineWithInlines' = 138, 'addressToSymbol' = 139, 'demangle' = 140, 'INTROSPECTION' = 141, 'FILE' = 142, 'URL' = 143, 'REMOTE' = 144, 'MONGO' = 145, 'MEILISEARCH' = 146, 'MYSQL' = 147, 'POSTGRES' = 148, 'SQLITE' = 149, 'ODBC' = 150, 'JDBC' = 151, 'HDFS' = 152, 'S3' = 153, 'HIVE' = 154, 'SOURCES' = 155, 'CLUSTER' = 156, 'ALL' = 157, 'NONE' = 158)) ) ENGINE = SystemPrivileges COMMENT 'SYSTEM TABLE is built on the fly.' From ec8b6c5590ff64b6ab1045a385b61ab04736861b Mon Sep 17 00:00:00 2001 From: lzydmxy <13126752315@163.com> Date: Wed, 22 Feb 2023 19:57:56 +0800 Subject: [PATCH 540/717] add __init__.py for integration test test_move_partition_to_disk_on_cluster --- .../test_move_partition_to_disk_on_cluster/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/integration/test_move_partition_to_disk_on_cluster/__init__.py diff --git a/tests/integration/test_move_partition_to_disk_on_cluster/__init__.py b/tests/integration/test_move_partition_to_disk_on_cluster/__init__.py new file mode 100644 index 00000000000..e69de29bb2d From fba2ec30a2a3d117e95f591fbfc635887e27ed6a Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 22 Feb 2023 13:53:43 +0100 Subject: [PATCH 541/717] fix style check --- .../test.py | 34 +++++++++---------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/tests/integration/test_move_partition_to_disk_on_cluster/test.py b/tests/integration/test_move_partition_to_disk_on_cluster/test.py index fe8606bd549..90753fc8ce3 100644 --- a/tests/integration/test_move_partition_to_disk_on_cluster/test.py +++ b/tests/integration/test_move_partition_to_disk_on_cluster/test.py @@ -43,10 +43,10 @@ def test_move_partition_to_disk_on_cluster(start_cluster): for node in [node1, node2]: node.query( sql="CREATE TABLE test_local_table" - "(x UInt64) " - "ENGINE=ReplicatedMergeTree('/clickhouse/tables/test_local_table', '{replica}') " - "ORDER BY tuple()" - "SETTINGS storage_policy = 'jbod_with_external';", + "(x UInt64) " + "ENGINE=ReplicatedMergeTree('/clickhouse/tables/test_local_table', '{replica}') " + "ORDER BY tuple()" + "SETTINGS storage_policy = 'jbod_with_external';", ) node1.query("INSERT INTO test_local_table VALUES (0)") @@ -61,10 +61,10 @@ def test_move_partition_to_disk_on_cluster(start_cluster): for node in [node1, node2]: assert ( - node.query( - "SELECT partition_id, disk_name FROM system.parts WHERE table = 'test_local_table' FORMAT Values" - ) - == "('all','jbod1')" + node.query( + "SELECT partition_id, disk_name FROM system.parts WHERE table = 'test_local_table' FORMAT Values" + ) + == "('all','jbod1')" ) node1.query( @@ -73,10 +73,10 @@ def test_move_partition_to_disk_on_cluster(start_cluster): for node in [node1, node2]: assert ( - node.query( - "SELECT partition_id, disk_name FROM system.parts WHERE table = 'test_local_table' FORMAT Values" - ) - == "('all','external')" + node.query( + "SELECT partition_id, disk_name FROM system.parts WHERE table = 'test_local_table' FORMAT Values" + ) + == "('all','external')" ) node1.query( @@ -85,10 +85,8 @@ def test_move_partition_to_disk_on_cluster(start_cluster): for node in [node1, node2]: assert ( - node.query( - "SELECT partition_id, disk_name FROM system.parts WHERE table = 'test_local_table' FORMAT Values" - ) - == "('all','jbod1')" + node.query( + "SELECT partition_id, disk_name FROM system.parts WHERE table = 'test_local_table' FORMAT Values" + ) + == "('all','jbod1')" ) - - From e8094c9707d46ef950232e475f671627b809a46b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Wed, 22 Feb 2023 14:20:48 +0100 Subject: [PATCH 542/717] Add test for #46724 --- ...2667_order_by_aggregation_result.reference | 3 +++ .../02667_order_by_aggregation_result.sql | 24 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/tests/queries/0_stateless/02667_order_by_aggregation_result.reference b/tests/queries/0_stateless/02667_order_by_aggregation_result.reference index 642fc2ed645..a89e39d87b3 100644 --- a/tests/queries/0_stateless/02667_order_by_aggregation_result.reference +++ b/tests/queries/0_stateless/02667_order_by_aggregation_result.reference @@ -2,3 +2,6 @@ 0 1 █████████████████████████████████████████████████▉ 1 2 0.5 1 0.1 1.1 +00000000-0000-0000-0000-000000000000 b 1 +417ddc5d-e556-4d27-95dd-a34d84e46a50 c 1 +notEmpty a 1 diff --git a/tests/queries/0_stateless/02667_order_by_aggregation_result.sql b/tests/queries/0_stateless/02667_order_by_aggregation_result.sql index 277430888d7..3fef0374d83 100644 --- a/tests/queries/0_stateless/02667_order_by_aggregation_result.sql +++ b/tests/queries/0_stateless/02667_order_by_aggregation_result.sql @@ -34,3 +34,27 @@ SELECT final_col + 1 AS final_col2 FROM ttttttt GROUP BY col3; + +-- https://github.com/ClickHouse/ClickHouse/issues/46724 + +CREATE TABLE table1 +( + id String, + device UUID +) +ENGINE = MergeTree() ORDER BY tuple(); + +INSERT INTO table1 VALUES ('notEmpty', '417ddc5d-e556-4d27-95dd-a34d84e46a50'); +INSERT INTO table1 VALUES ('', '417ddc5d-e556-4d27-95dd-a34d84e46a50'); +INSERT INTO table1 VALUES ('', '00000000-0000-0000-0000-000000000000'); + +SELECT + if(empty(id), toString(device), id) AS device, + multiIf( + notEmpty(id),'a', + device == '00000000-0000-0000-0000-000000000000', 'b', + 'c' ) AS device_id_type, + count() +FROM table1 +GROUP BY device, device_id_type +ORDER BY device; From bac464f89b3fdfe612e721a2fc976146e17dd696 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 22 Feb 2023 13:57:30 +0100 Subject: [PATCH 543/717] Fix --- docs/en/engines/table-engines/integrations/deltalake.md | 2 +- docs/en/engines/table-engines/integrations/hudi.md | 2 +- docs/en/engines/table-engines/integrations/iceberg.md | 2 +- docs/en/sql-reference/table-functions/iceberg.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/en/engines/table-engines/integrations/deltalake.md b/docs/en/engines/table-engines/integrations/deltalake.md index a2816c7ff57..99183ac7308 100644 --- a/docs/en/engines/table-engines/integrations/deltalake.md +++ b/docs/en/engines/table-engines/integrations/deltalake.md @@ -21,7 +21,7 @@ CREATE TABLE deltalake - `url` — Bucket url with path to the existing Delta Lake table. - `aws_access_key_id`, `aws_secret_access_key` - Long-term credentials for the [AWS](https://aws.amazon.com/) account user. You can use these to authenticate your requests. Parameter is optional. If credentials are not specified, they are used from the configuration file. -Engine parameters can be specified using [Named Collections](../operations/settings/named-collections.md) +Engine parameters can be specified using [Named Collections](../../../operations/named-collections.md) **Example** diff --git a/docs/en/engines/table-engines/integrations/hudi.md b/docs/en/engines/table-engines/integrations/hudi.md index 6ff998d86d9..a14134ecdfa 100644 --- a/docs/en/engines/table-engines/integrations/hudi.md +++ b/docs/en/engines/table-engines/integrations/hudi.md @@ -21,7 +21,7 @@ CREATE TABLE hudi_table - `url` — Bucket url with the path to an existing Hudi table. - `aws_access_key_id`, `aws_secret_access_key` - Long-term credentials for the [AWS](https://aws.amazon.com/) account user. You can use these to authenticate your requests. Parameter is optional. If credentials are not specified, they are used from the configuration file. -Engine parameters can be specified using [Named Collections](../operations/settings/named-collections.md) +Engine parameters can be specified using [Named Collections](../../../operations/named-collections.md) **Example** diff --git a/docs/en/engines/table-engines/integrations/iceberg.md b/docs/en/engines/table-engines/integrations/iceberg.md index 33ec5f877bf..4322fc6b773 100644 --- a/docs/en/engines/table-engines/integrations/iceberg.md +++ b/docs/en/engines/table-engines/integrations/iceberg.md @@ -21,7 +21,7 @@ CREATE TABLE iceberg_table - `url` — url with the path to an existing Iceberg table. - `aws_access_key_id`, `aws_secret_access_key` - Long-term credentials for the [AWS](https://aws.amazon.com/) account user. You can use these to authenticate your requests. Parameter is optional. If credentials are not specified, they are used from the configuration file. -Engine parameters can be specified using [Named Collections](../operations/settings/named-collections.md) +Engine parameters can be specified using [Named Collections](../../../operations/named-collections.md) **Example** diff --git a/docs/en/sql-reference/table-functions/iceberg.md b/docs/en/sql-reference/table-functions/iceberg.md index 036c1379847..fda4d274005 100644 --- a/docs/en/sql-reference/table-functions/iceberg.md +++ b/docs/en/sql-reference/table-functions/iceberg.md @@ -20,7 +20,7 @@ iceberg(url [,aws_access_key_id, aws_secret_access_key] [,format] [,structure]) - `format` — The [format](/docs/en/interfaces/formats.md/#formats) of the file. By default `Parquet` is used. - `structure` — Structure of the table. Format `'column1_name column1_type, column2_name column2_type, ...'`. -Engine parameters can be specified using [Named Collections](../operations/settings/named-collections.md) +Engine parameters can be specified using [Named Collections](../../operations/named-collections.md) **Returned value** From c4761d6cc688733124081b87a5fefffd1d7541ce Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 22 Feb 2023 12:14:59 +0100 Subject: [PATCH 544/717] Fix checks --- src/Disks/getOrCreateDiskFromAST.cpp | 10 +--------- src/Disks/getOrCreateDiskFromAST.h | 5 ----- src/Parsers/FieldFromAST.cpp | 1 + src/Parsers/isDiskFunction.cpp | 16 ++++++++++++++++ src/Parsers/isDiskFunction.h | 9 +++++++++ src/Storages/MergeTree/MergeTreeSettings.cpp | 1 + .../integration/test_disk_configuration/test.py | 2 +- 7 files changed, 29 insertions(+), 15 deletions(-) create mode 100644 src/Parsers/isDiskFunction.cpp create mode 100644 src/Parsers/isDiskFunction.h diff --git a/src/Disks/getOrCreateDiskFromAST.cpp b/src/Disks/getOrCreateDiskFromAST.cpp index fc9cd7edbee..997bd2c853f 100644 --- a/src/Disks/getOrCreateDiskFromAST.cpp +++ b/src/Disks/getOrCreateDiskFromAST.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include namespace DB @@ -17,15 +18,6 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } -bool isDiskFunction(ASTPtr ast) -{ - if (!ast) - return false; - - const auto * function = ast->as(); - return function && function->name == "disk" && function->arguments->as(); -} - std::string getOrCreateDiskFromDiskAST(const ASTFunction & function, ContextPtr context) { /// We need a unique name for a created custom disk, but it needs to be the same diff --git a/src/Disks/getOrCreateDiskFromAST.h b/src/Disks/getOrCreateDiskFromAST.h index c1d4bda1a49..7c64707b0bd 100644 --- a/src/Disks/getOrCreateDiskFromAST.h +++ b/src/Disks/getOrCreateDiskFromAST.h @@ -15,9 +15,4 @@ class ASTFunction; */ std::string getOrCreateDiskFromDiskAST(const ASTFunction & function, ContextPtr context); -/* - * Is given ast has form of a disk() function. - */ -bool isDiskFunction(ASTPtr ast); - } diff --git a/src/Parsers/FieldFromAST.cpp b/src/Parsers/FieldFromAST.cpp index 5889699c081..c46a9a08e68 100644 --- a/src/Parsers/FieldFromAST.cpp +++ b/src/Parsers/FieldFromAST.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include diff --git a/src/Parsers/isDiskFunction.cpp b/src/Parsers/isDiskFunction.cpp new file mode 100644 index 00000000000..e60229cb3f7 --- /dev/null +++ b/src/Parsers/isDiskFunction.cpp @@ -0,0 +1,16 @@ +#include +#include + +namespace DB +{ + +bool isDiskFunction(ASTPtr ast) +{ + if (!ast) + return false; + + const auto * function = ast->as(); + return function && function->name == "disk" && function->arguments->as(); +} + +} diff --git a/src/Parsers/isDiskFunction.h b/src/Parsers/isDiskFunction.h new file mode 100644 index 00000000000..97b3c58fa17 --- /dev/null +++ b/src/Parsers/isDiskFunction.h @@ -0,0 +1,9 @@ +#pragma once +#include + +namespace DB +{ + +bool isDiskFunction(ASTPtr ast); + +} diff --git a/src/Storages/MergeTree/MergeTreeSettings.cpp b/src/Storages/MergeTree/MergeTreeSettings.cpp index e5af0c772ba..e951b8f54cf 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.cpp +++ b/src/Storages/MergeTree/MergeTreeSettings.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include diff --git a/tests/integration/test_disk_configuration/test.py b/tests/integration/test_disk_configuration/test.py index 60d75e4dac1..34f8bea219f 100644 --- a/tests/integration/test_disk_configuration/test.py +++ b/tests/integration/test_disk_configuration/test.py @@ -262,7 +262,7 @@ def test_merge_tree_custom_disk_setting(start_cluster): ) expected = """ - SETTINGS disk = disk(type = s3, endpoint = \\'http://minio1:9001/root/data2/\\', access_key_id = \\'minio\\', secret_access_key = \\'minio123\\'), index_granularity = 8192 + SETTINGS disk = disk(type = s3, endpoint = \\'[HIDDEN]\\', access_key_id = \\'[HIDDEN]\\', secret_access_key = \\'[HIDDEN]\\'), index_granularity = 8192 """ assert expected.strip() in node1.query(f"SHOW CREATE TABLE {TABLE_NAME}_4").strip() From e433ecc18f896bb61193ea059ed217502d0f2101 Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Wed, 22 Feb 2023 14:37:55 +0100 Subject: [PATCH 545/717] Better exception message during Tuple JSON deserialization --- src/DataTypes/Serializations/SerializationTuple.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/DataTypes/Serializations/SerializationTuple.cpp b/src/DataTypes/Serializations/SerializationTuple.cpp index ce15e099222..2b703a15a9b 100644 --- a/src/DataTypes/Serializations/SerializationTuple.cpp +++ b/src/DataTypes/Serializations/SerializationTuple.cpp @@ -231,7 +231,15 @@ void SerializationTuple::deserializeTextJSON(IColumn & column, ReadBuffer & istr seen_elements[element_pos] = 1; auto & element_column = extractElementColumn(column, element_pos); - elems[element_pos]->deserializeTextJSON(element_column, istr, settings); + try + { + elems[element_pos]->deserializeTextJSON(element_column, istr, settings); + } + catch (Exception & e) + { + e.addMessage("(while reading the value of nested key " + name + ")"); + throw; + } skipWhitespaceIfAny(istr); ++processed; From 4fd4e77737b4f1aa29898849baf289c7e05b8710 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 22 Feb 2023 13:48:29 +0000 Subject: [PATCH 546/717] Poco: POCO_HAVE_INT64 is always defined --- .../Foundation/include/Poco/BinaryReader.h | 4 +-- .../Foundation/include/Poco/BinaryWriter.h | 4 +-- base/poco/Foundation/include/Poco/ByteOrder.h | 29 ------------------- .../Foundation/include/Poco/NumberFormatter.h | 6 ---- .../Foundation/include/Poco/NumberParser.h | 2 -- .../Foundation/include/Poco/StreamCopier.h | 6 ---- base/poco/Foundation/include/Poco/Token.h | 2 -- base/poco/Foundation/include/Poco/Types.h | 1 - base/poco/Foundation/src/BinaryReader.cpp | 4 +-- base/poco/Foundation/src/BinaryWriter.cpp | 4 +-- base/poco/Foundation/src/NumberFormatter.cpp | 2 -- base/poco/Foundation/src/NumberParser.cpp | 2 -- base/poco/Foundation/src/StreamCopier.cpp | 6 ---- base/poco/Foundation/src/Token.cpp | 2 -- base/poco/JSON/include/Poco/JSON/Handler.h | 2 -- .../JSON/include/Poco/JSON/ParseHandler.h | 4 --- .../JSON/include/Poco/JSON/PrintHandler.h | 2 -- base/poco/JSON/src/PrintHandler.cpp | 2 -- .../include/Poco/Net/HTTPFixedLengthStream.h | 4 --- base/poco/Net/include/Poco/Net/HTTPMessage.h | 2 -- base/poco/Net/src/HTTPClientSession.cpp | 8 ----- base/poco/Net/src/HTTPMessage.cpp | 2 -- base/poco/Net/src/HTTPServerRequestImpl.cpp | 4 --- base/poco/Net/src/HTTPServerResponseImpl.cpp | 8 ----- .../include/Poco/Util/AbstractConfiguration.h | 6 ---- .../Util/include/Poco/Util/WinRegistryKey.h | 2 -- base/poco/Util/src/AbstractConfiguration.cpp | 4 --- 27 files changed, 4 insertions(+), 120 deletions(-) diff --git a/base/poco/Foundation/include/Poco/BinaryReader.h b/base/poco/Foundation/include/Poco/BinaryReader.h index 280724a8a47..4042b507a2f 100644 --- a/base/poco/Foundation/include/Poco/BinaryReader.h +++ b/base/poco/Foundation/include/Poco/BinaryReader.h @@ -76,7 +76,7 @@ public: BinaryReader & operator>>(float & value); BinaryReader & operator>>(double & value); -#if defined(POCO_HAVE_INT64) && !defined(POCO_LONG_IS_64_BIT) +#if !defined(POCO_LONG_IS_64_BIT) BinaryReader & operator>>(Int64 & value); BinaryReader & operator>>(UInt64 & value); #endif @@ -106,12 +106,10 @@ public: /// See BinaryWriter::write7BitEncoded() for a description /// of the compression algorithm. -#if defined(POCO_HAVE_INT64) void read7BitEncoded(UInt64 & value); /// Reads a 64-bit unsigned integer in compressed format. /// See BinaryWriter::write7BitEncoded() for a description /// of the compression algorithm. -#endif void readRaw(std::streamsize length, std::string & value); /// Reads length bytes of raw data into value. diff --git a/base/poco/Foundation/include/Poco/BinaryWriter.h b/base/poco/Foundation/include/Poco/BinaryWriter.h index 30a353a8ff7..aa280d4ccab 100644 --- a/base/poco/Foundation/include/Poco/BinaryWriter.h +++ b/base/poco/Foundation/include/Poco/BinaryWriter.h @@ -81,7 +81,7 @@ public: BinaryWriter & operator<<(float value); BinaryWriter & operator<<(double value); -#if defined(POCO_HAVE_INT64) && !defined(POCO_LONG_IS_64_BIT) +#if !defined(POCO_LONG_IS_64_BIT) BinaryWriter & operator<<(Int64 value); BinaryWriter & operator<<(UInt64 value); #endif @@ -114,7 +114,6 @@ public: /// written out. value is then shifted by seven bits and the next byte is written. /// This process is repeated until the entire integer has been written. -#if defined(POCO_HAVE_INT64) void write7BitEncoded(UInt64 value); /// Writes a 64-bit unsigned integer in a compressed format. /// The value written out seven bits at a time, starting @@ -125,7 +124,6 @@ public: /// If value will not fit in seven bits, the high bit is set on the first byte and /// written out. value is then shifted by seven bits and the next byte is written. /// This process is repeated until the entire integer has been written. -#endif void writeRaw(const std::string & rawData); /// Writes the string as-is to the stream. diff --git a/base/poco/Foundation/include/Poco/ByteOrder.h b/base/poco/Foundation/include/Poco/ByteOrder.h index 4f2644ddf4e..09f673c2718 100644 --- a/base/poco/Foundation/include/Poco/ByteOrder.h +++ b/base/poco/Foundation/include/Poco/ByteOrder.h @@ -34,64 +34,50 @@ public: static UInt16 flipBytes(UInt16 value); static Int32 flipBytes(Int32 value); static UInt32 flipBytes(UInt32 value); -#if defined(POCO_HAVE_INT64) static Int64 flipBytes(Int64 value); static UInt64 flipBytes(UInt64 value); -#endif static Int16 toBigEndian(Int16 value); static UInt16 toBigEndian(UInt16 value); static Int32 toBigEndian(Int32 value); static UInt32 toBigEndian(UInt32 value); -#if defined(POCO_HAVE_INT64) static Int64 toBigEndian(Int64 value); static UInt64 toBigEndian(UInt64 value); -#endif static Int16 fromBigEndian(Int16 value); static UInt16 fromBigEndian(UInt16 value); static Int32 fromBigEndian(Int32 value); static UInt32 fromBigEndian(UInt32 value); -#if defined(POCO_HAVE_INT64) static Int64 fromBigEndian(Int64 value); static UInt64 fromBigEndian(UInt64 value); -#endif static Int16 toLittleEndian(Int16 value); static UInt16 toLittleEndian(UInt16 value); static Int32 toLittleEndian(Int32 value); static UInt32 toLittleEndian(UInt32 value); -#if defined(POCO_HAVE_INT64) static Int64 toLittleEndian(Int64 value); static UInt64 toLittleEndian(UInt64 value); -#endif static Int16 fromLittleEndian(Int16 value); static UInt16 fromLittleEndian(UInt16 value); static Int32 fromLittleEndian(Int32 value); static UInt32 fromLittleEndian(UInt32 value); -#if defined(POCO_HAVE_INT64) static Int64 fromLittleEndian(Int64 value); static UInt64 fromLittleEndian(UInt64 value); -#endif static Int16 toNetwork(Int16 value); static UInt16 toNetwork(UInt16 value); static Int32 toNetwork(Int32 value); static UInt32 toNetwork(UInt32 value); -#if defined(POCO_HAVE_INT64) static Int64 toNetwork(Int64 value); static UInt64 toNetwork(UInt64 value); -#endif static Int16 fromNetwork(Int16 value); static UInt16 fromNetwork(UInt16 value); static Int32 fromNetwork(Int32 value); static UInt32 fromNetwork(UInt32 value); -#if defined(POCO_HAVE_INT64) static Int64 fromNetwork(Int64 value); static UInt64 fromNetwork(UInt64 value); -#endif }; @@ -143,7 +129,6 @@ inline Int32 ByteOrder::flipBytes(Int32 value) } -#if defined(POCO_HAVE_INT64) inline UInt64 ByteOrder::flipBytes(UInt64 value) { # if defined(POCO_HAVE_MSC_BYTESWAP) @@ -162,7 +147,6 @@ inline Int64 ByteOrder::flipBytes(Int64 value) { return Int64(flipBytes(UInt64(value))); } -#endif // POCO_HAVE_INT64 // @@ -180,7 +164,6 @@ inline Int64 ByteOrder::flipBytes(Int64 value) } -#if defined(POCO_HAVE_INT64) # define POCO_IMPLEMENT_BYTEORDER_NOOP(op) \ POCO_IMPLEMENT_BYTEORDER_NOOP_(op, Int16) \ POCO_IMPLEMENT_BYTEORDER_NOOP_(op, UInt16) \ @@ -195,18 +178,6 @@ inline Int64 ByteOrder::flipBytes(Int64 value) POCO_IMPLEMENT_BYTEORDER_FLIP_(op, UInt32) \ POCO_IMPLEMENT_BYTEORDER_FLIP_(op, Int64) \ POCO_IMPLEMENT_BYTEORDER_FLIP_(op, UInt64) -#else -# define POCO_IMPLEMENT_BYTEORDER_NOOP(op) \ - POCO_IMPLEMENT_BYTEORDER_NOOP_(op, Int16) \ - POCO_IMPLEMENT_BYTEORDER_NOOP_(op, UInt16) \ - POCO_IMPLEMENT_BYTEORDER_NOOP_(op, Int32) \ - POCO_IMPLEMENT_BYTEORDER_NOOP_(op, UInt32) -# define POCO_IMPLEMENT_BYTEORDER_FLIP(op) \ - POCO_IMPLEMENT_BYTEORDER_FLIP_(op, Int16) \ - POCO_IMPLEMENT_BYTEORDER_FLIP_(op, UInt16) \ - POCO_IMPLEMENT_BYTEORDER_FLIP_(op, Int32) \ - POCO_IMPLEMENT_BYTEORDER_FLIP_(op, UInt32) -#endif #if defined(POCO_ARCH_BIG_ENDIAN) diff --git a/base/poco/Foundation/include/Poco/NumberFormatter.h b/base/poco/Foundation/include/Poco/NumberFormatter.h index e246ca16ec3..a320b576083 100644 --- a/base/poco/Foundation/include/Poco/NumberFormatter.h +++ b/base/poco/Foundation/include/Poco/NumberFormatter.h @@ -151,7 +151,6 @@ public: /// If prefix is true, "0x" prefix is prepended to the /// resulting string. -#ifdef POCO_HAVE_INT64 # ifdef POCO_LONG_IS_64_BIT @@ -255,7 +254,6 @@ public: # endif // ifdef POCO_LONG_IS_64_BIT -#endif // ifdef POCO_HAVE_INT64 static std::string format(float value); /// Formats a float value in decimal floating-point notation, @@ -380,7 +378,6 @@ public: /// right justified and zero-padded in a field having at least the /// specified width. -#ifdef POCO_HAVE_INT64 # ifdef POCO_LONG_IS_64_BIT @@ -472,7 +469,6 @@ public: # endif // ifdef POCO_LONG_IS_64_BIT -#endif // ifdef POCO_HAVE_INT64 static void append(std::string & str, float value); /// Formats a float value in decimal floating-point notation, @@ -673,7 +669,6 @@ inline std::string NumberFormatter::formatHex(unsigned long value, int width, bo } -#ifdef POCO_HAVE_INT64 # ifdef POCO_LONG_IS_64_BIT @@ -843,7 +838,6 @@ inline std::string NumberFormatter::formatHex(UInt64 value, int width, bool pref # endif // ifdef POCO_LONG_IS_64_BIT -#endif // ifdef POCO_HAVE_INT64 inline std::string NumberFormatter::format(float value) diff --git a/base/poco/Foundation/include/Poco/NumberParser.h b/base/poco/Foundation/include/Poco/NumberParser.h index de813e37dae..32f8c0dc989 100644 --- a/base/poco/Foundation/include/Poco/NumberParser.h +++ b/base/poco/Foundation/include/Poco/NumberParser.h @@ -80,7 +80,6 @@ public: /// Returns true if a valid integer has been found, false otherwise. /// If parsing was not successful, value is undefined. -#if defined(POCO_HAVE_INT64) static Int64 parse64(const std::string & s, char thousandSeparator = ','); /// Parses a 64-bit integer value in decimal notation from the given string. @@ -118,7 +117,6 @@ public: /// Returns true if a valid integer has been found, false otherwise. /// If parsing was not successful, value is undefined. -#endif // defined(POCO_HAVE_INT64) static double parseFloat(const std::string & s, char decimalSeparator = '.', char thousandSeparator = ','); /// Parses a double value in decimal floating point notation diff --git a/base/poco/Foundation/include/Poco/StreamCopier.h b/base/poco/Foundation/include/Poco/StreamCopier.h index 72b19306388..c24e73d88dd 100644 --- a/base/poco/Foundation/include/Poco/StreamCopier.h +++ b/base/poco/Foundation/include/Poco/StreamCopier.h @@ -38,7 +38,6 @@ public: /// /// Returns the number of bytes copied. -#if defined(POCO_HAVE_INT64) static Poco::UInt64 copyStream64(std::istream & istr, std::ostream & ostr, std::size_t bufferSize = 8192); /// Writes all bytes readable from istr to ostr, using an internal buffer. /// @@ -46,14 +45,12 @@ public: /// /// Note: the only difference to copyStream() is that a 64-bit unsigned /// integer is used to count the number of bytes copied. -#endif static std::streamsize copyStreamUnbuffered(std::istream & istr, std::ostream & ostr); /// Writes all bytes readable from istr to ostr. /// /// Returns the number of bytes copied. -#if defined(POCO_HAVE_INT64) static Poco::UInt64 copyStreamUnbuffered64(std::istream & istr, std::ostream & ostr); /// Writes all bytes readable from istr to ostr. /// @@ -61,14 +58,12 @@ public: /// /// Note: the only difference to copyStreamUnbuffered() is that a 64-bit unsigned /// integer is used to count the number of bytes copied. -#endif static std::streamsize copyToString(std::istream & istr, std::string & str, std::size_t bufferSize = 8192); /// Appends all bytes readable from istr to the given string, using an internal buffer. /// /// Returns the number of bytes copied. -#if defined(POCO_HAVE_INT64) static Poco::UInt64 copyToString64(std::istream & istr, std::string & str, std::size_t bufferSize = 8192); /// Appends all bytes readable from istr to the given string, using an internal buffer. /// @@ -76,7 +71,6 @@ public: /// /// Note: the only difference to copyToString() is that a 64-bit unsigned /// integer is used to count the number of bytes copied. -#endif }; diff --git a/base/poco/Foundation/include/Poco/Token.h b/base/poco/Foundation/include/Poco/Token.h index 2d62ed87de6..1aec9e620fe 100644 --- a/base/poco/Foundation/include/Poco/Token.h +++ b/base/poco/Foundation/include/Poco/Token.h @@ -84,13 +84,11 @@ public: virtual std::string asString() const; /// Returns a string representation of the token. -#if defined(POCO_HAVE_INT64) virtual Int64 asInteger64() const; /// Returns a 64-bit integer representation of the token. virtual UInt64 asUnsignedInteger64() const; /// Returns an unsigned 64-bit integer representation of the token. -#endif virtual int asInteger() const; /// Returns an integer representation of the token. diff --git a/base/poco/Foundation/include/Poco/Types.h b/base/poco/Foundation/include/Poco/Types.h index 156b3584d15..d10047344f6 100644 --- a/base/poco/Foundation/include/Poco/Types.h +++ b/base/poco/Foundation/include/Poco/Types.h @@ -46,7 +46,6 @@ typedef unsigned long UInt64; typedef signed long long Int64; typedef unsigned long long UInt64; # endif -# define POCO_HAVE_INT64 1 #endif diff --git a/base/poco/Foundation/src/BinaryReader.cpp b/base/poco/Foundation/src/BinaryReader.cpp index fb57371fbc3..f2961e03966 100644 --- a/base/poco/Foundation/src/BinaryReader.cpp +++ b/base/poco/Foundation/src/BinaryReader.cpp @@ -170,7 +170,7 @@ BinaryReader& BinaryReader::operator >> (double& value) } -#if defined(POCO_HAVE_INT64) && !defined(POCO_LONG_IS_64_BIT) +#if !defined(POCO_LONG_IS_64_BIT) BinaryReader& BinaryReader::operator >> (Int64& value) @@ -233,7 +233,6 @@ void BinaryReader::read7BitEncoded(UInt32& value) } -#if defined(POCO_HAVE_INT64) void BinaryReader::read7BitEncoded(UInt64& value) @@ -254,7 +253,6 @@ void BinaryReader::read7BitEncoded(UInt64& value) } -#endif void BinaryReader::readRaw(std::streamsize length, std::string& value) diff --git a/base/poco/Foundation/src/BinaryWriter.cpp b/base/poco/Foundation/src/BinaryWriter.cpp index 62e1adfe373..6db5ab7cb90 100644 --- a/base/poco/Foundation/src/BinaryWriter.cpp +++ b/base/poco/Foundation/src/BinaryWriter.cpp @@ -212,7 +212,7 @@ BinaryWriter& BinaryWriter::operator << (double value) } -#if defined(POCO_HAVE_INT64) && !defined(POCO_LONG_IS_64_BIT) +#if !defined(POCO_LONG_IS_64_BIT) BinaryWriter& BinaryWriter::operator << (Int64 value) @@ -303,7 +303,6 @@ void BinaryWriter::write7BitEncoded(UInt32 value) } -#if defined(POCO_HAVE_INT64) void BinaryWriter::write7BitEncoded(UInt64 value) @@ -319,7 +318,6 @@ void BinaryWriter::write7BitEncoded(UInt64 value) } -#endif void BinaryWriter::writeRaw(const std::string& rawData) diff --git a/base/poco/Foundation/src/NumberFormatter.cpp b/base/poco/Foundation/src/NumberFormatter.cpp index 5c8126e9b0a..0a9334059a9 100644 --- a/base/poco/Foundation/src/NumberFormatter.cpp +++ b/base/poco/Foundation/src/NumberFormatter.cpp @@ -234,7 +234,6 @@ void NumberFormatter::appendHex(std::string& str, unsigned long value, int width } -#ifdef POCO_HAVE_INT64 #ifdef POCO_LONG_IS_64_BIT @@ -424,7 +423,6 @@ void NumberFormatter::appendHex(std::string& str, UInt64 value, int width) #endif // ifdef POCO_LONG_IS_64_BIT -#endif // ifdef POCO_HAVE_INT64 void NumberFormatter::append(std::string& str, float value) diff --git a/base/poco/Foundation/src/NumberParser.cpp b/base/poco/Foundation/src/NumberParser.cpp index 56eeb167595..4081f3b2663 100644 --- a/base/poco/Foundation/src/NumberParser.cpp +++ b/base/poco/Foundation/src/NumberParser.cpp @@ -104,7 +104,6 @@ bool NumberParser::tryParseOct(const std::string& s, unsigned& value) } -#if defined(POCO_HAVE_INT64) Int64 NumberParser::parse64(const std::string& s, char thSep) @@ -173,7 +172,6 @@ bool NumberParser::tryParseOct64(const std::string& s, UInt64& value) } -#endif // defined(POCO_HAVE_INT64) double NumberParser::parseFloat(const std::string& s, char decSep, char thSep) diff --git a/base/poco/Foundation/src/StreamCopier.cpp b/base/poco/Foundation/src/StreamCopier.cpp index 6f34cc233a2..508d1e7b2ae 100644 --- a/base/poco/Foundation/src/StreamCopier.cpp +++ b/base/poco/Foundation/src/StreamCopier.cpp @@ -42,7 +42,6 @@ std::streamsize StreamCopier::copyStream(std::istream& istr, std::ostream& ostr, } -#if defined(POCO_HAVE_INT64) Poco::UInt64 StreamCopier::copyStream64(std::istream& istr, std::ostream& ostr, std::size_t bufferSize) { poco_assert (bufferSize > 0); @@ -64,7 +63,6 @@ Poco::UInt64 StreamCopier::copyStream64(std::istream& istr, std::ostream& ostr, } return len; } -#endif std::streamsize StreamCopier::copyToString(std::istream& istr, std::string& str, std::size_t bufferSize) @@ -90,7 +88,6 @@ std::streamsize StreamCopier::copyToString(std::istream& istr, std::string& str, } -#if defined(POCO_HAVE_INT64) Poco::UInt64 StreamCopier::copyToString64(std::istream& istr, std::string& str, std::size_t bufferSize) { poco_assert (bufferSize > 0); @@ -112,7 +109,6 @@ Poco::UInt64 StreamCopier::copyToString64(std::istream& istr, std::string& str, } return len; } -#endif std::streamsize StreamCopier::copyStreamUnbuffered(std::istream& istr, std::ostream& ostr) @@ -130,7 +126,6 @@ std::streamsize StreamCopier::copyStreamUnbuffered(std::istream& istr, std::ostr } -#if defined(POCO_HAVE_INT64) Poco::UInt64 StreamCopier::copyStreamUnbuffered64(std::istream& istr, std::ostream& ostr) { char c = 0; @@ -144,7 +139,6 @@ Poco::UInt64 StreamCopier::copyStreamUnbuffered64(std::istream& istr, std::ostre } return len; } -#endif } // namespace Poco diff --git a/base/poco/Foundation/src/Token.cpp b/base/poco/Foundation/src/Token.cpp index 98e8bb25e93..4e81c6ef885 100644 --- a/base/poco/Foundation/src/Token.cpp +++ b/base/poco/Foundation/src/Token.cpp @@ -54,7 +54,6 @@ std::string Token::asString() const } -#if defined(POCO_HAVE_INT64) Int64 Token::asInteger64() const { return NumberParser::parse64(_value); @@ -65,7 +64,6 @@ UInt64 Token::asUnsignedInteger64() const { return NumberParser::parseUnsigned64(_value); } -#endif int Token::asInteger() const diff --git a/base/poco/JSON/include/Poco/JSON/Handler.h b/base/poco/JSON/include/Poco/JSON/Handler.h index f9114a59221..c412a05003f 100644 --- a/base/poco/JSON/include/Poco/JSON/Handler.h +++ b/base/poco/JSON/include/Poco/JSON/Handler.h @@ -74,14 +74,12 @@ namespace JSON /// An unsigned value is read. This will only be triggered if the /// value cannot fit into a signed int. -#if defined(POCO_HAVE_INT64) virtual void value(Int64 v) = 0; /// A 64-bit integer value is read. virtual void value(UInt64 v) = 0; /// An unsigned 64-bit integer value is read. This will only be /// triggered if the value cannot fit into a signed 64-bit integer. -#endif virtual void value(const std::string & value) = 0; /// A string value is read. diff --git a/base/poco/JSON/include/Poco/JSON/ParseHandler.h b/base/poco/JSON/include/Poco/JSON/ParseHandler.h index 4669dc8638f..1b8ac3066d2 100644 --- a/base/poco/JSON/include/Poco/JSON/ParseHandler.h +++ b/base/poco/JSON/include/Poco/JSON/ParseHandler.h @@ -73,14 +73,12 @@ namespace JSON /// An unsigned value is read. This will only be triggered if the /// value cannot fit into a signed int. -#if defined(POCO_HAVE_INT64) virtual void value(Int64 v); /// A 64-bit integer value is read virtual void value(UInt64 v); /// An unsigned 64-bit integer value is read. This will only be /// triggered if the value cannot fit into a signed 64-bit integer. -#endif virtual void value(const std::string & s); /// A string value is read. @@ -126,7 +124,6 @@ namespace JSON } -#if defined(POCO_HAVE_INT64) inline void ParseHandler::value(Int64 v) { setValue(v); @@ -137,7 +134,6 @@ namespace JSON { setValue(v); } -#endif inline void ParseHandler::value(const std::string & s) diff --git a/base/poco/JSON/include/Poco/JSON/PrintHandler.h b/base/poco/JSON/include/Poco/JSON/PrintHandler.h index 34a991653ba..390f4d8bba9 100644 --- a/base/poco/JSON/include/Poco/JSON/PrintHandler.h +++ b/base/poco/JSON/include/Poco/JSON/PrintHandler.h @@ -81,13 +81,11 @@ namespace JSON /// An unsigned value is read. This will only be triggered if the /// value cannot fit into a signed int. -#if defined(POCO_HAVE_INT64) void value(Int64 v); /// A 64-bit integer value is read; it will be written to the output. void value(UInt64 v); /// An unsigned 64-bit integer value is read; it will be written to the output. -#endif void value(const std::string & value); /// A string value is read; it will be formatted and written to the output. diff --git a/base/poco/JSON/src/PrintHandler.cpp b/base/poco/JSON/src/PrintHandler.cpp index bf735d0869c..ea81cbdd1c0 100644 --- a/base/poco/JSON/src/PrintHandler.cpp +++ b/base/poco/JSON/src/PrintHandler.cpp @@ -154,7 +154,6 @@ void PrintHandler::value(unsigned v) } -#if defined(POCO_HAVE_INT64) void PrintHandler::value(Int64 v) { arrayValue(); @@ -169,7 +168,6 @@ void PrintHandler::value(UInt64 v) _out << v; _objStart = false; } -#endif void PrintHandler::value(const std::string& value) diff --git a/base/poco/Net/include/Poco/Net/HTTPFixedLengthStream.h b/base/poco/Net/include/Poco/Net/HTTPFixedLengthStream.h index dcdd1cfcaf8..4de211fdb92 100644 --- a/base/poco/Net/include/Poco/Net/HTTPFixedLengthStream.h +++ b/base/poco/Net/include/Poco/Net/HTTPFixedLengthStream.h @@ -43,11 +43,7 @@ namespace Net public: typedef HTTPBasicStreamBuf::openmode openmode; -#if defined(POCO_HAVE_INT64) typedef Poco::Int64 ContentLength; -#else - typedef std::streamsize ContentLength; -#endif HTTPFixedLengthStreamBuf(HTTPSession & session, ContentLength length, openmode mode); ~HTTPFixedLengthStreamBuf(); diff --git a/base/poco/Net/include/Poco/Net/HTTPMessage.h b/base/poco/Net/include/Poco/Net/HTTPMessage.h index 5c54bf7306b..0bef50803a8 100644 --- a/base/poco/Net/include/Poco/Net/HTTPMessage.h +++ b/base/poco/Net/include/Poco/Net/HTTPMessage.h @@ -56,7 +56,6 @@ namespace Net /// which may be UNKNOWN_CONTENT_LENGTH if /// no Content-Length header is present. -#if defined(POCO_HAVE_INT64) void setContentLength64(Poco::Int64 length); /// Sets the Content-Length header. /// @@ -73,7 +72,6 @@ namespace Net /// /// In contrast to getContentLength(), this method /// always returns a 64-bit integer for content length. -#endif // defined(POCO_HAVE_INT64) bool hasContentLength() const; /// Returns true iff a Content-Length header is present. diff --git a/base/poco/Net/src/HTTPClientSession.cpp b/base/poco/Net/src/HTTPClientSession.cpp index 323e9526df5..c5697b556d1 100644 --- a/base/poco/Net/src/HTTPClientSession.cpp +++ b/base/poco/Net/src/HTTPClientSession.cpp @@ -264,11 +264,7 @@ std::ostream& HTTPClientSession::sendRequest(HTTPRequest& request) { Poco::CountingOutputStream cs; request.write(cs); -#if POCO_HAVE_INT64 _pRequestStream = new HTTPFixedLengthOutputStream(*this, request.getContentLength64() + cs.chars()); -#else - _pRequestStream = new HTTPFixedLengthOutputStream(*this, request.getContentLength() + cs.chars()); -#endif request.write(*_pRequestStream); } else if ((method != HTTPRequest::HTTP_PUT && method != HTTPRequest::HTTP_POST && method != HTTPRequest::HTTP_PATCH) || request.has(HTTPRequest::UPGRADE)) @@ -334,11 +330,7 @@ std::istream& HTTPClientSession::receiveResponse(HTTPResponse& response) else if (response.getChunkedTransferEncoding()) _pResponseStream = new HTTPChunkedInputStream(*this); else if (response.hasContentLength()) -#if defined(POCO_HAVE_INT64) _pResponseStream = new HTTPFixedLengthInputStream(*this, response.getContentLength64()); -#else - _pResponseStream = new HTTPFixedLengthInputStream(*this, response.getContentLength()); -#endif else _pResponseStream = new HTTPInputStream(*this); diff --git a/base/poco/Net/src/HTTPMessage.cpp b/base/poco/Net/src/HTTPMessage.cpp index debda04c3b3..0cd234ee9cb 100644 --- a/base/poco/Net/src/HTTPMessage.cpp +++ b/base/poco/Net/src/HTTPMessage.cpp @@ -89,7 +89,6 @@ std::streamsize HTTPMessage::getContentLength() const } -#if defined(POCO_HAVE_INT64) void HTTPMessage::setContentLength64(Poco::Int64 length) { if (length != UNKNOWN_CONTENT_LENGTH) @@ -108,7 +107,6 @@ Poco::Int64 HTTPMessage::getContentLength64() const } else return UNKNOWN_CONTENT_LENGTH; } -#endif // defined(POCO_HAVE_INT64) void HTTPMessage::setTransferEncoding(const std::string& transferEncoding) diff --git a/base/poco/Net/src/HTTPServerRequestImpl.cpp b/base/poco/Net/src/HTTPServerRequestImpl.cpp index d8ea7398c9b..d893e49aafb 100644 --- a/base/poco/Net/src/HTTPServerRequestImpl.cpp +++ b/base/poco/Net/src/HTTPServerRequestImpl.cpp @@ -49,11 +49,7 @@ HTTPServerRequestImpl::HTTPServerRequestImpl(HTTPServerResponseImpl& response, H if (getChunkedTransferEncoding()) _pStream = new HTTPChunkedInputStream(session); else if (hasContentLength()) -#if defined(POCO_HAVE_INT64) _pStream = new HTTPFixedLengthInputStream(session, getContentLength64()); -#else - _pStream = new HTTPFixedLengthInputStream(session, getContentLength()); -#endif else if (getMethod() == HTTPRequest::HTTP_GET || getMethod() == HTTPRequest::HTTP_HEAD || getMethod() == HTTPRequest::HTTP_DELETE) _pStream = new HTTPFixedLengthInputStream(session, 0); else diff --git a/base/poco/Net/src/HTTPServerResponseImpl.cpp b/base/poco/Net/src/HTTPServerResponseImpl.cpp index fb6783c633e..55de22c876c 100644 --- a/base/poco/Net/src/HTTPServerResponseImpl.cpp +++ b/base/poco/Net/src/HTTPServerResponseImpl.cpp @@ -92,11 +92,7 @@ std::ostream& HTTPServerResponseImpl::send() { Poco::CountingOutputStream cs; write(cs); -#if defined(POCO_HAVE_INT64) _pStream = new HTTPFixedLengthOutputStream(_session, getContentLength64() + cs.chars()); -#else - _pStream = new HTTPFixedLengthOutputStream(_session, getContentLength() + cs.chars()); -#endif write(*_pStream); } else @@ -153,11 +149,7 @@ void HTTPServerResponseImpl::sendFile(const std::string& path, const std::string Timestamp dateTime = f.getLastModified(); File::FileSize length = f.getSize(); set("Last-Modified", DateTimeFormatter::format(dateTime, DateTimeFormat::HTTP_FORMAT)); -#if defined(POCO_HAVE_INT64) setContentLength64(length); -#else - setContentLength(static_cast(length)); -#endif setContentType(mediaType); setChunkedTransferEncoding(false); diff --git a/base/poco/Util/include/Poco/Util/AbstractConfiguration.h b/base/poco/Util/include/Poco/Util/AbstractConfiguration.h index a0e5e2c50dd..926ac3ba8a9 100644 --- a/base/poco/Util/include/Poco/Util/AbstractConfiguration.h +++ b/base/poco/Util/include/Poco/Util/AbstractConfiguration.h @@ -167,7 +167,6 @@ namespace Util /// If the value contains references to other properties (${}), these /// are expanded. -#if defined(POCO_HAVE_INT64) Int64 getInt64(const std::string & key) const; /// Returns the Int64 value of the property with the given name. @@ -205,7 +204,6 @@ namespace Util /// If the value contains references to other properties (${}), these /// are expanded. -#endif // defined(POCO_HAVE_INT64) double getDouble(const std::string & key) const; /// Returns the double value of the property with the given name. @@ -255,7 +253,6 @@ namespace Util /// Sets the property with the given key to the given value. /// An already existing value for the key is overwritten. -#if defined(POCO_HAVE_INT64) virtual void setInt64(const std::string & key, Int64 value); /// Sets the property with the given key to the given value. @@ -265,7 +262,6 @@ namespace Util /// Sets the property with the given key to the given value. /// An already existing value for the key is overwritten. -#endif // defined(POCO_HAVE_INT64) virtual void setDouble(const std::string & key, double value); /// Sets the property with the given key to the given value. @@ -335,12 +331,10 @@ namespace Util static int parseInt(const std::string & value); static unsigned parseUInt(const std::string & value); -#if defined(POCO_HAVE_INT64) static Int64 parseInt64(const std::string & value); static UInt64 parseUInt64(const std::string & value); -#endif // defined(POCO_HAVE_INT64) static bool parseBool(const std::string & value); void setRawWithEvent(const std::string & key, std::string value); diff --git a/base/poco/Util/include/Poco/Util/WinRegistryKey.h b/base/poco/Util/include/Poco/Util/WinRegistryKey.h index b28f6aefb37..9aa5e35ed8a 100644 --- a/base/poco/Util/include/Poco/Util/WinRegistryKey.h +++ b/base/poco/Util/include/Poco/Util/WinRegistryKey.h @@ -123,7 +123,6 @@ namespace Util /// /// Throws a NotFoundException if the value does not exist. -#if defined(POCO_HAVE_INT64) void setInt64(const std::string & name, Poco::Int64 value); /// Sets the numeric (REG_QWORD) value with the given name. @@ -135,7 +134,6 @@ namespace Util /// /// Throws a NotFoundException if the value does not exist. -#endif // POCO_HAVE_INT64 void deleteValue(const std::string & name); /// Deletes the value with the given name. diff --git a/base/poco/Util/src/AbstractConfiguration.cpp b/base/poco/Util/src/AbstractConfiguration.cpp index 95e8da68a57..2c892decd9a 100644 --- a/base/poco/Util/src/AbstractConfiguration.cpp +++ b/base/poco/Util/src/AbstractConfiguration.cpp @@ -163,7 +163,6 @@ unsigned AbstractConfiguration::getUInt(const std::string& key, unsigned default } -#if defined(POCO_HAVE_INT64) Int64 AbstractConfiguration::getInt64(const std::string& key) const @@ -214,7 +213,6 @@ UInt64 AbstractConfiguration::getUInt64(const std::string& key, UInt64 defaultVa } -#endif // defined(POCO_HAVE_INT64) double AbstractConfiguration::getDouble(const std::string& key) const @@ -283,7 +281,6 @@ void AbstractConfiguration::setUInt(const std::string& key, unsigned int value) } -#if defined(POCO_HAVE_INT64) void AbstractConfiguration::setInt64(const std::string& key, Int64 value) @@ -302,7 +299,6 @@ void AbstractConfiguration::setUInt64(const std::string& key, UInt64 value) } -#endif // defined(POCO_HAVE_INT64) void AbstractConfiguration::setDouble(const std::string& key, double value) From 7f5fb77ed51e6dc8beb842dded898b891972e51d Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Wed, 22 Feb 2023 15:09:48 +0100 Subject: [PATCH 547/717] Increase table retries in cluster copier tests (#46590) --- programs/copier/ClusterCopier.cpp | 4 ++-- tests/integration/test_cluster_copier/test.py | 16 ++++++++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/programs/copier/ClusterCopier.cpp b/programs/copier/ClusterCopier.cpp index 48a3578dd7b..bc882719a08 100644 --- a/programs/copier/ClusterCopier.cpp +++ b/programs/copier/ClusterCopier.cpp @@ -908,7 +908,7 @@ bool ClusterCopier::tryProcessTable(const ConnectionTimeouts & timeouts, TaskTab /// Exit if success if (task_status != TaskStatus::Finished) { - LOG_WARNING(log, "Create destination Tale Failed "); + LOG_WARNING(log, "Create destination table failed "); return false; } @@ -1473,7 +1473,7 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( if (count != 0) { - LOG_INFO(log, "Partition {} piece {}is not empty. In contains {} rows.", task_partition.name, current_piece_number, count); + LOG_INFO(log, "Partition {} piece {} is not empty. In contains {} rows.", task_partition.name, current_piece_number, count); Coordination::Stat stat_shards{}; zookeeper->get(partition_piece.getPartitionPieceShardsPath(), &stat_shards); diff --git a/tests/integration/test_cluster_copier/test.py b/tests/integration/test_cluster_copier/test.py index 0aadcadc064..b261f7e3a39 100644 --- a/tests/integration/test_cluster_copier/test.py +++ b/tests/integration/test_cluster_copier/test.py @@ -565,13 +565,20 @@ def test_copy_with_recovering(started_cluster, use_sample_offset): str(COPYING_FAIL_PROBABILITY), "--experimental-use-sample-offset", "1", + "--max-table-tries", + "10", ], ) else: execute_task( started_cluster, Task1(started_cluster), - ["--copy-fault-probability", str(COPYING_FAIL_PROBABILITY)], + [ + "--copy-fault-probability", + str(COPYING_FAIL_PROBABILITY), + "--max-table-tries", + "10", + ], ) @@ -606,7 +613,12 @@ def test_copy_month_to_week_partition_with_recovering(started_cluster): execute_task( started_cluster, Task2(started_cluster, "test2"), - ["--copy-fault-probability", str(COPYING_FAIL_PROBABILITY)], + [ + "--copy-fault-probability", + str(COPYING_FAIL_PROBABILITY), + "--max-table-tries", + "10", + ], ) From 9ab4944b9ed0d0d126688d153477e5bbfd0d0fdd Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 22 Feb 2023 13:55:55 +0100 Subject: [PATCH 548/717] Handle input_format_null_as_default for nested types Signed-off-by: Azat Khuzhin --- .../Serializations/SerializationArray.cpp | 6 ++- .../Serializations/SerializationMap.cpp | 6 ++- .../Serializations/SerializationNullable.cpp | 6 +-- .../Serializations/SerializationTuple.cpp | 6 ++- ...nore_unknown_keys_in_named_tuple.reference | 8 ++-- ...json_ignore_unknown_keys_in_named_tuple.sh | 23 ++++------ ..._as_default_null_as_empty_nested.reference | 42 +++++++++++++++++++ ...t_null_as_default_null_as_empty_nested.sql | 25 +++++++++++ 8 files changed, 94 insertions(+), 28 deletions(-) create mode 100644 tests/queries/0_stateless/02573_insert_null_as_default_null_as_empty_nested.reference create mode 100644 tests/queries/0_stateless/02573_insert_null_as_default_null_as_empty_nested.sql diff --git a/src/DataTypes/Serializations/SerializationArray.cpp b/src/DataTypes/Serializations/SerializationArray.cpp index 24aa9e8320d..73b232690c7 100644 --- a/src/DataTypes/Serializations/SerializationArray.cpp +++ b/src/DataTypes/Serializations/SerializationArray.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -510,7 +511,10 @@ void SerializationArray::deserializeTextJSON(IColumn & column, ReadBuffer & istr deserializeTextImpl(column, istr, [&](IColumn & nested_column) { - nested->deserializeTextJSON(nested_column, istr, settings); + if (settings.null_as_default) + SerializationNullable::deserializeTextJSONImpl(nested_column, istr, settings, nested); + else + nested->deserializeTextJSON(nested_column, istr, settings); }, false); } diff --git a/src/DataTypes/Serializations/SerializationMap.cpp b/src/DataTypes/Serializations/SerializationMap.cpp index 98067077178..34da0f11cae 100644 --- a/src/DataTypes/Serializations/SerializationMap.cpp +++ b/src/DataTypes/Serializations/SerializationMap.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -211,7 +212,10 @@ void SerializationMap::deserializeTextJSON(IColumn & column, ReadBuffer & istr, deserializeTextImpl(column, istr, [&settings](ReadBuffer & buf, const SerializationPtr & subcolumn_serialization, IColumn & subcolumn) { - subcolumn_serialization->deserializeTextJSON(subcolumn, buf, settings); + if (settings.null_as_default) + SerializationNullable::deserializeTextJSONImpl(subcolumn, buf, settings, subcolumn_serialization); + else + subcolumn_serialization->deserializeTextJSON(subcolumn, buf, settings); }); } diff --git a/src/DataTypes/Serializations/SerializationNullable.cpp b/src/DataTypes/Serializations/SerializationNullable.cpp index 8b0bdc05d00..20188f7cec5 100644 --- a/src/DataTypes/Serializations/SerializationNullable.cpp +++ b/src/DataTypes/Serializations/SerializationNullable.cpp @@ -219,13 +219,9 @@ static ReturnType safeDeserialize( /// Deserialize value into non-nullable column. In case of NULL, insert default value and return false. template , ReturnType>* = nullptr> static ReturnType safeDeserialize( - IColumn & column, const ISerialization & nested, + IColumn & column, const ISerialization &, CheckForNull && check_for_null, DeserializeNested && deserialize_nested) { - assert(!dynamic_cast(&column)); - assert(!dynamic_cast(&nested)); - UNUSED(nested); - bool insert_default = check_for_null(); if (insert_default) column.insertDefault(); diff --git a/src/DataTypes/Serializations/SerializationTuple.cpp b/src/DataTypes/Serializations/SerializationTuple.cpp index ce15e099222..ef11b3e4660 100644 --- a/src/DataTypes/Serializations/SerializationTuple.cpp +++ b/src/DataTypes/Serializations/SerializationTuple.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -231,7 +232,10 @@ void SerializationTuple::deserializeTextJSON(IColumn & column, ReadBuffer & istr seen_elements[element_pos] = 1; auto & element_column = extractElementColumn(column, element_pos); - elems[element_pos]->deserializeTextJSON(element_column, istr, settings); + if (settings.null_as_default) + SerializationNullable::deserializeTextJSONImpl(element_column, istr, settings, elems[element_pos]); + else + elems[element_pos]->deserializeTextJSON(element_column, istr, settings); skipWhitespaceIfAny(istr); ++processed; diff --git a/tests/queries/0_stateless/02540_input_format_json_ignore_unknown_keys_in_named_tuple.reference b/tests/queries/0_stateless/02540_input_format_json_ignore_unknown_keys_in_named_tuple.reference index a1b4e2b5a83..b7edddf46e0 100644 --- a/tests/queries/0_stateless/02540_input_format_json_ignore_unknown_keys_in_named_tuple.reference +++ b/tests/queries/0_stateless/02540_input_format_json_ignore_unknown_keys_in_named_tuple.reference @@ -3,11 +3,11 @@ INCORRECT_DATA NOT_FOUND_COLUMN_IN_BLOCK (1) { - "row_1": {"type":"CreateEvent","actor":{"login":"foobar"},"repo":{"name":"ClickHouse\/ClickHouse"},"created_at":"2023-01-26 10:48:02","payload":{"updated_at":"1970-01-01 00:00:00","action":"","comment":{"id":"0","path":"","position":0,"line":0,"user":{"login":""},"diff_hunk":"","original_position":0,"commit_id":"","original_commit_id":""},"review":{"body":"","author_association":"","state":""},"ref":"backport","ref_type":"branch","issue":{"number":0,"title":"","labels":[],"state":"","locked":0,"assignee":{"login":""},"assignees":[],"comment":"","closed_at":"1970-01-01 00:00:00"},"pull_request":{"merged_at":null,"merge_commit_sha":"","requested_reviewers":[],"requested_teams":[],"head":{"ref":"","sha":""},"base":{"ref":"","sha":""},"merged":0,"mergeable":0,"rebaseable":0,"mergeable_state":"","merged_by":null,"review_comments":0,"maintainer_can_modify":0,"commits":0,"additions":0,"deletions":0,"changed_files":0},"size":0,"distinct_size":0,"member":{"login":""},"release":{"tag_name":"","name":""}}} + "row_1": {"type":"CreateEvent","actor":{"login":"foobar"},"repo":{"name":"ClickHouse\/ClickHouse"},"created_at":"2023-01-26 10:48:02","payload":{"updated_at":"1970-01-01 00:00:00","action":"","comment":{"id":"0","path":"","position":0,"line":0,"user":{"login":""},"diff_hunk":"","original_position":0,"commit_id":"","original_commit_id":""},"review":{"body":"","author_association":"","state":""},"ref":"backport","ref_type":"branch","issue":{"number":0,"title":"","labels":[],"state":"","locked":0,"assignee":{"login":""},"assignees":[],"comment":"","closed_at":"1970-01-01 00:00:00"},"pull_request":{"merged_at":"1970-01-01 00:00:00","merge_commit_sha":"","requested_reviewers":[],"requested_teams":[],"head":{"ref":"","sha":""},"base":{"ref":"","sha":""},"merged":0,"mergeable":0,"rebaseable":0,"mergeable_state":"","merged_by":{"login":""},"review_comments":0,"maintainer_can_modify":0,"commits":0,"additions":0,"deletions":0,"changed_files":0},"size":0,"distinct_size":0,"member":{"login":""},"release":{"tag_name":"","name":""}}} } { - "row_1": {"labels":[],"merged_by":""}, + "row_1": {"labels":[],"merged_by":""}, "row_2": {"labels":[],"merged_by":"foobar"}, - "row_3": {"labels":[],"merged_by":""}, - "row_4": {"labels":["backport"],"merged_by":""} + "row_3": {"labels":[],"merged_by":""}, + "row_4": {"labels":["backport"],"merged_by":""} } diff --git a/tests/queries/0_stateless/02540_input_format_json_ignore_unknown_keys_in_named_tuple.sh b/tests/queries/0_stateless/02540_input_format_json_ignore_unknown_keys_in_named_tuple.sh index f37a36fa192..eccac543215 100755 --- a/tests/queries/0_stateless/02540_input_format_json_ignore_unknown_keys_in_named_tuple.sh +++ b/tests/queries/0_stateless/02540_input_format_json_ignore_unknown_keys_in_named_tuple.sh @@ -60,7 +60,7 @@ gharchive_structure=( closed_at DateTime('UTC') ), pull_request Tuple( - merged_at Nullable(DateTime('UTC')), + merged_at DateTime('UTC'), merge_commit_sha String, requested_reviewers Nested( login String @@ -80,16 +80,9 @@ gharchive_structure=( mergeable UInt8, rebaseable UInt8, mergeable_state String, - merged_by Nullable(String), - /* NOTE: correct type is Tuple, however Tuple cannot be Nullable, - * so you still have to use Nullable(String) and rely on - * input_format_json_read_objects_as_strings, but see also - * https://github.com/ClickHouse/ClickHouse/issues/36464 - */ - /* merged_by Tuple( - * login String - * ), - */ + merged_by Tuple( + login String + ), review_comments UInt32, maintainer_can_modify UInt8, commits UInt32, @@ -122,12 +115,10 @@ EOL # NOTE: due to [1] we cannot use dot.dot notation, only tupleElement() # # [1]: https://github.com/ClickHouse/ClickHouse/issues/24607 -$CLICKHOUSE_LOCAL "${gharchive_settings[@]}" --structure="${gharchive_structure[*]}" -q " - WITH - tupleElement(tupleElement(payload, 'pull_request'), 'merged_by') AS merged_by_ +$CLICKHOUSE_LOCAL --allow_experimental_analyzer=1 "${gharchive_settings[@]}" --structure="${gharchive_structure[*]}" -q " SELECT - tupleElement(tupleElement(tupleElement(payload, 'issue'), 'labels'), 'name') AS labels, - if(merged_by_ IS NULL, '', JSONExtractString(merged_by_, 'login')) AS merged_by + payload.issue.labels.name AS labels, + payload.pull_request.merged_by.login AS merged_by FROM table " < Date: Wed, 22 Feb 2023 15:51:13 +0100 Subject: [PATCH 549/717] Analyzer AutoFinalOnQueryPass fix --- src/Analyzer/Passes/AutoFinalOnQueryPass.cpp | 87 ++++++++++++-------- src/Analyzer/Passes/AutoFinalOnQueryPass.h | 16 +++- src/Analyzer/TableExpressionModifiers.h | 6 ++ src/Analyzer/TableFunctionNode.h | 6 ++ src/Analyzer/TableNode.h | 6 ++ 5 files changed, 82 insertions(+), 39 deletions(-) diff --git a/src/Analyzer/Passes/AutoFinalOnQueryPass.cpp b/src/Analyzer/Passes/AutoFinalOnQueryPass.cpp index 10efebe0731..fdf818681d7 100644 --- a/src/Analyzer/Passes/AutoFinalOnQueryPass.cpp +++ b/src/Analyzer/Passes/AutoFinalOnQueryPass.cpp @@ -1,8 +1,11 @@ #include "AutoFinalOnQueryPass.h" -#include -#include #include + +#include +#include +#include +#include #include namespace DB @@ -10,52 +13,64 @@ namespace DB namespace { - class AutoFinalOnQueryPassVisitor : public InDepthQueryTreeVisitorWithContext + +class AutoFinalOnQueryPassVisitor : public InDepthQueryTreeVisitorWithContext +{ +public: + using Base = InDepthQueryTreeVisitorWithContext; + using Base::Base; + + void visitImpl(QueryTreeNodePtr & node) { - public: - using Base = InDepthQueryTreeVisitorWithContext; - using Base::Base; + const auto & context = getContext(); + if (!context->getSettingsRef().final) + return; - void visitImpl(QueryTreeNodePtr & node) + const auto * query_node = node->as(); + if (!query_node) + return; + + auto table_expressions = extractTableExpressions(query_node->getJoinTree()); + for (auto & table_expression : table_expressions) + applyFinalIfNeeded(table_expression); + } +private: + static void applyFinalIfNeeded(QueryTreeNodePtr & node) + { + auto * table_node = node->as(); + auto * table_function_node = node->as(); + if (!table_node && !table_function_node) + return; + + const auto & storage = table_node ? table_node->getStorage() : table_function_node->getStorage(); + bool is_final_supported = storage && storage->supportsFinal() && !storage->isRemote(); + if (!is_final_supported) + return; + + TableExpressionModifiers table_expression_modifiers_with_final(true /*has_final*/, {}, {}); + + if (table_node) { - if (auto * table_node = node->as()) - { - if (autoFinalOnQuery(*table_node, table_node->getStorage(), getContext())) - { - auto modifier = TableExpressionModifiers(true, std::nullopt, std::nullopt); - table_node->setTableExpressionModifiers(modifier); - } - } + if (table_node->hasTableExpressionModifiers()) + table_node->getTableExpressionModifiers()->setHasFinal(true); + else + table_node->setTableExpressionModifiers(table_expression_modifiers_with_final); } - - private: - static bool autoFinalOnQuery(TableNode & table_node, StoragePtr storage, ContextPtr context) + else if (table_function_node) { - bool is_auto_final_setting_on = context->getSettingsRef().final; - bool is_final_supported = storage && storage->supportsFinal() && !storage->isRemote(); - bool is_query_already_final = table_node.hasTableExpressionModifiers() ? table_node.getTableExpressionModifiers().has_value() : false; - - return is_auto_final_setting_on && !is_query_already_final && is_final_supported; + if (table_function_node->hasTableExpressionModifiers()) + table_function_node->getTableExpressionModifiers()->setHasFinal(true); + else + table_function_node->setTableExpressionModifiers(table_expression_modifiers_with_final); } + } +}; - }; - -} - -String AutoFinalOnQueryPass::getName() -{ - return "AutoFinalOnQueryPass"; -} - -String AutoFinalOnQueryPass::getDescription() -{ - return "Automatically applies final modifier to queries if it is supported and if user level final setting is set."; } void AutoFinalOnQueryPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context) { auto visitor = AutoFinalOnQueryPassVisitor(std::move(context)); - visitor.visit(query_tree_node); } diff --git a/src/Analyzer/Passes/AutoFinalOnQueryPass.h b/src/Analyzer/Passes/AutoFinalOnQueryPass.h index eacbe0f8235..3489597108c 100644 --- a/src/Analyzer/Passes/AutoFinalOnQueryPass.h +++ b/src/Analyzer/Passes/AutoFinalOnQueryPass.h @@ -7,13 +7,23 @@ namespace DB { - +/** Automatically applies final modifier to table expressions in queries if it is supported and if user level final setting is set. + * + * Example: SELECT id, value FROM test_table; + * Result: SELECT id, value FROM test_table FINAL; + */ class AutoFinalOnQueryPass final : public IQueryTreePass { public: - String getName() override; + String getName() override + { + return "AutoFinalOnQueryPass"; + } - String getDescription() override; + String getDescription() override + { + return "Automatically applies final modifier to table expressions in queries if it is supported and if user level final setting is set"; + } void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override; }; diff --git a/src/Analyzer/TableExpressionModifiers.h b/src/Analyzer/TableExpressionModifiers.h index f61c2a61610..9b76c9bc0fd 100644 --- a/src/Analyzer/TableExpressionModifiers.h +++ b/src/Analyzer/TableExpressionModifiers.h @@ -28,6 +28,12 @@ public: return has_final; } + /// Set has final value + void setHasFinal(bool value) + { + has_final = value; + } + /// Returns true if sample size ratio is specified, false otherwise bool hasSampleSizeRatio() const { diff --git a/src/Analyzer/TableFunctionNode.h b/src/Analyzer/TableFunctionNode.h index 292ab740c5b..a88630ffd00 100644 --- a/src/Analyzer/TableFunctionNode.h +++ b/src/Analyzer/TableFunctionNode.h @@ -116,6 +116,12 @@ public: return table_expression_modifiers; } + /// Get table expression modifiers + std::optional & getTableExpressionModifiers() + { + return table_expression_modifiers; + } + /// Set table expression modifiers void setTableExpressionModifiers(TableExpressionModifiers table_expression_modifiers_value) { diff --git a/src/Analyzer/TableNode.h b/src/Analyzer/TableNode.h index 4965de535df..6d47f87c78b 100644 --- a/src/Analyzer/TableNode.h +++ b/src/Analyzer/TableNode.h @@ -68,6 +68,12 @@ public: return table_expression_modifiers; } + /// Get table expression modifiers + std::optional & getTableExpressionModifiers() + { + return table_expression_modifiers; + } + /// Set table expression modifiers void setTableExpressionModifiers(TableExpressionModifiers table_expression_modifiers_value) { From cdbff57e6c91930b7b4eb9535f6b4d17e17641be Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky <43110995+evillique@users.noreply.github.com> Date: Wed, 22 Feb 2023 15:58:06 +0100 Subject: [PATCH 550/717] Ask for password interactively --- programs/client/Client.cpp | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 3be96a4b0a0..af66a4ac61d 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -327,7 +327,29 @@ try showClientVersion(); } - connect(); + try + { + connect(); + } + catch (const Exception & e) + { + if (e.code() == DB::ErrorCodes::AUTHENTICATION_FAILED) + { + if (!config().getString("password", "").empty()) + throw; + + if (!is_interactive) + throw; + + String prompt = fmt::format("Password for user ({}): ", config().getString("user", "")); + String password; + if (auto * result = readpassphrase(prompt, buf, sizeof(buf), 0)) + password = result; + + config().setString("password", password); + connect(); + } + } /// Show warnings at the beginning of connection. if (is_interactive && !config().has("no-warnings")) From 986dd728705230e8debd5d88289193e50ff284b2 Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 22 Feb 2023 15:18:13 +0000 Subject: [PATCH 551/717] Fix possible clickhouse-local abort on JSONEachRow schema inference --- src/Processors/Formats/ISchemaReader.cpp | 4 +++- ...local_desc_abort_on_twitter_json.reference | 1 + .../02669_local_desc_abort_on_twitter_json.sh | 8 ++++++++ .../0_stateless/data_json/twitter.jsonl | Bin 0 -> 9940434 bytes 4 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/02669_local_desc_abort_on_twitter_json.reference create mode 100755 tests/queries/0_stateless/02669_local_desc_abort_on_twitter_json.sh create mode 100644 tests/queries/0_stateless/data_json/twitter.jsonl diff --git a/src/Processors/Formats/ISchemaReader.cpp b/src/Processors/Formats/ISchemaReader.cpp index 48cb093f0ab..48dcdc657e6 100644 --- a/src/Processors/Formats/ISchemaReader.cpp +++ b/src/Processors/Formats/ISchemaReader.cpp @@ -223,8 +223,10 @@ NamesAndTypesList IRowWithNamesSchemaReader::readSchema() break; std::unordered_set names_set; /// We should check for duplicate column names in current row - for (auto & [name, new_type] : new_names_and_types) + for (auto & new_name_and_type : new_names_and_types) { + auto & name = new_name_and_type.name; + auto & new_type = new_name_and_type.type; if (names_set.contains(name)) throw Exception(ErrorCodes::INCORRECT_DATA, "Duplicate column name found while schema inference: \"{}\"", name); names_set.insert(name); diff --git a/tests/queries/0_stateless/02669_local_desc_abort_on_twitter_json.reference b/tests/queries/0_stateless/02669_local_desc_abort_on_twitter_json.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/02669_local_desc_abort_on_twitter_json.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/02669_local_desc_abort_on_twitter_json.sh b/tests/queries/0_stateless/02669_local_desc_abort_on_twitter_json.sh new file mode 100755 index 00000000000..e4f738f18ff --- /dev/null +++ b/tests/queries/0_stateless/02669_local_desc_abort_on_twitter_json.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +$CLICKHOUSE_LOCAL -q "desc file('$CUR_DIR/data_json/twitter.jsonl')" 2>&1 | grep -c "ONLY_NULLS_WHILE_READING_SCHEMA" + diff --git a/tests/queries/0_stateless/data_json/twitter.jsonl b/tests/queries/0_stateless/data_json/twitter.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..4c5eaabcb1b1478b4b126b2168b2353668b1fe53 GIT binary patch literal 9940434 zcmeFa33uB@wyymP?teh5SME9IUd6&Z=&W9C*_LA`p5h@>t3NG}1SQxcp>VJ)=j8tO z@7YzDL25GPc%VBiQK&)zEEHg`Z*aG-!DdnX&I)!Ye&1L zY5ZT4e}-+^=GOLBI@sE^>>#u)*EB6B@@&g@LRT6NXY(mTENSJudGb!iKUkP%KQvxE zy!-0;Q~CE#+Gg2e+E4OUrym=~(`5LMgRSHF{A9L!uye4JjFL%~ax|Gv7w_Mvy=;D~ zpN$W;jA=5W=Oi0uqfvHxpf2Dfce43@eu`Yt=^yOG>O&&8GF?pi@|CxChw*5ZaPO1D zX>u|;JDg{HiFm%49m@BdEJmXhjrzNGJ1l0&bj7)O)2cIPJcnd*IEly1gdSMV*xLO= z&$$&@t`)kj>)3AO`>yHK=e*m>7u92J6*FPn&u7nIdSu$`SKO*i8EvDC8A`KBc`MUV z@o|uFVtS%mw;mlVY}1O;@kutF$CG(t$UG;Z5x-5QqtrNAjM90kW<8$FM)5qG9?s8B zFltdEhelZ%XXset>H9HCWG=I6EaF?vXqNn zbcx@xxYBu2_Ofigxq8OcG20A$#%sI@kF$yKBztR^u3?3{mb>fN<(ptd-1lPMKg@>1 zS;GD3=QU5q$>DqXR{4RWBs)wdac@-nE~9vIRJ8ZA$$Xmj7EH>l(=nYL&Zotkl}nY` z8m6PUcCF>{qX*tp3!!nbQ~WghYNX%TNm^=RIlC2esiwgxtmcs zc~jnpb#KqQZ`ZCeO9x3Wo*wqH>44XxxZk_xz5V@rOZsv1qq5gMI^Fkcy<{gTTOIYU z*2+6mpBwePw|05udCiNtt51e}S+&KS^k%o_r|I}eZ7e&DzO#cJOWrWs4qe9#T{b;8 zun%?~?4NlLy`%7Ol1<0)=+^6#quQ;Mmnmno+R)a2wO%}#FuD2>Uw9$iyZBzseT6UU zvBBG+6UaIrB*SCZ5F% zFLWc<^u1u~&;RvDb5FaXbayQ)G%Yi->XN3E{bj4%Mj_q(K@c1){J!HJEF9BM4i;YM zNBp!+UmY1b-^g!|UXJPZ)G-mA> z*vPL6Yn2C?yjz$AzA(;MmucMcXoG_VgAkw36_Mm=9WBUAS7 zCOB|=HE|!M?Wz6&Vk?8WAM{A`<}dw4o{la35isQO4&axak^jkRY{xmiAPNs%sqtxu$)~;K_aSq?- zlf!Y=o^{8%&z%(*q=xlMOu=SfLYKG+T~r@L`}{aTIOM;hgPq>dt)p~!!MIDX${L2h z7&O}V@7nwIFTDaZS|QW3U;7e8zXoWuJ=?VfycAQ|0FA!-S8Kf~(CBz(WTL9E?d~{k zFLC2Ij!IISOAq%~p=T;apSrdp{!_}_~Doe2hmUMt34#oqI32%YM5|y=% zZ@}co29vKvuCE4@zO@E0=|xfGhG8AP)WBrkDz{O9NdzV$C<+rTPL-hUA;f?LNbvbaEOtk_y$WAAtEFK)}q$kJd^eeIEOM#@x)EH({ zBYmd2ei?`ptbn2$L|8^jz*!#SIhAb~i_^(`=E{CiAkP>Bw4jY#OuVObwBJ?lLkRg|tG!t7e79G%^dg2=^2Sn)M=E-+@|+tQiogTs86& zMsO3GBl2zO>6&~-Iw`~^`cdEsBJ%@ZEnq>8UXYz%jf zfDoIH!NJaWcKl?q_x4eRx9X3i#Meun!v!|Gt6%Jz=&Ia<#ZVX7mg5Ab7)Vv*tzfi z{$QBe{;MB3VS}35KEOI&(I}$ns589~l`aT8I|?n&bXkVgL$Nxo%yEvS#_1F+hPu#_@)m?_wGLg*x8kjR{RVvTq@f_dwJlA(zyBVa@X|3eYi)=AJKEB{pBk15{+j#Nwi>Ke+dD0P7IjG?HWu?^CciF&*Cj;ZIxLl`a+s4b| z#CX0KB}O(h9>u5Iv`l7rTo>X?<1bd)na>Q2F62o6t)Qb32 zTTv_QGas%CZ*Ps00U-dp|H^|Bm7upGq7rRCGDX>`>pI<59!xqJqa|Ku`_P{lw4jO>}XpmtGElE4tDTB%u&ph`&|c5sv}J2EPb!zkmkpW zaZdu0P6>|`E9`Opdzwfe{KfrgcCtm_kxas3oOccKu91UwR(2f{Ttv@NJQW0{#yq#< zjOaIXP}R|Mu|jZhUkuF65&!)2-;4^jBW#n2sV_+NWHt+Hv#|2Tny7Nov4D}xlO6U$CttKI=a838Z1+#?lhUwD7^C3u4yRj?j1U4 z#&P+wlF$5A+X?ecDzxxHs1jGowR}SBQH;t0Y`mk_d8fe|!FJKA84!6X6Uy zipR#o`N9}yy>w)Z2s8^B<%PG!gO4s>|4G@NpNp^8ka%{D+-r{=@5L;4%2zDz;BwI7&duT z00?u}t2~X?b@Hlo_{7bn$A6V!y}yKkSph_WAU0<2v1 zYS#oS)&8SE3SW*aHi^mJfRweQ;J1zCT@jIQ`qB_j(@Y|eDtXU?(Dw0;Ad-d)#8l<< zwEVVx9LK_CEGOrZ0f>OPS?h=$;sSJ>x@KOpjufqO8#;eWLX4lJ{h#n5#xpb6HGYka zWM(9wC4LRCnxsa5F+Le3WVoQjTiNUuK|4k^NHgPYI$g|YCMYMn#h9N1SkiL5j$lfA zon9Q>4kZ{`*|h4+1`yF_H}DOVO}_=|73so<|bgs zRj+nk7}DJxG#+sPU3S0@@Ok2qD}_Eoen*yrR0n2Or?rxK zKg7fm6H_egb%nmKNZy*Vt_Ei~_p(^{y#G}*rY-aQp5{aD7jb5UX#wCP#mOx-N zFa*6Bv7&UlWf&0EKviQpvB;J6dKjE8<%;DoRhO_1c&5E1?blMLH$+_?uO%zRulWS-zrIj*IbeH6XQm~52 zCVbDk0rSK~-W+c&-kiY17+C+Z3XBO)v5PlHwN_wE%oBH#Q8qa`e4LyfEk?uLn!4-4 zE>ftzZm6X3-x0@&4KD~kVf!!>VFxi2i=JlU!Y*PaCY&F%!skINoF_$3VgBGV3?Cd< zpRHjh>)IYp-N(ZhzkJu{Ca-Jn_M{UXy5=CAJNmZsKffAT~+6R(!_+RC&;T zZ;Tt7UHNNGyOtNmC9@ajtnYfngOEA)e43obk5Xs-i0$%M`#Nk7LEW9&wmk&4?cz4K z2-X34)LakB_E2xF#9d9C{l%hfX;6f@ZABjt6Y&*`J-F*V)F~|dRzSM986X*IOQO?S zi9L;ejGxo_u>l174hTeJ)8f%YNUP*vtx`iPc5;05?lhjfj?;gpN3-K}@DI!LqrQ{8 zBi<|!_Wx!)r2r&L+(C%|uqMRHp;rw=3F$9AAz&-Om8*_uxaIm9&vC+_@5oR1?Buft zZYw_bL1V>dPr^L%KEhR3d?v7=`lDAnxd(k)(W$TBDs_S0pmu`XUeT#><=WMR=eX!2 zq=x0^+THbfHFz4?# zDgBsdqFa`I%ZiNu$KZBggI~F}ID8V}{pGU?eHns!0TLkEVPeW6Sueiv;4WGMaoPyS z7rRv_=G7^Pp zXYlCTea4mlj=$YZIi+<5G-fm{VbSkE@i91yws@Zp#E-Dba04ZETS!n zk`6XI$O@~}ABFQq37NHN|Lmca(Zv(+BRP}gAzZ3@#Il4+yiTx$NR1#1ABco$Rv{@l zF05J`mxq|dy?P@4(fIw|PyM}p@7KYb4@tydGES9_p*>qV5x-#{DL~|keCbBt%O&DB zkxnxoU(IcWx!0VJ-^j8LK=dTfyW(eQCeI_}YFYLwyPl4`)7_4e*;-}zuO9NxtK=+h z7FdnxVY9%lu)vD#_RGr4{}__N_O_jiL_^zU(A>z_>2oXRa&=EfG&He;>!8k22WOosm-J;_(U+BNO03aoUaBV)E%@?GUT`4gBvCy0vFVu+`e!qwU@)T1ePMgqpkF;5cr-gdDdNkcu z9!+jJc$Q3uRA?Wa8Fz6bzL2mXstva#Z4C+0c4=t~595GzzrpNa^6kOoAvu3AjS~nz zH=a=Scsf6(wD{kQ(|F!Lt^=%zZoEY+=Gfu}`-{1Ve|>vv71Lr=_eI#YM0H0{TLk_t zlht%=rV%%c_OVZ!*ljx(qXF9J@h8!6z!okP43UJlKis z$H#ZxzuRBG=)C&Xt|>YRD{JpXB1EZME`IwjJmWpE*mxUGN>3`|eZ4@6S8TG`_(4`xo~9S;Uj>Y|X1sCI>G6-x7U{j$heP|p-j7#zW3K0d)t9RKJC_0=-L*qJ z-spQde`ifxH*Lu+gm$o_W|Mj3S8qe#4&DsChrRx3cj>R_uc(ql2BhNBtkaH;?;2q>q0JhVSm*8as>dpDZSSOA9lZpq|DAMp(?~?`a__BQV=K`d*BfTae zS=Sfn!pe`{g|&6T(QRee3PUYRy<6r-zrJV^=0VwoHZbj4xUjvm>9Dj0^eXIB8<^G& zO1msfW4lzIvbM{=g-g5LN*BG-Rmd+WZIHZY9hZOiZESZc#_RUHlFNVdzP>&Yzg(%n zU0iOXfn2Vw$9;D1tHM#tl=@xj&KoQ_pPy)zLru(8A?^lC{)Jew+*8Xq0;-Z}gG_yc z?sh9S-)`TV+01Vk?_xSx9lG-2T-HwHQH&S6T^(;SbvBU%k`?*Hs<23u~oMu9sESnw)^1@vJMfAbWWA`MP{dlssey7>h zuXcT>S$hi+r&$=l&JPm+ukII0jZlv~fte)Obt! zjLMBmNLktfxX@M!zWmzL<+Q_F&GEUG5C?EASqW<`^=>*#(%B_;i7+ZKC~ zXEA*Zv8N~&iF>t8B;PJ9i(XCaDNFpkf0zx2vt(WX7Qvv0@A0k{2aa%)CKGYAR(9S^ z1%~4UhVp|QxH-qkCZ$saeg6C4K{D`fqOSav zuXartTWw1sS_A5un?WcL^79m}mr9vuw`oM9sJ|O2^Gh=3(Oy<0%%AIGp|MsM_|Gc7 zrP_YVK-5dsW&TWP9GepBVtFUtq-rg$lAGuIj@C1~$i|h13olT#jTjIT@eboZQ%QPd zjU{~EQDD2on-U-$MsAausN2eWR%Vl+D8ptDnx1bHJ{}M~j{Ix@k*y_Ngwu2R{UVdY z5O?;U|Lc#Y7!}?-3PnOwEZU&d?S}Qbmk$nZybt|HZ zDCq?eWOdEJbJW3BE3=%xhI#rB^*;MctBC6Mccb4qrNWw9v&C5A)TaM2Gp6ZqPUA`w z%VKPRF9+F}VzKdvg0WKwzITVC#q2m4>~iK>;V0@~$L2zzwKn<`!%Qck2-%O}BrQ01TRJlWrS{rCo~ zE?2(VH4z~xcI~Z0pkZYDc7R&T%llPBL)mKBMd}RS9Qq`n;fd_Y%fUc|(&Aem%sGu; z#d1k&TaE?FR=Ew01%z|@>0RW?$~F0zdcXB zGzp1Nm;9UY5C_{^ssAxIPP6G7L)aThWEzol6;IA23+l+24{eT(2*n)SGF~rc^Qwmt z_xssmGG`p-)|h8BDILxr`)xQhCRsiNb?TGEpxB9g&9&kVAyI3&dZ@D8h8_8av5*~O zR%A7>$%ac{hezSE7U~cVN0lH&+45-Dad$~a{J%VDNlM&sIDUm3j#B*-u)#|6Jfw02 z>agBF_8&Tv>;_QBm9KVP)Y09jltGI89CYc=w{Cf9++lkcJ|8g^G_;DkHtz1)?k>d_ ziub#rjawlM!mbP7p+nMGJIlD$DqEeF!0jyKhOjN7H>K~Dvy3<8w}AXq-2xj^)LT-F zo8;u()=Kn(iHxU1BS%HXK6$@dL&}y^xEdz@`;+76aSvD|++0+=ti=$(-_2)QRdS}M%M+s-n9^)8znVnMa9VvbL zD4w1jp4b*;uqX#V*fs7R$IxIS#s?5H_r~eG4)R!JeOhiEk$^UNK74ijtGi0==IT+S)78mFCGnNikYcPCq z=wuZ4OTV%th)vVpVxCRQ4wrI9vBfl;QR4M(Wk$n8+ci2w5&FZ2n;n+r+X$9`6%0k+ zazc;GRO%!VmaSIavdSxSQC9S|^PB~r%ul;%&aIolZ@b7-ozq_*|2X{h{$Tz-`8ga$ z(bZF(*LKZnhplHjF9l+{YbV>e(f4xM&Mn-~%y@30#Y<&8*RxqjQe2(Ktg_Gr=bjSz z?CFl`cggPDpJpfGxDsnXnelw?nDv0}b2?rr+a8|*;%-MY*q=wty^46!u3^dk=;b+T%Vc0 zkId&s{X0LWhtHqfdDghaMo#nRvlAAcit~a^PO{l7?TyX^4^dDjeG8_^@f9$54qP6= zJcYp*#q*RmMMukREhYom!Y_q!kO2*hqyBCL+;F70*TO`~q3gh#c209k4)O(Z>-Tvo zxGpQ&fu(g3u~fUSkPx}en#Q~q{j*-IMYSK-Wm{UA+q?l=2RpG(Mv%c048kWj-db4(QZ52|j)|c0_legS`)7qVi z@wz>)l(*bMJoUG|D}TAu>TTb=i%XTQ4I;$YzbBao@v_6@_5;z75R~S=x&d3r^I9Y zZ(}l#Cld-&nk4b5pH$Z*k4Orrw|?qQQaChkaHSHvZ<9g9$+0SKS(LA=m$F>_xhR8a z4!E@fA~hU?d?jl06w3&_z_lO>j*a7KGW^GZs)?!$*-&!LQ5-ZW*8sQCD2vesKW2!I zO1TdMh(v`Hg@`s!(vR!mwathf1x84itDm4md=-#ehVbC_vRr6|-AHO~l6)tHH|t~P zJxf^9>E0T-8GO@P4)m4r${2IoCg6}1Pa;&gFxl13V609n=jR%aS|EBwQS7x?IJQ)f zrSO1SQTh*;x~Ck{D@rw!V64lfGFtO)_h-{M-XGzA3&?YpFxZ-Za(`sJ!2kFrjkk^c zqcattra*y?BWmVlmz7xiQU7^z9FPC^9^bou)VTCNb^q$l^>=|rR=$z`s_L!LZ;Bo1 z7fF3#{hqn1S=H6ePjEfmNl%?VTaT?~s5nh#$oJ_aC1cCmZk3f~e%9;xG{vLYbbDH6 z$M2cz*Uca#7~FZ9>OH6;lfrhgxn$klPkK^F!7_KfaM$!ppQj2?SbS<(clW*f;r%^k zy6~X2-Eq0k^MW3w_i}l^i`|f4W^n`CCo?mQLeuf0(8r`m!l3{A&&R>t5Fnp#0MaLKpzDgeQd|vtJ*shoXVlK?QhdA+T2fbBaE|(^j!T~$ z!VKhi@mXb4YkOWt4A&7_SElpYR~o*#ZTan*f0#-x169wnqQ;W^v>J~3A5N6R3mOEr zZ?@gIIUFro_XD{-#s?Af|BETRZm>F0RMQLx$(BSngPbRsj!ihwkUDv(0kjIp+ROWX zAw@Z0^dJ)@vT)A`9oNEJwSB8Ky7>)4?J0clqdK_mXICG8gr8V zQ^R6E>bXN%QoI<(!4B;NsOu#py*#g99SSX3@B00+`=N}*a+hEE8cXCd@>X$?Ap{YP zmuBEtH1tAGEySVxBu;~$ex%+h^~&TU@lG*pFB-5%>hpLfc$a!yR$qdzxn4jj`3G%4 zYV(krR1nMK@T3Y@2awXWp^r-t-o2zxIJ{vwyIw>JYY9j+S*lr-;je^3}|QLUN$05@Lv*F@Szy;jSv)`gCYI3r2T~f z(SUWpTqj#3d4pED4UHis?eFxMgs1s2nUe8jmq7D)PU3hvzhzYWkoL`<9qlEj?r?u9 zVd2+BR3Fo}s8L1T;8$AIwaKsBp|l;YY+7|@1MK8@rfv=-2@C~Q43`+4P0thCPZh7( zZDo#gJoPf0&KHFK6Xky&K$hKixK-F9;wxk8x4o^;%H| zc%p?dBu=#nF_zlA9N%AJ^uj*e0vSbG4&}&5_7|&{TY-#Cvj10@?C;~!AJK+t)-V@`0#j9Oc>~}W^CH5Uy;wZ<(X3>hVFDh2G4u5K)_ZOz!-Jhn4 z%)?fhJPs6#Ra$L5?as&e6o7!N$n|mdG?i5}*=B1c0WT}>mO9!nHc8cr>&n8f*SY}t zlJB}v1WyI_1=DF*y}FH+EF|P9R{7{NGS+*j%r?q2%?;_hRfjY*)@0$}jIc-}`;VS@ zV#q+}55_QUn`+wG8~L05D?MM?q@Ngz6C5qEAwbSZF^;!YW8|ju%6DM@<^i?b zSB_MVY{_z|G5R>3BFN#qk{YAlr%v3{NlaVe!4fFF$=WLn3!i4z9(t0At0I7o4FXnH zvlt8>Md{IhS?TED1*vd$3HdXa4CP8@~}5T+TqUZ zID6CqzJA@9a zNJ41Ct8TXW0x3RZ4f^2u&cql#ri>eLt>6xn7a3lc(8RUiHV3-e$pCL*!jwzoBU9Wg1 zdNE!Ld=+r-66^uxGc^Y0X*TIbeYpLf#wW_bOb?Czah5^K ze3HPBhTm##Y#X!qO#Dv=lY80ZKUhEU=nXC}S$FumPN%rJZrwVVyp(nGGNCL8zORR@ zBua@G6cjNCLz&2e${68pHkunGWl5hW$>|9}DlE34@nnDhp3zIV%F`eIDQkb7HO_e1 zFmAMhPKX2~`>yG)l4L;HT$=7GvIJ3^JKidIyE_S2dM}t5L zsB}aucgfWFg(h4PEFXrUPoP(1x{{1^u=6q+jDG98H&S`*s#m+F1F5@5i37>?q0=Rt zjb+|)6n!a?Cv{N|o4XV;EF;V6ktYx>dcuB9Ac9Y}Z_8TRZk6p$$6kIavi-FG!uv@c zV>*K=^XyP=uf{Qr75D=}T)x_HPvWxJ4gxpyq@agK+DHQec3OD=x!lJ+_J*pL#>p`z zS}@;?rl#C40U|AzmiY?l!Tzj~;#{RHNyI-GkAiU#gxu zJ>@)Gr<8^py(^^c%cVnG7L}8`GCBwZS~bwR1eENZ5Ks~srn4*mwSvbs68bBX&=MaQ zMijfJ(!Ciu&cV*3hu#nF=<%JK2=Tb;)vhU_t?p){Vm#z9N{8(2$uM4w=7)KT&u1c{ zFBRe;XulERAtB-|VrB;;SDC(1>CR2P(q)aRzEOVY0YW_ilL0)o2r%PQxEST zsMK)~xy_`;?;hS)2>|TU^e=5~=LCP~v>45os;GV?5XAiMFv%nkw^w(yUV6;s zMm%LpXyc6)K_QL4m-E_}aW8jR*-@QsQ8j-{<*URfkm_8xbL^N4!cz$Yw=; zW-BtDK$1^6t;3^q$P2Gu<8(5F=>Ctbcy^NX1*BuLBRNge9>1mbd_0*;w%Q+Cy%eL0 zl)hy%Y%$xZp$>M&C#n+rR0w0MOfS_~CuFO=@!kDH#`v>AxqD4c@pE5?daS~^9k2A> z+^k*sPxkcL;qQxVp5z|XGOkh|tS2yjq=d#c<8HZKvcAsIJ#h5a-R9#XKj9HNo8_v$=P$8%w0iYzAswkHmZEeSv=3qvWfHKkMDMky%!Jno*2)b zK6_z2-urIvw}(&m4;BtlRg`BuSa{Ab5E2cCAS-nk&C4+d>T?3Q1;^n7a7>}_aLgfO zV2;^?l7P>ur!Z)Qaisc$y}jpqFZZO;Q(Yp}T?T;oj3Fbd&u7Y2L%PBp4D^Lfj)hjj zF^?)9(&D1{VXuaKz%gNi<7c8q>g%6SCvXLIi@xe9Y#YT{z8YH}@gXkYI`pq!f63ih zLcdTRhoAw{B7G0)OV-ECzmu8{8Q<$gFnIZv#ZRs^Rv-V8@8T}%YR>A&4$Ny14TALeOYz4?eAO6x2x{3_DmYzTJA3D2{n$JgzDb)U5AWG zcFKo%UcO!njBPp&CDAF60_}z%@y3?bX{~laSxZ91p?`WjU)q``nE1!1!|9LCZ}+m7 zq5o1gqDvcMSg)QCY3l~jmI}Z;&J7xUE5&O*?q)d6GTBI_3?OXKTm_rh1t9> z7m3Q|C5hLB4~T^QmO(a;Ko3GnHBwZJZQJ+ptTc9RxW1aLvdd}wwSik3xTPUz3vLOB z>JXZ|96ijE?;bw-NxYb!N@UA?x=2{=%NUT0f_6(Ab=cub+LC4GVoRJ7*t_b?2IA>5 znnct!#M2GGF&^)TeBL_iOt+CK&Xr7oJAryWT2b(Xkfw35t(p!+TLrDQfK8DHG=tVj z;QwF%pvUneY;(%Mqi?fmI)&(WD71*fp=e6Op|3s*%;q>ijGv66TY*k${p3KMKA|ln ziymK3?pwEt`ZF8~)YnkU_WA&L&k^=wFA?%Z$gK4@)Cz}!^sf3WDg$RJc&U|=%c=~) zFjQx0nI{F^CH|QmSimOx-J$^Y_sgj_Y@wre5 zMI{M`Lp7#cB^X3%#ymY6`kN|+E?2*RuKK!lh}r11lUox0W_~6w9{#pp{jDZN>AYSfkCy4+XZ`ZSp?r;EK}38#76Ff0AK@!; z)B20Ki4aiN&*gf2;iZ$P9)q3;Q%#vX=HgnS;xm40G|?{P4rM9Sml}L!YC`oZk}nrV z(EYn(N`x6YBrNLm3A9epwgZ%BsqdSAYjt&5g4~;$fqZ84GF3l48JX}kzO^NH;wJjr z>G55$Bd%4;upN|zmY(4Ve|9r~YpJH;Vj(1Jx2^x)~6z1iBG!&kr9HC2tSAQnx7ym$}B z!lmi|8i2GEkk2LiA2oo&;Nje5X7Ul+Zl)m4KV_I9sfmy9;n zSOsV*Yb=tQBUm8754qH?0W74jlKVM)RMtEXEWtc$c%(l23X@>V$Zge24fml&JsJdP zFahis4#Aa^zc-(v!XlHut6>mZs{;9aDUP~=&fH`z}iO;V*qquCn zlSg8zrCdIsa>bvCG!<=QEZ5H_`Jp<$CLXhMJ!6RfR#b;iNL}H@D}p*4;U!w}9r^b8 z-Ta}Ig__?}Gh8+l!pp@Rq$I~By+**n67xjyNVF5WQFf;@Kxy%>O(+$8EZ5W%dj2$4 zQ*vHart_%oU&o(Crgc+AJWsV$S=LupR;$^^ed*nR`;vJEH?OR4wT_YkmKBATW36ys z7Oip{mHYBVI?jf}J4rmBrtf!++fSeT#QU^>c5NR}-HQ!)(?+pz7n(9-Z@`a1GBTbM zC2A&vj6bCD*(jbH$b^wCAY%cwN2t2yhtrKQ?$3sCuSYw=DL zi><2Zc)~FKv_Cg&a(M!wk*#ypt3a7gt$5O1^B!qbJe-2I{cE6}`yvyEJUa}0%anqH zCS}#}iXQAFuLr~9WbvRl^vnDp|D3R(|7w1|R%R`+qgE8{*S8M5jpGnm4g-)n? zVh?$H_O4e^MmH0Cd@_10$99~Ex;*ubR8>T7w(@jxDf$GoDWy@X^qkJUzDJUUNU-76q0}x%Yju(vQZ1i%T{%e-nmrH@t2{|7{{CAH$WP# zmle=k&ywjZJwn(m=Gln`ZSjnxU$VkalIZ{rTHxCx>!;E-bS62Vn;`;y<&6z~Y_ZdeI3DKY$Q0q?rl=vx9_Qh~nb|0c#>YuAKFI;f1j9$)LK z#$IhEBa8m10B;g!J59-=&Ze`&WDVt8K5CU`PF{weq4K)*gfsn1#$MMkodEIf)mI zzDQil_NiL{&!^JHkn@gzurpaaJMw>YPbvYwCi#Ait6kHyTg)yaWS-28tEs;L$6`LktYR7S;G_%$W~h{J^6qFpYeNbo3HdA|zXE}6VfC*zF8ftm8;t&yb^qL>nu(o4p4 zyRM<~bBWRiUEmnUv>15UD}XhUv%gwdTGCx@(uX=@cb*+K*Jf*W>08&HELeC)j_+IL=HSZ&(5C>;)ywa=~ z%9t(0Xs`F!DrqjApn+i99_#9)hw9|?@So|?>^L3#!+CFxPli$d_~=A(cbma!$%0ZJ z5H9tyUc7cLBujE0TFR9raWGBYD&N|gL`XI+%dTn#V_C;^O3c2hJMS2_xoZ+eW0#th zGN)Wsv&xO^H>X-{>dvEC-JG@)gfMiV3_wdDc1@c&nrLz8?LR+!aRb`Um9KV9ZD+~$ zB9Sx^RU#})g^PVzYR@MFkGe7;-_unC8jt7+a~z8!Hdnz&ON35COmrleoLAQyI-J^| z$U+IYE*3qRbCq+vz0QYY8I#(QHmoL%FI!7i)vau zs_Ioyi_ytHaJ8k%BP1QshnSo{CSFp;{=oL&$@d5ggx*ae-7l~>Q-iR6f?8q~>QU;j zUaOSt1=L8LEf1@u(;hqTS=zEY-CH9!aZLV%=~Ks1w&i&kUR8z7(h6N2q<8*?31Tum z4DVKFF^@Lbxwz4Wpv!cAT(nOtvApq)P^IWl^8)C>yY^a4-SOaTIVTWU1L{ zJRQvR{mY#XB$A8!SDz!GW${^(3(C(#RTrNlzE}BKe|L4R1ooH5iNz~Fn~-W^-WhDe z0y;LnPiI*sh8)`Ej6v9AhAO-AVP4>7!raq~LvFRI$apB!F{e1G4p zS4eMLiNzCJHwa&*ZhLgF(>uDQ**yx2@v?E3czD+^{2b$Y+qT`H=!~e&XiENYac_l% zy=95Nul_CR_i0+<$sG)+3riVnmraJu@PnP_i&JYn-uv+;u<2E=c1>(r?cF&frLdU0 z)Dpj`1-}-PT_sm@NxAp|6FT#pd08?g_j!n<$|rZztuQ%g7wc5wlG)nh9$bZ7Dw%go*mb$r)U7@3;HewJ7x_0d#aDoQ0YPXdyn!hMN z@CGoAGUou2WOCR%F3B(ze9f;ca$SFHn(YK+QmvPRbCBa)6#|A~Uk*z-U^Tn(WJVlp zHeCZMWBs+vcpXT#)QGplf=8fPyY8+8L{%0%kASF(b;A-2@W~;vud?p$eJ|YKtHaP0 zkgPK~MZX3l>-1^_M|4_w2bTm#F;fhufBe|{aI31)3d8bW8#~iSVZxKTx+Pvs(f;ms;7P zZRa8*n}!%;5->)I@IKBdDsnEoc&YmsKPNr7wN(ru_M7X#3KhxeMgb_;2?@r6M_*8C zWhK+bgfe>qz)V%-<_UcB`WYqc;63+U;zMD124%Yzbh}G?7Iw96_twZw$dEq``ZTc# zY(+3IPz-=jFCW(7M$9natkuf-IR;TC9Wf^L9Yyxm04THLB%Q^RB%2&^#!`$@OJzGf zJ-yZPNLB+J>@Y$T$_%mQ#(m94DWGtTum$F(4sa3IP}>2y)SMJTuei*jWKxQyWl}1G zJ8?nULK{{O?i7jn$pM&+3-U%?kOYOZdP36=15(`qGh}JT-`)G}o_!N8$SYs%+K{5N zg9xO6g~t$`S-xs2q+*;rI zEFLhs?^oV}L{hr*zKO846rY8orughzr?N6M+uQ%Asfb`XGgAq zPp3BsT{(!!m_PbM`DqGOPTv#vRemJu^RRkkEAo>QX{82~8dBJZ7+4w{VbbZ@{ZJaB z1bToz(Pz3>HI&nr6YFEeec`FTtj{) z?)LV7au5}ZKOIjNRQXxf>$&#MMi`4Dj8o3D@>&uOfb?iuO z^(eG`hx{W4=%{M3moIF$k+buy=s&-+aTp3s)qlxCvMgDLk<{=Rm;=fAYTm-5bRH*% z?+)M5QXxqm@xV)Dak4~rWl4FKTB-wb$ zeegb?Ls{c--*j2_y;x4*#idi+C4TX_XhG1TGA<(FMVTbK+IwE zjMSJJy9N#p{_gXaSjiOF>`FmtxIji$VDmUj@LIU$%xe zC7qvExeaZ@d;Ze6efa3N=XdvB?iz!1z@&~+d`8JT!jH!3yH4u%&+*SsfA0M@H$_;V zhsKnfux`BD8DyuEQARm~osStQx5S&Lq*`gh>vkxi*UBcnj=IeaT%()ofQq#J5c`vB z3OX#FPj!$P%@_Dji8wGPk| zQnqSAVVm^qX||XQGza4f){o1?gc0mS$XU~uRtq`PF<~v2-d6#mJl`|n+eObIfth&z zgPrN~(afEnW;X$&u6ng=!l-Jm6Zj;w7St(T8Wi?fFn*S)pU8uFz97Z-kLDIqt1Vzf zLhDl7LK?I#CZW$kh^HAR4gar+2kl^Lw6e+P;`_=Hd#Q1SwB$`ax+G zBa4UAz*@zjN|N7sw0HLh;1i$iAh#$4gC2FT^KgF< z{`Btf=}pMwt6uGzGP&5Zltd<(IkLm09NeE$B+ws9$=`AYLw88TWzUO!bC|>|xwgSK zK1mxPBsZy5Sy?!nP(H~oU)w> zm3Oyg+j#j4t;!2HJ;U`k5U0hf7u7NOa{_S#eE(nvXd$+@da|LD+yk@FJAo2;=FGEf zbatffs-I<3I;ifu|N7UzzKz0LSdu-Ax^{Gn7@f%T|GnKhjDgB@i^C+x2k=kQd&S{= zkbg@B)p)MCqlCNUk3Tt3XOk*oP%lMg;UXRyo>I)nEiMagQVWN=70DMSuIqS9o1VhF zIJK%mt7G6LXvdM4S;BWD@nQ+$sY5MAhy{JH+y6Z{WZ@!w1zLz7S$$?XY>sme98kj!V$;%7NwwAwBC&$@b zUf)^zUajt}%NGf5RP>cP>XghT)Ik=sjK+ICOy@%KB2RoVmKXUHilFm)HmV7)?)ayw zYs>@FQg<4&k^?k6zyV0A~i$#B(>szgO*0JC;t5HN>|pKNT!dV4z= z9qjyU{raN!^x4__SJOx1$a!~b>@326!fWym3K9z4e_2u-j#_ogt-H)${7cpquP>Ez z?u>Qgf?CE#E zA5HJy39ab<%f>B=p#Kba!6K@vN)#%MF=gry9{yR{8=W1_67s9(>Dxja12obRLB)rV z)Irg(i`IS0($(xP;Em6MqN}nQY42##Q-I5Pd zG;e(zEVp^vmg+~J)!U{(3@c6jE|ts;mfNjGeX6;+2Fl!EIpE)4sL__YWSLj>m+#3p zi!9Fz09r?RB~nZ*GYsoo{vzXwaPz#C-6Z!Q-iA2qDC{3YLr+*o5K72v5cOf!qEBJ3 zk-m-63SPJ{Nk`V~B)Hoq8!@=#Z`%V*72>xL(rA`MH8X#Bn$zRHtw2 zfb4;!p*64idT}NxeTRoof6-X;Msk|plZbykge4_;dsqtisZ$747J|-5=tr`S;=V-! zf$l#Dq{4N7Ad%30e5U-QSO#m2_{r*sBnuElEuqmWaiG^zx9drObkFIR6}4*{KU5a; z#jRd)uya33504)FW<_`8Yc+kKatTS5$Zy`0;?#Yp=_G<%jouF;mGjV@pS1=qkpkK+ zBTPmp=I18;>C)^&cuqOq4$7wBh_IqILYH1Zk|&`{7NJYtt{uWqzTrP5TIuTzC$fB~ z0;%cb6R|+4d;4JLS998bbLUxl6RyjvUhSH$%O$&=_F|fp>kcH$b;E!9Hmf_-uK$Gkg@<{}Nuss<% zILA1}#83Ilvz6$M`$ZmI0Pp@{f(8JZ721wv!nWjl;<~Nn&6SEoxxfVdHcn6rtfF)% z5gkkjp%=kLbS{vqdbP%BBB|!nq-ztzSwwZZaLh283MJviti2PRR?g4gvj;er&y4Tm z(~*QbG_R@Jil!QNyGPntN__ayS+(>IBPoUIong`o6SG@0e|A~i7*Q`QY8BR+i@Ljw~k12jOA z3eF^$!?p~GBy)CMX9JSHW{~8=RT!DR=RlW2X{?Yyp$9J?jt1Yp9^3#(y7bjH80JPW zOlO16H&}ff3Cy-8_@&zKYg=@6Pt3xy zB-MrLhZGo;OmdRg>j_ZMpXp5Sk6&jQ%;v=*A*+-`;N@0E2p>L|q=ekc2vCOrlm(_q z>@UWo^%K-0T7pR-K9Hg-R_gFuY3;dFYIXt&e7HF9%jsy{+HHZUsKREW?p8Ouu$ z=@J9w%ILOoe!j7t#M79kn(&UL{%t(ohNMo>ib08r zS_HcaE0qk#5w!qbFCa~w&8aCnHnO1s;+^%=Y%!B`ONL)Byp?(iEqH57xvwU?U1|b! zAf^NNwrzk^wUKqvfVYw>@q*d{q#RKL$st{$%9XH0IsephB{uwHH_AU2h@ZkHcL7>1 zhl1`*qteJnTB{}k)pY2=k`QlB? zlf)$>y!dQI?8%RxK7C?PVdL*sUbh5bHW-G&>Z}ch9U0buB%M|{uMWW-RWh^|!L|&=O`a$MiUk(Vy}xe2xw#{7;^UP`o8Q{BM3>bG`>fQ=>(VDSke* z0yU1HX{tsuKJ>_7YEW`0XV_j$p&ZHmzdcRTetIUX0qVED7TPpt!a~X&cupV6InR!9 z!ErY-is^AS<3ck>_A9Mik9*^|x}LtaufBu2duic*2&Izm7@O*Qx!i3O3T2bN+ZErv zT#1hI5-#H!1HpS27(hQ2atlG8%o`Lm05urm$WLpc!)4ac)(PtE>EKBT!>}U9Mfs5x z4it2S;`BV^+3DUIxrqVw;-yO+bA}Y?gB%w{G;kqwP)QS|GSY42?0nymr(R#F`5I-F zu+DW^B28$hK4a;mzh%CFV0iLUeHLxMd4u^1=6KDL`dn_fg-sK**|7k+gPjo^-_Fy=Kl{&u=Z}8*DR2&U{t0W{;Xg*%=rDUr5Hf={0W2@mFl6!b zlLZa$nRu%4+y(lM^FMA=?DGbj*ly&>Y`_AEvR>t}~&VoP2x$c1QF5bQK1S*0Wp3&b7Lgt;zR!d)}2g(3=| zZR1?ZG3+0sptZ>(KNwq1>xQf&KsyH`KbllYX>}R8Fik!P^-rL{5#|mlZ z^EUEy@6qYL0Is5!iW(}$fJ>WywN_sHCFK1&AG;W#%w*1WLAC9+@_QP6g-ze43@~M) z{0O%h`F#!nCOfD0@gn^pB2$yP$nhx8l53p0g2QC}SGy+A?e2aW=X#OLmdV6!%)~ch zCNAYvP159ONTC*uI70PV`tgi0WvA139)q=9l#=T639jpPRzgBW= zHA-JYMN>P3;LvWce$=d_MJrDxN9n?_cK_Mo-DA{-_-KM5^e!HTLA-6mBdS*k<$=&y z=uGi6fyRr>S5ir5m~Uv$u#t*63hs`{%)o3V<$P|xdLmku70rwXi!s9fIZ*-gnQ@l{ z@;>x!)8vSvkaDK z60w2OL(aHG6>_RW1YRSwqua_F$ltjK$zpN@@H$GS9Bf=?tJWqMw8EPC9SGReAKMTS zScvZC4P$;r{6iy3>>^#!R&H239e5fL-p|JWqG;!y%c#L5tK_W>q z$i}LK$ADOo`LRJr!IANU{9s($WeW=jJ;v1voZs(7G*Hn^bh;#BSHsZ`>JEl-*}xwH zUvX%0nqutzdvg#si8wEa2I7tJ9U;zT%Z#ZxTkejDi{4jQ4_1_8T`O>PT-2NvY;~Q> zam(3IXLLcUjftv-S*19H)Ek9bj?e?wsn@6fG^<-}jI%dsoR#{`mP5>eO+YzOLA*jc z?|a`TuV#0S){j}Qezj|2)+Jk}fY!hv4Vg-fY zFy0WvPsu>nBdosx_q1uJ*2;Y4pyIEB5E?vmNMT=?VLa@A_voRrqWo19{aSadTc}Fw zS0O@|Hm<@0H-{vo5DZ>oxADBogFNOOH{7T25n&2u5@hM@+hpZK~tY2G-cp3yNcH+%+1u%>8vr_2zS?zxPPut2d_%z&s$VV5Klzkc^t4(jlsVJ1hv#65 zQ&S&XMe2s*QFBkJB3D9frTFjuVpO7R9}v7O6djS&J^uSI>Et!RjYr1+#_aOnNK8W3 z|6^uAou>%fj9JoWkDs4W8Tk}5OQq)CGQ?&1Jc$R&b|PO-)ZuwDCU3?drU(O0RcuJJ zbPOK<^fKE!z5Am?IxUT6Gq-Iw#M-qY_Xesz?b0o7n{g^*8ODZf%e9rc4Dl=}I)8NJ z^pZPS$)E;n(yh^%kRLuG`SDrprnU6%wpTY}+n)L^V!KlSN#=z))pyl2ZM=HI&^eUJ zxelMG2OuF;%HT;6%n0XJ9rosj8Z+Sg5)9Zj7h*5`eUZ(RBCx?iNpF%2A}L!egOSv~ zDPl3p6HO_Uv*Ovv_dI{n8|ZiCq&G?-q_`xKUW>J^sS0$S%J~CmWZsxHi)3dKboIKq zAffAqA5}Hy=Ot`?*?dhc@Vz#OuU@25{?uQk@?c|Gu$e3i-d@`Z2;J=vPXzDHdTsj` zML>INsGj0@C}vU!qULp}$P@BbX0vA7|A8p_**qK0-kcHROcff$3dDF+%%GOa-up>B zHBK3hYfXt+O0# z$=x_fQF2+##(Q~RX}Dh5_-u?E=_!QP>1+5qaK9ENZv%!sKH0ua}})e0CJQ z-iz6rSC1y)Ph)QOJjr^a3G$zfotr35Di`5_hGN~aBWS5=z^dEGoaVCiF$rY=CQPq_ zt&&*u#in$vp)T5%O4h1a%2*AD*+L?r)Yeh!hw4qn-&8*uz*CLY7uP|1|7+zN*pFjM?j6?SQ;O^b}mo6l2%h21I zoT6VtB6oVpo{FJEM{;&rG2twU!p!AV0gTEix=I+P8IlMoBxO`27dkZpnRmY1x6Z8R z{_pFz^j`gH*R=F@w=J>wx)kU(ZA;vrpQ+Hlp2hdCm(;nwJH>|kQ}#|@YamFIZ&;sK>CNBnh!b1^fQAq;Mrs^iQDgk+PX@QS5PVjwn$t_5o7u_07F}`sC$PaUZTS6bF*_ ze4#N?%#tmeUd!V0XUjY)tTqkvC}eaU`zZMrOAMsQZ(P+E-qR|pbOSbB+d#SmFBVg% z&DLu*khUiCat6$E6ZmJJ1*?1$Se=8%#O&DJ3@-wBl_LQ&BvctUG0o@^D>=W}<#Z8HB@bMY^Hu6FEk15M`8AqGZitnD^B7 z!mTCUTO&6Wit-uJOGXmkrn0O<^XkJNW_u8Id7k7=*VZ_6p6yo7&)*tIb-!Kw&G`EZ zdHg>l)QM7TE6wXdKrmX+bHRC)8nY!?l@2^_@*BNGvIY9Wt5L zfkB#|YwlWBrO!$<=_hAn-b78`WYjc#MUn6)A__c;$ikgPLCz;nUhch`pA{-*v$A-p z$u3d$FV6J4>ea5PVlELXD%FoS!VH7YH)?v7irEO_z_Qg)Qqch6KxyNSUVB#VPDe74E$uLfiRB_VkswcOCo`#FLPjV4@!` z^Nts-avM6BNiyHwWYA^sKO$LokzN6%N=X#z3Dra3SwjbiLdFj%K5o)O>_X;1xNzV} zs-#0DT0RE}Z~}4&a*i}bL_tPT(phj^@{)!_YbbEZC-0Z@NGg&uOnwq$#PvLTz_s-e zA13TbEK`ZtPV$#C*|9$oWq*Yo7ggm^=)YLftx4^#H=6F>=d;njWlz4nYTC$zWcl`$Pf z24YDXYC(;l=2MO5!v-S9g%~8zM`BCOjxR(VnqI@U)P3r2=<&^UfBh6-$)qPKU(69l z_{>%GJ?baLH}%cF%rcqFd>**+B&?y#a&IX2tiC(v>RFQS?GQ^|UL}9N`aae8rv0D) z-~adD5}zlzkMw7{)zuj-f5!O}lC^Y+G{+lt)8t0fx-wG+HI#9dHT|V*{OA!6yJ$o-d;h0db2{F zB=tztXIW*wC;I_}HEP|-8;WA4)}~O#}J5ykz@ZYZO!xmFP@#~Pw-FUdi28!P(6vqYP5`N*>)Xs5m{RHsdyx33PnD2fp zp!n+(Vf*Yjm97KDuY9#@Lh@3PD4X>&J7aWAi=dD!4*ZH1hS2&84 zGhDI@sRaBlG%=#%!7iUbCUmKs8me+sN5D&QuUfsTG0z88geT_^WzRR*-xl+B&uw4Tr?#&u{9b+ zs{t{VI+Z5D;M`I()Z_3I)EL-8m%(R+YK8Mnw>HnJYrH~}*hQaBbS(NFm=+C_I|271 zp4DB64$k=Mh-l94(9-v;kg&c7O&M7yVARBnS`QSRvmiI>Ykg16Oka&4`mtl`<2|Lh zNf|e-@p@5T&x}I!%XB<-w`vALbyxaM^lZ0D9`)>jztvS*G?!da&5~oQD{5UqXyNM` z15Z}nxt9ovCkU3$j!oGAT8KT~~gdY*;Jo_SDrUeReb=f_v0$xNb?b0_QpdZlSy ziq)dmNsTI$BYHi^+V%%S_0(j=!h$2?xbiL)`oFNwv_?>OE`(V6S#XD(>n0&<5{7<8 zs8e<}ud7JS)~XjdYHJUbVEnFKTUP`ohu|S zs7X|9D#Pz2IscP$S9HR|zJ~a_$hv3W-{WxVstLRQ7%CNz{-yn`M4 z$-9U5-yc6&yFzyLi(OM6>w*OIF*hWgzN|md1{8LlNR=Gydmd?qy-e^%9 zd-}(-r?)lsLhc6x(-jMU-H%h)KPw+W)5{>pwB}`)v@_~oKD{^j;q`M$$8>Tq?uJjF z&VKfuy;AfF%lNy|>wE{j?n0N~s(8if>g*^h;}s=*W}J-`;d4AC=W|5S5#_+y!8Npl z+h_+Hhwt4wo{vX=-~E5PqI~}2oBiOM$onRWzS;M_*}M16p0AF$-$eJn+4rUCj`_{L z^G#@dv$wDM(6pBy30*l7-TCIuy>0c${3g8X)8GB(&Yt`yW7^+D!8iN&zqxbwn~DZa`zEJ5-sT`Ro&uXHVF#(8HT@ib3Z;cZoi4<>va-b^Kau_n; z0--Fo-;JfC2wT0C$56Yj?{343!zI+H!rOaJT)OIWu(RJg`z`Q)dll~O#b@Iuw{%!Q zIl#p>$aBZ3Zb5sLG#SIvm8!d*r2Sis0HsFXl}oO1N}>DWT5S^vG;DGQJ1584oH{uY zojyN1!Su1Ej_~I!eV<@e{6kIUVl1Cdw|3!Q-`+Zw-x%~*6?Ed9VLF!xRtoTpnDC1` z&fy*H!<#(&+S-WXk?J-6)C^V^^>p2%PzD( z|Df%AY-JAU>KFfF0&Hf}i-e!Nw_qomfv}7Z@k^PAcsu2!{t*H?XN2F|v z(MYNtx8vbLj@7rfrDfh%q2Gq%V*#htBBV4ZO)D z7Bfrc;30E= z1xJb?tgfBqB2P_)&u=KP=OSwVH55RFJVL5z@Xjp@a4cGx&02utcm}oH9R4#HLf`MI zWRLmry;x@7`1|Ae^Tq6V3`eQ)_m|Vf_+)lBBU3Scn@$(A=c@3eKAwMm9K$bg!g>6j z5yXEI!^}qjK)<5}pAO>TgGYBB4TQ?$JO}2dQmINuCe*!=e_gzrlr^^eP|_dE2VEYk zUX~{b+<(uK0XZI-WSBHPj8AGUm48uxO|`@6kBt~NeQ3Qt4N0+TWnQ?l|F`@ZLIBiZ z67V6r;(N821T6TE*q0i<4B?FZ9y$RFDgi1idAm+n`7%VY{N#KY9Ri!L!Ixp(+q3SA zW#Agm9?HrV@cKc z=%Tm4w5`A*WuDxK*)TX-c6+DHGLJYNd=LTtj^_UuP%rS@BWGf(~|G5Sfj6 z|GQ*7`}Y1c8-M%w?AyDecs846OSpvgCr60iV~8BpaN7ocf^FkelEA0@PKOCUvnjO^ zA%=Vh9>}CxlThH_G9Hm2YRuu0q!9Xl3r+riNMZ7{|3(fPU>X=t7M&@<0K5v@E$noI4sux0B2l7DV^=^F+IPH5@Csgzu0D$H-fdKiRQqeeV&v?*utc+>3zUN{yoAL_Kj)%PnfiH-v#8OR zT9SmUK{h~qfpxIs4tkM|G7v|Dfp5mWp_wGZ$R1KcBIxxZw*n%`n!?=IaP?~wAd*~Bk4)s&3xHBiEWt*j*YPGNM%p_QYT%2d$kW;w8}*ZVCu zBQ(x|p&hdM+SuwGWnyUbqh5#)J9zCLl=-32hZ??qW2}y8qS-wvxSx@(LJ7abP{pAO ze8U3qD24@NWZ)8iSkuxK!AP^UaedO9ApYPArDyk?x4m@fnmzAc?^S*xCNgtUG-EPSTDVn#g5|Oi6 z@eP;xMrL19W|wR5(^>I1j36-ettJ9;Ke$Ye6GiM4zT`WV?o+je5D{!zWh>i4F3fK~ zTD(bK9w$$5iuYi%`#=3rd~zb3b(nIK36)0?2r14R{nMyb=8Mgcu zrJu}xi2|8t)vjA@r|lCI9H`DH_f&`UIL9F&7z1g2$h<>7VAZQNPSZl(d?Iv}Q6vOc z#n+nLlT<qI?A~ij{U)h<=1gSSPMp}WW7)Bj(Hxy7*_0Slq{4?~c{ktv{tD;^ z(A}gSrV=S;c1M;PXfzr?qY733`d8dxJAg%2{;wT(u(n6fTH%Y=?z_@#ye6wLThL{n zRtbu`w=AuyAmsRMRRWUd0)<)d!@r+ZEwx?M!D`7Vo)fmBmldgb_w<#q^@k3 zvj3KQxUub)7p2y?4-KvuSg|AD2#vM?o|bD7QTwyC2-t!=;!K!L&M{ur$&^O!g7%=L zpefr6l~qgkdWAweSX#8~PD?Up^4HP<+I}2_ZJ4h+5QISHEt*V0atQ%lMoTs=Gj^q9zel(PTcv&4D1NV{Z8fYIYQ*@%-7v;kyg_sF^a7+<{nG4KkX zz3!|9th@KMYTjsntp)(*cE zYysDCb8i^cDgpd=4D=RD+N>6oGkj`p1mXR`w%Z!Ke|7Rx>jspDD?ja;u4Qt*S9cD9 zzp=r@5vVVAhKl-eN3@b6yny{y3~O?hvJM66`{rPv4+S7-Be(dzP52wodx+FJFg}qy zEwHl<$m!B9WxO)+xDkgOuWw8oLapVJ|G^DCzYQN($MQ>u}oc||(+tN_f z7_G=qWXQsY$+VY_ngMDmf6>u2Inho#TBhkOul>*A5vJ_^KfF`24hOyASri`Asx2VN z_JGPLS&LYl&dDO3&L@&qZ>@4Nj2q-65QYjQtnp;c!KjH@N4|xMq?F_JetPQoV9{YF z1dbjTUy^q#d3842qc_SuDgdPr>}VXcy@)i5NXkb)%;!(Oodh=^DX;vrYf8%ct|EdG zel4Oi+AZ~g50b@bewd-w?uekgRN!1tjeA7bu0+m}kVgYc%@celV6(YS^qkn2OTvoZ z%a&I;`TqjxzAQOZ6L%Yg05&NHSn?T5;a4X||apWy+6M3nK zPi>p{dUiZHOx{mh
EN1Dr1ePMli1QRC- z(PWkDk4_a|w%hraQ+_k@BJvr=U7uHs;(q${2S>gTl>k7($@txuPdFyko&4N! zKYsty-c6XO{zv-*Ymr7irmG%cgR~C5_KcwW;y@>b!w$NrMy|aa* zl+QMd@^QLI@3e1F%cKEN)M*3lY55*W%sQA9l#^?@-dV3*Y1dZcNKdLKiCU)ONrEY& z_@VMCXFI<{&BBA&Pe?;sPG%?$F#7@206KCsTZ{*zGq6jR8TH999rc~bK=QxOo&GdU zPG*kuWSVnDYNAGo`K_2$PY<>qKKM0w^yuMtenRALZe4Aq{>P*7X?pbG^aCx~*42k( zm*Oo+T#J`;<{YJSr#qdzC;$7wHhFiSezW6zopt!|BjHbu<| zSRs6rLq|<{B9Tw0nuJwY&0qB9jz>6dd&dvVF#cN^j{D@v-d^~KDa)ff*Ab3OKq%q3 zc5>0Cb{nF{lI~2nKTnRd8DGVv5^G9!;*&K0_k!kiIIc$Py~<9?r$cmkNzURdc|A=T@_CnaqNh(gt2K*wW3Wf3x^HN?m)5`>( zCFq{TAhZk^AMAHIC?%i49Zt3T|Ts#v+%u}MIG4-W;f0=}e* z(sf6C?H7X5Olx?w;|iLlfYO+tw6=$lCIngm9vG|z`;%p>sg%l&s{Nw3L^{Cvu}BY;=URW$3;+9RV|tfk9=XBUBZ!T*BoVO))~I zc6Mr;vaGt8vaT++i>EZ`YHNSkMn@Py(5b@a1o&amLEsedu|v7-5d4?mpsLj}G>Z05 zSalJyR^=KJ-c&lsi|Ow~Wu%L(bW}TGD;;)esdSW92-0stp1FX2J?TOYE59D4)ARcr zUbS%ogZeCc$T3KGuwA6iuamRUEVDfPnw+82L)$6Fi~(GrSo*ceVn(Yo zNV|smBw8~;{Bk<_?&aIWfB9sY_JKd@;zZP|98$@+&k47GsSx+wtS-*Spm(qh283TE z@DOYASZp%byR>21bOqU~gD8Y{*ztUlzqVXI6dLu_9;xaLJkGqMKth%3cVmTY&6Tw( z?JPG%m>wuVRwS&)1qFVzSWx9f+RUsRub>ll9!?<=;F{g`t5jWY$8t6(?Hu>^k}>v% zDWQnFxF)B`Xh=lYUP8cPGIE5cRym-W0@$9VgZTqYAV7} zAUfAzMY)&(_AhDX5T28$-Nx$#^@b;#9;U;80$8qhp)z;+=dxpqPH+r+wc=uC5agKb`%SEaNx#9eujZeHN5`n{5pip~52bX5aWE|)HR-(?A@ zEDh>%1gN!qQUc$u>5KXNXjF(rA5?3g@FSI@<@utx=?Dj(K zXv&(LN(0%{(DT~%qBU9dLI!8^AG>@TN5{xoyN(gcn!Cg(nV-9NOERkA%&sjv*}57I zm(|r-DDVw++1jk7yg7(&4x*KltCD^V(8y9g%J)*MusMis4x*cbs7Y8>H6L#dqSFa> z#2lPwLp2L4_Z4jES2!A4X80u&ae*=Y3QyY{L~q=(yg7(UCXwHB9WmGBHwV#-18{Q? z-Tq4+L~*<2Owsw8=hB6Clz;1po5MfO90Ab-86SA=nmIJY@v?%1vh&%&XCV+$*Tb8I znr{WaYmjqZD&5e4CILCWKOdz>)8PP9pts|^-QEB8=?~w$e({6zpSv&r%X#p?`EGJN zCQV_F_#Z&c7*GV;I)f7UWYU}B*XcoD1a(vDoK8^%rsp7_@%j)=0X>u04Oc;xSNQ?| zNTzG;!!plCg^y65j_MT(H{a5tWp`S(G<~<}(epU2b%@**-h^@%vuli}iSj4Vz2EEW zuy*zPKp0rtf%;v7b)Ddg-M-b&%+%Cb1+2bVA zzWCi}HZ|&`br$VcX)^k5l8j#r_Q%QeJuhf@Ab)@qwU7Vt&HgS0OWlnxNnNR&+^PpE zU^0QQfZj--`mP=b`MsC=&QBAOG~!@XK|fp0tH(dRc=j5@TPv%nS{U>n@4j}Pzj*c7 z*?mPRt?8EgDo~T^v8P`-t8PU2x#5@2FgjevItkqc3C z&?Zc@b+G;Hn{?v&?#K1JKd=6@Yht(xiJ@ceV%Q&4y3yKU_HM-7T?yyS_Tif3!Zze8 zBsWh7hDo+ifWTUxMbbm?N$5*B(3k^%ON^L41EV zy*8XlOTdge0+C09V@T#(@W8nBNcy(K&D+6Saa+c{9==6@vgE+7iuKrYBfcg}tnZVy zd-rtecrbpGCWPRN5(jy$NcwEEx;eT2MTT#=fM6d=G+i!imk9RS;N?h$Z%VKeJ0Fr^mk3{uO^}RBMC9b;?N46sm)Y9o`qdwHO|hQueiFPz z7<~xakq{HyQK?=-Gi2Yl1!oZarG;l;qp5MF;^@c1QMDxjYoqFQ%t<$4>A|{tsTalq zX1%Za`wt>_s));G2`k8jd?7YV<`P1L;Y_C<{X-3Baq#(uvw&U0hYa0#LxEuU+HjVa z^6?plvq74UW+3aafX#MlK>6L)kE8LckFCe@$zIn4wr(hV_Q^Hcx$?7fW8pGW){Pm# z+VE2QfgcOMQYV(25ysTi3mN08SuE|8_NcT2k!5C^O53IVDs9&@yGSran)H$r_!`my z4I66Y!^Maal*m2y8SwDVv&E;0n01$0ri#b*I^|<~J;T!0Gp=`y>s`nIvM%?`>sfte z%KU=%{ruX$LJ?)zMTYZ}n`wUls--mzUF#P=UmfSEd;Voo;x6BRkc)hI_7Rl*!>40fkCKBF5_@B!Uu5a6SmSDA9>}m0B%KJBU>gpZ!G6!D|(ts4R819L_YhwIz!v zSO~biFxDVat7aB)0a`W|i5<%*a*&!X;bSF5jxv011ZQcA`0KOQAj(1ruE^zt)bwFm zY9lxUf86tbe7AnP$2FdIO~Ki)u_(_5xnSYQ66(g94MbZ&E0=S;Vx}JIoq-Ui{HUWZ zfQSBVkc%e6)&$03MCJ^Xab@S*T`S z<-}lb$?cl(=fA3ffG)Ns{s;R^AZ!nxo1j?N7>26lhi&#cf zRR0zgd(+9O)Fg-L5jS4kq)fx&sY|Nd{9Ey5tpRjFojv${5`tG-~bwz_irZwc%u z{Kh4yV*<@UcXTdW`)nu_1Vz!RY*#?t<+HfdcCEJOQy!7|-0IskSMt6ZmCSGAIGG*K zlOu_={I}9<^WR`jScQ8uHx{;t3pH=OpI=?2FZc5%3-5X*27C#j#1j`g{)(sqRc9kI z{u&}9w~}LOHo1GMGO}Jx+otcKo>uFM*n~i&mzJ4S8&u0`rJ}vshjQ*}XE!8!(OrjR517M}eRzT_yoZGl=XXe6j~I@k-~U%78MR5l1#NJr zW){iAuQPw18-SFLRbsjlU|o3o@>0>kwZZZ0XP=%Aq49RUKK?jP&kwt!k7R5)ahB#} z!ItSq2(eXuzdsT6rA^?_+`2(0h%xWU(Q$>}Pxe-Pq1FwIj(JDn2xNl|+g|LoTmY7B zXnvlb73|U)R8C3NhDKhIuUTr2s@=0L=msWL(944d}^0+f2J)xAC5o5 z-)^R7iTf1Wk9{@ijwT=Tx)d9BN@Y(}5b&_1+aH8Mk)cH!1Z?~m=t%1K;QXZ7EGxBYzU#9=`h0x);^^p< zPWe$PJ%l_@asv8Z7r;7pC(rD!(J!knWb(`V;-$t79;x~A0;?~Q#;_}}N<$+H@ymYX=FS@~A_u97DpDQ+KTo(^i zVi`#*GX};ZN{$c*vhLuL}^ z32t3Bb;7y!$z`Jilqe&8UsBvW~qwB^DP27=ddkqQxNt)mxQHH}oMxVsU4q#Zhwm zc?t=uDp5LT&{ie?bbg*J9I$C34QiW2BmyNc^iN=_uVB+d-7&=81#UNHV4J#&6W z6!I)Wdg1&dH&OaG=jmLxpm~IkoH?cU9YX0GNQFseG$MEMOse>TV7a9S9IbJi=ab?@ ztO6JMr%sENM=io8ei!`WXu#n!Z6-Ch&|E7&WBz)^q+MEZDm%FR_9JJ2t?G~2e zNtXE$E!e|o5btsu+JwP3CpbkZpJ_2+lJ@6O)Id@sov6kZ)8`PJVJw)-&3k0wd~ zV0(DV&wB#6`^Nzobd9b#xTiV!UuzJB!sQKLQt{f9{dEET=Lz?Kv=UIfThR%{HO4v< zPPL{3U4a=fK^wdIuoFFQO6z>B^v{w2#!3<``#Hc2o&LF2 ziYk9YTogEhoG+=S zWn`et%JsWqHe$`Jp!mO)xCrJq) zl7hcZVAMJOo8yS(pc;Si{?)VhFMoRZ_SJ8O%+Scn&ffg{pN`)=_Fl;nXjVLYw}Y$n zLOj$my7uzvJfZ=Rsj#LONU*$T8kKiJ<ux=>#3$m%!(m-f5KbZsl`Hzl&Y$%D`Ip=V#x#?|$6fJ^$|Q?q>jNo7!mx z);3Q666m8v=n&{go#KQ!r78B+LV8)IphyuU$#o1Kg zZ!@5-3BUE?0E;12Id|}jUG*xk+RgH^aod~aC4lh@m)Gx=^eEWUEPeuyC7&I=b1NqV*k5a)Fmjh0?;nz4A|vw|j)*xp93iU3 zk~^B5Ap$fP4&!WE~SIIoh?>2XbCd-&w}^JFi5|NI#bTjkzaMcw)ZN6dmN3?sKiy7f2?J0z6E zmQhY=y_8YTzK&;${>e~#_LK`XmgrR545X^O?}=riv;*DSmz0Rf(S%7KIge#EOvuenaZ?Sl2AmxSOru>3M!rUfF#cGG#%hD5v$5Zrskp27J|YTqE;_#xyaK9 zO{i_j)OsmXTWFj)VitF9fu3y}zjtuOh13C)02;Yk*7GnN*OuBc?gnsf|=A0>eTh^-@1G67H@Zd`@R_LW^?7ANMWI^Y_}!C3%h*i z#lRh1$t|V;5#<_oYo!cp?(<7iXKzb}8IM*vKFltU(lk6i`NZ|o#CY$g;q-`XLKkO4 z{3cBa^BZWiH(5B-*?pq6#4jjUAua(H{*b2;3aHTFPsbFERgCGlmpbA*xlc9c>E8(< zoJqxmsM9%Ea>+c%|Ar^`VfF1Dv#+JYDailcVxHc21XM_%SQl^fNZx6!CQQ$(gTjTu z#kOE(nyek+@3k?`fUm0p!XXoFS1p->2!E9J9FHh}QW&*%qLN{E6BU+WcJp*THcI=^ zuJ^=e+Vj#pVNeC7T|=peOdeAZ`({%Nv6E% z{W$;S7x(#1`2Mc^wCno*imgipHsR9j#ERskCW-HMV3U=O5A>de{${7CfWLudG@!rX z3KW|78*%z+;m#@}?EwkfZ72v7u%Z~0rF>qc(f3+x^s^Vo58w25ANTj%?q~S?&KlZj z_W3pHBujjLR^7|_{0#Z1C5e2q*|?Po)tcRXW{689(hhfDOHOnv-F+)-nvpmJN@=%X zJrd_C&c5p6u>oY&(#_J^KG?QGD89(0b%UnEyTU_X%sZb%MXOhik3w{sA5?MC`i}6+ z)d;^D3Fhr>-fTGE%$5iNcAcPoRpdw0?p;7fu9A~vBL%YTAuIQG8QKE$v9ji6r2xk! z%>=JyUha_7dwMDM>eWh5M71vK*7mg#URNyYYFXY)QQ<1?x2ojXtoFh^vC(-NqK`p_ z+{)E{!=iGfo6TJVZG4N^qmB^KmChYSjGnpKbSc|{;bwb&xLAZUI#MVJO~ zB8O=g3g)=ehlo3HW3E61q^NQs)Joq%)k$i-+r_E{PEwMgKf2V z5`q@;1t84v!V~qM>K~eCeJDz)UiZHI(N{cDY4Trk^WKZ+!6RQc6O$>B^%W_Ksk(n| znjVqidP=u2>s&c^G%>k#(>N;_{|48nRm1qyq|H8CSLX9wOQ!DaCq<7*T5sG2z*Avua zdL})wln=8F+EyxvD;hv2hJiC2bokk62dc07$*sY`w)8`5nwqXsQ&;x1u1f#H0ksx{ zcBv}^`qxfV)RXqW?fmc!eW)6?D=yYSGKS?G*brp$^U0Lx>`vf%QS{Hn{N%7l)~956 zH2z0cdU%=~rT-x%+A8pm*>N(}7xXgz$DAkH+?VnKGI*#(N|Dx+pi=}E1bWKsh z16Tj<@?zCt)Kk?_8kwn>4^YD5pr`H*m8u>yVJs9isgYU?x;;_X+HFzZ(Fu7nX=bQ7 zNL^J^>w0oT$a|5hrDFDR5bQ{?CK1gDMb!<)GI_7WB~R zH7XsAyqGRV16RFRtn|(NWzZ{snlHe7v5+F^cgOXWDq887T2y`g7A(4)2~~HtuSTiq zj*Kr=jfy@X#j1<38cXT4JvS2C%dV@tXwHMxX&LqMGsur&_UEYDVaV){L_W-DkPVqS zlIdqP&ohC=3|xO833URu`cdbDu4IJ$DXRLB8b`?w#ipX(*X*I^o-E*Y8M_=Wzdkxi zh9jlQQCBgS)|LA)TQ^_Ibu@dA{LXP*x$gw{{%qWW%O9BFZ@pCj#5G8s5&Y$ z^67_nuiv=4j}PC)zkLpTp`o3smZvFSExTiW2w4JOucUDH}- zqD3_ez~3d~pt(d${|{=dGPuJ73D$v8H*Io?|Vt z)mqjSRiYaN6a1Ro3383n=KIH6pXJ*v4dBLhUtYAn)Fx);W?Y|j05Tf3l@N7!vY+7I zkFJ~&;jH0Uqs~r;e8~-N)2ke_?)F}Ag%9`gBInZE#^v{d+?j=E$D|mj9trE!azvhj zi(39YmQ@CyDjZ7Xs;-u0&6E|c2<+P|cY@D12R7KcGhi(FpjKD|isg5EmU~5$yWSBp zxc=^KAN{Vsd)w`7cJJa4aDzw4JFuL~V{4N#QkJeo8L1scueqFS?cD9ebOpG9cUl*fcdN>0VHJJ{_lO5-~zb6oiNrsUGllY_Pj76W*ps5MAvRJSo(fW+8>9g36$ z&ac@+Qr)5?1|H6eq*$vUgd1KkZQFFi$m@hLi90+u@?i)w;e)V*RZE%m!ji)bh2Kib zfS&i297vg%B6#5*S)6*w=#YsM=(mCRdpVtaNFnihusb?UXAkzIfk8M=o&Mi<$pl4Z zwRJYd8$1C^q*4^L5;8^WJVXGs`hDQdk|9BU?}aWIK%Hl)^Ey2_W%)SczmtB7{8E2k zj8kVZ9?n%09mzihkbJ>v;+bp3B6WmWphBQR6b=_FNTenhl27`G1|~4<&d4|$hV=S) z;q0fUl2r&85=1Jl0YhrbNX~nQB+Q<5dVLDE_qqHOz)H&Sr z43Gq3PG0Q=Ajny*BILg1z}?%aU~vqH(NdBRqfR50$6o@5bo-J+S7xel=qrZAJbELK z!^k!V(iZ`}3&MbuHV513ksAfwz5e=fNO~)}%G0iiLu%Oc1dtSv1v>;~&sSAJ9u@1p za{aa;lCBhN5bOk`nvV-qqaADzwS_pV9kj@AU)2);4q2tLxoT#!KTjoarb%xIyQK3^ zdb3I`SQT0D1rw!tBnxcE^yo-xBR#tI@yJXuU@=){rJtR>~ePX$?ltX&dXOXo;-cFpWm=QS z%h?^GWXvXL#b~)un!cVaddI0`xa1bP=0bYvJbAVI!`@@(Z_ZcqR00bSw*)piU;Ubp zn|s_%7t^WqyA-sC^Yc^W_r1?SOS?XbUPVRmTn}%!t)gVuL!09*h{eKCje_(!8B`S% zf*?gf@mz-hW6$3SqGCt!WHxV|g0e}vbhAm9;4Jh>kVft$l_f1c*#7bTyZv`RzJI!Y zmBDXa^=a2t8R|Qr!Z3F_J`z$ws4J?$9Z7q5soWi5HUBo&gF1@YmF)(yNa2di6L@$) zY1V;ZzNEp0NL~DR&FQKK(>npDsEfOs60-{URA1H?iv1D`4xFM>O<%H}nL-xcC|A87 z3UOi^VOHumC5c8}Dk8Z|bp7!4{>ugQ3cca{e8=Ihw=Z5jd*r+r47ikxoF_>y?M^1| zoyBQCLAoD|55`|p?QniJM?}tHJ3nUZ6xsQHFCAk7dwD#WPi8gJj3iDz{P^u;vA>`P z8(nrW8Zl4fLmfvkpB!2WVm(gQ^Me+~E=ZXP$9*g_vmpU|=)7_{4P$eq z6xVxH&+B~V?{s;ss-=*VR1{0%(yMwUgI?#?;EaLjsOgoyE2JA%0G@E*O8}nRfd+pE z8hnFYLmgE&ie<^S@g>B>wE_|)5CPhWJ#tO^9Tdw4=PzHkKMnnxP%N+dwCgIC_038Y z%MMl-95E7TdN&oz3r$gP(?0y*BggerfNi`Jmi*wRDGH|)CkYQ1MjMu2iT^dW@>aD} zwmL1c|1bfaN0-TS56d*$ErEgF4dD%_RmRaLz?-pV>8a2OQ{d*P#uBbry@7!@MRb8# zY{SDHMlIr4;YkKpFQwjv$J3gbHtyHK-dRUjdzVLYQd}8xmU#KtSrz!}suTjiTb?JYc zuNW$Ptm6ksuQyp>zxp2sCQ9eSWIh}pIb3wU?#^fW&yy7URF4{?M0&L2yr<3_+bDX= z(U|Lh-=8Pv&Nq`$|L@N8pJ-_Aj3;N3$mi$D;x#l3VlDGJYfm7~x`bG}rhE9EnoTPf ze!K?lgVf1&?Su6$?l?>PklpDyFNs+TZZF@5RI@Z|DaN#o->t9RA4@>$YjEpb%4|2h-^)fBjy9pbWLj(RG#CWt%$BSMZ+C|&5@`kK@k z_lG@_0Pp+@T7xjG+JYKFYm~kgAhY~VD-5iLGc5%4QBi6=g0$|P7CKb{-@6R+fP3`z zVeoFY`*at{@fl%uP3=_0w`Lx&j5oP%sH>h1dqQ_OFJcX>x|d@fFjXFlexRxeU!8tH z>wSbuwwv;rUmpEx;sq7$08NI1mnpE^4FR3Vb_@p8KRpl9&)cJF?feDysOxP!YQt#``^Hdd@Sen> z-gi2Zxwg7PSwQ|3HT4FSd%sGUU10?&5Jabw*=*PyogdE9@oYFBe!$J2X*sLBSLygp zc&|+9#E0>DN9yYx;|-nUJ~X2e;$J(Pi%aIR-IMQI!i;mZ=R#E|3h&4M&8k8j|2~;$ zH@cAFk^6FVq1>v7$;^y3uIhjFYPRvtw5hVjwWacKrQOc1v)p9&y97vJ4NWl_^+6QR z+2`bx@ZVZ@=eSVNZhVHew)qHplaqsObIaX)1T|+twq}cO<&XOa)}l~edy&^P!Q$9_ z1wsiGT|APcnW#z^6H%ILFBcLe_|u z5+4iBgzmX>LJi=*CxC-y4(=4EH%Y(>&!jro7S0|5EGFrNGbfyGN*(^+=_RN0WH@#P zlc_3`tN;)AA)X&UE+jX`Fu@xKy~H^tZtfokWv*PyHr;0%kI{M>9 zUlV3;e>m9o_ZCN>zN`Me@}bV&D4ETi$w2&TE?xH=Skw8fqtMsxs5^6J$snDdJ49f< zOwu`Cxo?L@$1}n*CnM*Zm%BURmhZ?@xv%TuAB5~(qW9K{47Lu5&XdtB{i47@e9~xsK^#SG(C00F zA$w5ghmSA&!=KmhFuVHGuIVsq*o?$u7U3O)>Ph_T70=imL7f+7F=VcsVdZF8`@VB< zkWY}>j_ViSw*?L&0e9#EKJ@%H&@gtC!cQf-Ql(V(IW5Qk1evYnbq(wLv^#EfK0I1< z_GLDQvr|A9hZQIhQTd;NC0ehX*UaoDg2&KLH5|6FP-|E93WdR8Y0sj|~4Upn%$onkTmd}|?B>|oW znf{dL3{U4n^oyN)qB$t(?d)sujlSnO|MM2?(5tkUVmY2V|FfsH!~Y?5iRb~LgPaAS z6KsXChcVGVx4_L317sze}1{_EV^h-$jP*m=!l=rJrUV`6W^yZEm2?z&W z+kgnF0lO*$STEk2i+$gOmfdVf4U@WFP!*MI3B&Tb$Y9kk*VpLR_l zU*C&F#DxW3y}WCqyBf%=b}2uuk8<*992a2kLl8S}@T_vKZW#lhOH+>Pv4j znNmSj-}%?S{`G)^&Q=^oF{na6+9Fuq?fCz`Z(R@Kt%z6a;H~-BgHFI7U~IknOD>nK z{9BAG$y^h2DC7C>TrjCR@{p5d=%c~_KR-)_9pu0DIQ@AVkXS7|>==`IHyMqbS(-4# zTQ!a7`|*dDz0q$!s3f+_G@?L|K&EHmIGG*KlOw?*QNrTOe?ui8XB$3<$vC?-J16Qo zFIQ$!%Oio(7<5|T?@6gy&$C=9W!aPtM-My`^g2AhFt)PHj_31Jb>OeCZKyhu9p%&6 zH;4Q0@c&1@yk2~U|G%l7>Q|^SN{vITtJW||R^7{Clx+RTc1l!K@=KwVu7?1)H1r4< zH#P^sJ&BIE5(2OaB6TH@Qe_#{&n@N>+$=IJLM%h(m%GT216GQuT6A^|St^+MN@TXg zbYHlC{&!7^>;1zzv|esc7#?h&!2q(Ym3}mw#^ZUu-Rr%`n(tZz`ao@e8+^cWSpHo^ zz{-uEG0aEg%SG$hpAaLN>rf3RV8OuDHC1EZsux;QL#%ka3W~YpBwT=t;#j*pR^0o1 z9}y5hG&YHOwQNfnS7Khu%NjnS=Zo3={BZw=?{}QH$M}GVqtOTSY3kt15g*UVu-}I; z5o3NbcEF=8L`hL@Au8h!N=Cy^sdeMobTU5rTTNdYg}=Ps4SNsI#E!oz;A}&PsE}N3 zj#hCT;`u{yk6Of~88cfiEI*6yzukTH?CFyiuYP#?=G$*&uq_V|#m^o;-GBU;62l<$ z99F92ed;`&VRw*}$!c%Y_A1-p^K^fAch7nK?PKSgZ(r=c{@dRqE__c^<;AI5GN+@( z(S7vh)HyrG!E=^6oJ+=u)1Tl%>?h}Cytt$Q_6dQ%8UGHJCLr1s(+{q+%XxUBb_7JO zX8j2|t^{{Bnt{EJVJSb%c2##++Y8Ro@qGCgF|9dT~P63P+g z(WA$YpNOwHX9!dV>bw-dKK~h|h~C30>tk-U$XmD;A0a^kZG0c0a0IpqkS6u__QS~t zo9Xx_;-aqnv};;`>YI-8@<3CF5z1v?|Ds-=*Kmh@w-`GZs6FDp@$!^GmX?=C3@EN2 zvF$|Awy^hDlDJaJcx96Kj}^VS?y9~GBM$ub_-?Y@uhBnKSd*5AeJ}OUG~Ju|b*|~W zX`yccZ(z=2`QX9m7;|1PWXf~ZRn)Asd#wL~Sy29R-s*p!>VJOG|Gd)wJktLZva|0C z1j82Xg*r!xA=-DR=~;3gFh)UNh}g}4PkU5eD(h7}z#GuNXwpRWyJVH;e*L>~J<=43 z>Uz}H57da}45+Q4k7?PS$Pn36a@ViI=C6;^9fiYtQfU z(0*GOHrDGFg9W-CqaJqrnrBQvGMvh7;zewT1oStkM=j)DMLl{#j(S1CzzNO%8HGDu z$>!>bxW1}?Y@;4^vG8Ls&Q+T`rXIy?l+gMEhX}oP8(-PM_D}P7VL1ELSw9NKzw*g)H;CsU9gCo}f<6C=M8?T{#H(?Z6jC>de$_pT}Hxc zvxce>f4b;fo+n(bxYXUCI8K@RR<&V6_BKB@)3+W@aIEx331?GrdSpgh(yAS7kDm2{ z=k3#WOMs9|G$n7NDP@fPmWE~;8R{?>j}P6CV2XlUkEA~)MMj-gXF?R?;@3H{-00^^LB@Cp%~ z2A%$iLOgR^Jei1dcD*thA{ilnte7zHx^uR$;VT_J!Mc@XX5KncL`5`_RceMI!$x1EQNTis7LAfm7Q zv}=m!hFwU@=yoI!d$kSRUJ#4k)vkiAu>5Zkcn;jgrv!B?aLK(L@h4ZvT~xl9iVrSY z4>@uJ&5D-bz6(wYB6qjqyQr2jJqCyYa7u{(>?#n3dS1VC zcOQGRSMg`=P1Sbp{@!G653$&&*qTC*f2%mC%WqBAcDJ%ka}!%pQSVJRh}QsA3KqdS zJRhE>(-VB~M^D3G2veb7K#hoE_*W>GZWGNmDk4A!!)KcmpJQ`d}M>1;2<7 zh;L}z1MFP_#4Ni^X}7C8#tNg}GXGcR9$UZXtl=bP3fPT3h``vvWHg$b5n-R7*Sw8A zXaeBFLT%(OkWEa}jm9)x*+NqQXD;S($?a-IK2QJ3PutjoR9f6(pPn6Rol0-;1$CdL zXl?94jdq;Epmg!VRaFU%X>6yaI4*ZE>8cxh&?kw0%P3LJ1K-9TRI^%cR+nj(>`f=9 za=tVV>MIrvE-}u`oa-*KbP~&0PylZ1K_+4IpKYr)Cvb&HhSSmU;-s6+v=?ccw)(d) z9Oa8=V-G695av`@tT_S$Y4kDNkUdC!=Z4ct{jFk!^J5WLs-_?=T{~**7TbX!o$_3r z%d%<+s?rCWSkN_tPQ~U^2Rj+Z_j(~~qjcR7V-;L8#_IaC)a(RH!>^*w=Z9bU#Mc9c zV&R##F2KQxqAxEMbB(67R3V%-@0h5AF1BoVH!FR7mbC8aK~QwqfR zLtfDGim$E+MptgrZK$fgra}jqjt&V;6ArsZ z*p;?5+Muc>&jjYodQe^Ecx zLjqkve?9tk{S{<5Z3Vk-Yny5^CJ=5undY1xpONCeAs3i9)olP*Cd6!%weJf2tBT+9 zeLNyo0GhzJ6k?JHl@2a@vXc~uNl#!cw+>-lLo8PVu?)wH`pnmmr)!l-Y7Na_A`RH1 zUGIrZM{e=TquA~=oEu^=KY%T!9gr!h6%rA1upNHbnBVndf*b z)iAAqX=N>T; zg`v&(u8M1nwxJJNt5U6uKpzauMVl>mnJ62J|2#P+B5C&|&|;}-wvuS$Icsvu9YF4^HL*hF5s22O;82^eD4BHfBYZobecjgC{b?26j6 zk_uAk$eh5Sf)u_!lR!Gxpp8=Is5Hg4VS@Co%mlgUoc{8hFp4)%pY2Vq4^{3qwbSg_ zY*Y}JxXrD)m&0+Ix{HN*tsXxv6-RMRms8}TqXBEyXzj2k*mBZ2#SZqt*aZ}u0xIhv+g68*IIeub3-hV4gEW0h^=&ZN#@mdT^jBDka- z#*nSm)-h#$S`S~CJa02t9m+; zHnVIf4()8FxKZ7=MaHdZUt#}f!gh!~9L~~@i)7@mZP0^sy5&4y%oF%E!CxFZaLSM8 z54y<=_V=Va9I5oATh1N~;ZkXI?)1~+^Zrz@|69&~I$yne{^}(^Vd&)F7kl>|bbDF4 zDv4V%lP672?+ZucIm_iZJ#kJJ)#JQ|T3ZakUqrSgmOU-WjZNBsy*q;k$z z+OvJmZ*+qRxk1$l=82wb9rNEZOyUG1*%rtA5Sz1W8=P;Q6KsQ7zfsJ3D-NPoC+HAb z9tIq|!-MVf{bwKE?Ed=V1`gg=e%dt;-VM8+9Kkz*8<1=7?$R*UVgob7!Rq`F60`h= z3-j~8InE4j-+flQS)Z5rOX+0|F+w-+UUjTEx1k4&8dZU>}S zB$881<+3jYI5~l~_)*&xd}^V@s!hu(WK|Ubd43eMNWMT8!Qx`(tSw9AbkvM`bNvo` zyUxcRY#HY%0xqEvJVt$e?F?RY z$szN8_D`O-N}1l^#Wb(ZW;c#|dyO$;i5QJb5pt zsBe=`$rOiISt2uFfyNGB>>XB==$DIh%K7WnuqOmuD|~~nU<$Wqe}EgS_`OBaxPpnR z=eKEMt@^4mY*ZB1PQ>Wg3M-sCV&Sm;QUK%lENd!j2kUH;(h9S_+j4v|NJbWNOYo1k zPHA=3fAlQ(bbJZOt#G&F1$)LzFF|hA1}Cp)@s!tU$>vy)dDcq#OdEL0E2L!C@lAEe z#0YeY6mBhu`F!FiVELN84SMG%i?wUWSAW{^edpnD-aB^Y$q0qP*&_>{vwu9CZ#j=9Jy6!#;DrLway5&${2UDW>S#CHa(+(E z>Gk{JY{(0Y$S*V5avuJl-T(8_PyyPdb|81|sopvNA@(xqpR&EGzQ0W7)0C8cy~)YJ zcJ{{3|1MvBt!hLVOTO5O9^ti74=Q>@<}a=sP{d(}K^Dg}HCQ^`G`Dxdb?F}282ze;P zJ2ChgwewVjymn{Ya`=e_r7qyvEs#T?H`}BIM>%85s8-WLL`b%=Ay{=839Ir-c(%Xe zd&>^`)rKu;)AZO(X?RMyCjg9czMtc9fS}AdO($|hCwncNHh3`l0&yI_J|4~-{{M%c z_nhhQ=y;AT;0(EtoRe^6%Xx}x&6#pV*G-+T9+GPJ{WJ24&z_Tb7Wc-<(Jq%&*PB5-bY>S{zR%Ac)9gaY`f8zek(5QxrD)g<8V zkE^PL@}r?j;6C%+orsJbMm94E_`7wgMD4`XeFeoE4&D!$H&k5uihw?QiHkTj7H> z;J5DmPD@cCAmFZ>3BJ{eV_+~OVqudHgyh|@2cV(ZrPlMk04@gg8cDwoyks&?Mr5Jy z50k^;crX#A@dVDv`S4TH9pS%HOA@D2!I8BB2rnkh`PrdLo^hxku`GB|RPFwNs9hXv z4^NQbzx|dx_$7YVI{R+=^P8u=Z?BFlUfVNk`i27VG_DF=wMO1$)xDf|bpKf3M&l>x zbP7I4i|TUB+ODLOj3f&6U|WqUWz6RM{8VVO)D`~JZTI6R@w)}(q zlETH*9imo=CzXtQ2sVr$b}zGWIXRUbbzME9Q6ie#?nkUs2ix!7zJC7f$-sYl{Pf*7 zip*?#(Z|AC`7fC`*YFnE%F^OM!Zf3wNVNG#1#WGB5Ig_Fk71Cx(i4Lso{N?xJksYnQW z9Zqj@ipc;gJUxS4ttmam4cJRd>#(9mRvVu2@z$%Er+79;qb`d$YqKyY2R=~8Fbc&i zs2`UVWvvwD_kNc_l5g{89vJoK~ z6?R=EL5bpx2+6m8ts@qGNNosSuLwbgSTMsdR>IUE{GG}HL~8?jyndouX> z5UvWqxjrjiKD$NY&P${dc^&M1d*n{{fA}8tBp=r0;ZwgV`|qhT|5i>lHJ3#;ury|& zJ}XUV`@Vo1YsLp{P`4H=M;lq5#)8F&U0bk%ilAOT+WCSdt2b!UW$Ts6Ldwg^mEQo< zadsZd2`RgD^_jc?YFeAFLVvS|vf})yY!G%`CUN5_va`Ti8^^{|Bo-?J`(pZU8UQMF zL#KI~sF$+-HlCvOkqvG^Hl89g*Y(Czgkhl&!=`a=<0&#YJt``q>DT@&fWyX9RCVKT zzy=$z!Rf|Rbhp3;xp6wToo_rvB?*Y$;w`5}&DE5-HN@K5hYLYYeI9_Vji;z7%H$?I zMaqq7_=)OnO|DcuL|ifzCc~R4KB1-Fp=!FWGWu|3w5Im=)Pg)s3wvV-Xt|^Se0tYR zzIB0f3Mk_83|nxGKsB_$kgcQxtnhO=P(mO2mHf|o8Z$3(*&mZ zIOszV&lOMT-vhTxRxN(Q3Lm%w7~cc8r>^uNgb!S~BXIj9d7zYFl)z14;mCjQ3OBsn zM1Gqp=<`(WQ1^A=hUfQ|Jhe3lAcUt~Ro+ui?Mgxo9pME&fk*G$zn(^ku~tFC8TlIn>|DWp++5>YV7uJ~K3aoxRG zbyb>;S^*XGCl?Onvkcr8KRfwI^0+wm^oJwA6Zs(fNVn)%lme2^_m9?q>v7BP@g2r~ zN46Aiz?L%@NN>UcTWwaA?@)cR@~~FYp$Wc2ezxKkO(2^ige3+$&!pVcDvudQkNyqu&x-G<70%ubvh)9-)vT#D0(KWLA_Bf^&;D)%Puv zBRfp0Dz!Wr5Rzc(p%Qy~1*U3%WHh3l)Xbk^uJCN=)imgjG*(k()-m^Fd84Q+GN9d;39G&l zpYK}ZJ;@e(RFM4yrJRO$+aNkOdBFP+X zU8Ls|^&dUU{j* z_|GUs_BD`CX`Ef&Lg81>P=J@qixRZqZAJ-nTW$c0a|}IeYwx?mAC5-vZXnR}%1^tl z`@T>Gh;@?qPFF&=8PEy?MZtYvxqjOcqg=>7rD>N(>>!RC?L=T-lShx#%ZujQ zY0+2i%d7%t&aR{vW$Q$>@G>pk?I(6NR(jhfnJK&Y^AbdR=*y(Y%>g61#A zxcX1_{Kw`?=elo2NA}tb&tu~y@_Lqz>__l%3)M|NgqDsRtN)DBo6?aBW!4C3_D&&NOapRZj-zWT$it0Cv&Kon$%q;T)JEk?7V2p3Dea{aa`$h8=x%$8Z; zX&PazvZrpe+KpB#qV+mjEf!I27E?E0-s;ceP&#a1lV!|Pa=zYn-0{#PT76$>AlhV> z8JCrkB4$LbNoW&cI~5ZvDvUDxDT}F6iKwHlL`pqtvU=8d`+i%z`q_;}-*wMrlUT;Z z6QHYK#}O+kei?9?QQMf@2Vu& zYnA{{AAQiygUS4#l<5o$Dv$`ey=q013IuBgXB7mN^+LWTO@v9wL*M;BYnV< z^;49tYyG@_)85@ZJ%9e;GuKbGojbmM%&ErmxGXm+XO&BH&X|kB^CbDWEDB|;udW~Y zG8ZS}Nj$r^gF8uAg@C8NelE}5ve}>PgovB{NiDC<{-m?~DApZoP2?)XZIo|!Xn*>> zLJXodv6bjKEH2kU&n=_(uxz-OQ$%W*%S=2QZV5xj)C=Sqpv~5O58LtY=zR)?^0}lC z*QEFOHEE>C$d0ELN5;i-M~&;uUBeJmk&n4@$iRPwaacK-a>Ih@TFS3kiWMM8{{}#i zj5wOZQV=Bk^q-A2gbg0jPT|%m@sTCgFfWyEV1K|JD_FyQOp4^#cr`g0r{_D)^K;3n zF`bjW_BbVdA=x*C{5oT2pA4LD$*bD4)6LgVLa(rsCG&DQZ&EiIL=p3SVnnUvRH-AeI$!CryCtZJ4*rf*Es~HhB+K& z_TgY?To_D7qsbW=1@gE9;hnOPGah&t+bxi7XSP>BGAvYG0>XA>VP>98hosN>;P@TM zGEB}k&tONtm6(}57sSUSJ(yUz>u?U^@UC>G%lHF4u8~${{lV zqdMe@BFR@Afc7-De=$8)N*S+A=>G{4UJLh{?e_TWFcwhO8)XT?K-=sr|M$wU>ozbl*(947i0LUMN-@y z^n^<)U;q^4ZHnT^W$vv&n0hJinJLYglIJ{USRAeD$&g!kb4wAV)R6xB$!P98OWvo> z-f(_?-+A+z^3vF+cN}wkO6o0f$pPO=3N?aqf!=*RPGfK zu}YVGGf$4UXyaPaU(abx_x%r;dmpdE9urUpDpdye0B zym%*!F#{KB!H;dqoYX4NhLqQUhwWELcFTv>6tVoxgLv zq|lVEf}M5SEsu1~5vF4RrV;4?54NLMhaZ1={4rU-9&q)iT~iML3S8a(6h@#8KmbXo zwwp?Jw`??Txkhk3jKH5D#??Vx?bUmGFP(&}nUk?2bX-nmHQjPv=ZUgTNVVp0{(G;J zcLK3;@S_dc_&EpyX=2+aNBbVB-w?`@2X5vZk;qNXgUJ!10P_GzIBP1y-rn1{ryqvt z*&CVfW_Pb2wy`W-=+l$$Qp1jtByXR*mp^{ zC~hl-Zpn|{?UsbC0Jmc-()^+a{{1s1%TghiPS$EGZKJZBudO?7=1s?U!pvLMJy)u& z>ikm|2Il#vUdnb+u5>KCI-52|3F5A7;}%DXri-b*Rp&cMZu~-_lU5I zH3Yf`ew&;&F@0z3 zoWDOx2P6d*!Eibu$x!N?q}T%;gt~S4?@movv?eFNzU;sI;h~)PR-G@kGfy8p&MqAMlWf=`nt%KK_{r1r=g&W^|6qCbr(O48 zneS@KzU*T<4^@zB<=9xL0F~>v&A`0uIQhr;^p~BgELpr%Q6HZJXOLW2CkNYv=8*^v zdYEkpAGVW8@4+lNpFQv%MBN9}>p$o&`ef^TfNOAe{GdPVkLM4LCJzS5^nunPbk=9< zM6|N~h9gnizYl`*GAE+TX^+gmJ#=op(6~XNO$pXu=`Co1Ra`ePt{7>gj8gwNuW=0(Ge=X)n6-Zx+E%&ywGq z55vUyx_3G`Nsf|F!!g<1^G12@D*a9S2WIpKEppNuCL^_kmg#K!s-m;6YOeGR+#Pkz zC-0vhkR6b7ccFim7a71zVZ#bPJD)jEQK{>3le#c3Gt{z&1f(US$uzxR+*wh`#2~Uz zZ8P>;LpciB`q-P(zB2qLtc&4zHat2;<42w8E@sU2Of-7VRFhL#B{Kyf8K-9;P{cn_ z>GWDvH&QCrG&e8cL$EbBVk-1EZhHj$ph0uPOUYGBag#uJ2^91~#|w6%aK{Tu#ZBcS zo%aqWgTX8Xn4`fJa*{m!Bv6O!!cjV@1pwY$q~M;+Of|P&Xm*#J zl)PZjCW_1MWKpo9Yw*dn)?ius~ z-)@EQY#N_Kd6DtXHjv$uY2xfnrU~|JuG+>?F3u9%V|l5mg->Q<&Pbv`f18p_9Fz@? zMp{t1cmpIiQzX|3pyj)*TRPbc@K)FgvuM?IZVfjM!Ul~w4#aU_Y0UeJF}v(ksH41S zCk%HWqWiyMF@urLi{zzmoyNSu(gGmPZI~4h&E3M%dJzsdHg69h1(9Y)Or0-(IeT+> z`XXMx#(edsT~lK&HzU!Qk%}F+r4UdxjrmSsY5!bPCo)p01WO~h8u&?Afd}CReJ7C@n$6Gq1)LaO6Mq-@V`6{V5m=L9ih5iv}9t_mwO*C0js4JEZ|g z+uy`)d$8T8U+Ahewg9W{Qcd4NZ&=c?LK$GILu;lCFoQ319)PHgO{f~jZH9cee54=| z-;fa3D_Q~HD{B7!`&Ht-H|tA~-`a(+SziUAz=iAUGT@hMo;FX#$G;pG6Rl4F(D6Ny zj~sYlzLR64xoOr5MKP3Q>vAgR%+v|mFl2e{I1W837lj!>re0&^agJtIpESwI0w-4| z7Ve@DM=1~r<1ElnYIfIieMnHDkpZ?t(h5-iwNqz9mDL?ax~m)(%Z|n%+S{!t4r8Gr z!Mp1TJ#^tfXehNi?6yMY#`XOm#52r$R|lArLAA_>#-#0y15+c zg;sYtvy0K8KB{@as;*(w$R^)9rkD)3ATl~T98mIOiZ5qu1y?JhaQ!e~QCRk&@+{Ai z(WTs%o#mmcYzwJrn6?!|qoOVkB)tYaiqJ^LT_I)<2SQyLM*)mat!|s^VJgJ?gMri! z(*QsYSJZV9RKO8WbQrlHAjIU3FqPC5nj=5*6KR!?uDn~|JgjKA9!b8zb;yng*lh(W z=>}R1*Huc2FqNDf?Vbw84=BD!rTzOqQ|nqBGRQ8N49Z%w}a$l6_ISNbf{+;tn4?oQwrkzR0;f`=e0 zN94jsVGEU7DKn5+nq>MUEl)dW%Y-6S-hS5nDq5fd{ zd`_)Y@S@n{Z0#m7cp(+WE zUaIUC@;$2WBm3_AlA=O?E9wJ~1N-(3qGfUg6+4jL#o|Z^_{W!Lw*H?snDhvI}YI zUWAa$LQf5z(KoZ@XwYKcj=RFgi&W)Z!a-oXg=7k8sq_}%K++|3TC#VU{fxUZ?|C0o zjv!UP1!}QZ5ugg=+zO@ahp|`$B<(DzXHtV^XJ9E3$?VB>vd?Qmf$T%e3fg(J#_HvRU;v5cvUuFhxpsY&EQ9`__S*pRpHgCif0v=NN$fPP(;yxE$iw<+{L-RUh3D; z=De~wuV@7Sf$a-)JY}YwHl(7qw!bC{Diy6zu36V|MAnScPezbIh554J6QZLg6-KX z4?ljMd!^P+vwNj+)huzZSamPwUNQ7dSeCSohcK&~uh#5cF`Xvp!pdo)NlL5k71Ixc zNm#pA1j1XbXt`I2S6LIE?q-RZagv)QR(M!0T4KLnnG%&noqJ{SaCt5*kp^sCYy*-P zei^RPvd~>_J7}at)gd89v>21hz*VY^^$w=J3TI^-b7cr1AoEumO1 zHM_>!1f-HBnz>DW1JX57mYwi*Lp5ZN0{-&WZzL_2^29O0qkUhQ}vbKTuTd#iZ8@&CY@VVNO&& zDZ7tF!K$uFilNnA!_Ou2PCBr57?H@{4(zHGnq4FQQ3jBOV)rb+zM^A3R$aal;jvvd z!3-Aw>8R7;LvhLVD9*TI%<6?^$5_fO`M#90bNl)(g$< zl2KLkV{t;-oh%Agb&4ejs%lSM%K(%m^;$8@4W(x1@b*Jx!A!LANJn8n zv~jhY4OLckm{fTebyyT!ww;(0p$8Pzx4)Nqsnua0KVgTh2Uohwoho?D9#}xogcnRQ5J99icoL4=e6iptjE>a{&5nhLCb|~Z=C~+W z)isyZyn4(K&A>g_RuL8qY7okNGXVBD}S6UEPm z$WF}1h||>QT)oij9Q$G$-%OoJrzlw2HCQ=Uj~gQfs#r$PV`D_rFhRw*)eFt8p>Px3 z#&=O0w!C0f*EpSA@YVX9yJNhA5u1uA46z9Wsu!AFBkqpS8YF17Yem7Tu8Bur^{CN# zf))|aY$p)}6c#m7z0m9$pLHDg$j55eih@;L3z1-}yGCDH?7uDcEcDf=I%?HIyK6X? z31z5la3Od%Rj{IKh@{{GjRr1gW-$Q1!I~Bfoh2h`g=W`Sq;cG4Qmdm@6s+hP?9;&o zUntNmiAD`1ktT3ABC<^7rJ>a99ASifH!xeqB)A1J4a+8%YuEp_I;EkD>^3_%4}P)oYGgt z>k%N)mXJkjx*0TW1<6b4c!t;e`uoF2)8r(1I_^*4Og%1>8X@ z>`NZNyAg4xjlmoO&|hGjh>5CXY{rLV^%$KW&L<#illfu>05`+fHr&8Sn0I7r@7|8Y zVv1HPQqgG9UYkXwN-H*?$(RP}Nehx%O@T2XG=VRjev>~JG!-Xzb8_pY+@Haz&L~gJ5r_1-3M!cRbWlJu$bYmkfu1TGOECDy|O(f@1d_^m4|Cp^YYFb<3RTT-KKv)HQ z%M?=jRr69_HdiEGeDnRo$GfkbugAz#GGCBns}I4p(4)?s^U31xeFy$CCvn~nd+!~n zuOtQ6{FvNIy$MuJ!kF5fOx|xf&nBZ9`r!PR^Zn6o@3$vHcGIYpVEYCK`7u0_&c;!&{`??XCLO?cl>TYWA8grzBzH(;O;+MPm%l?6-7mS_m@yqq*@ho zs&p6YqZB^A_yD7PKHH*6(|B^Lep)s4W&jm z-xfuhv70`{1WOO?utxngrz#(lGvZfPFqkSYz7NvTQ0b4V`88-43vv{)V|wN%7V* za;zyza5YWkc{CRIx||S!^YE;|IE-yvg)mSn#k^VUC*UoDuMH;T;+V3O76ps!+7VKE zEjGHtE(^DhPhNP6kcaS{_Q}-?^{&A+3r-FM3#(l#3Kn#Y47aXy%>d=A?vz0gp)}UC zfh(8f!sMQpqT6bxOlhpKG$XmvbTZq8t-($mL?zlOMZ?>v&wpJ+5ZX-p$8khb;x}Rz zZH`dH5nE0}U}JuQhmZ~Y2CE;}a2lP_QK0=XMYqh2PR?bpzb8I*V)S^=CpN)FpKEDZyPR$>${FW=b`~WGi0*^=@DLNO%yLsaqNKl$+=ujz`>S4~)Rg1$ z!nsuLNxfLN@#R4OKExQm9p)D?BQm*LvAI5yR$&F$sI(=}cEvaaFXxr4Q{aHq%~Pa`33IQp7QRiU)-xE# zG2M_3yt^B;N=I6RJK7qkz9|D$C34Jgy1^g?=m^s?@@heD5*#ZZoSmbc-+t=6|KQ$> zufdVF^3~RiiLOtx6cQb~9tbpkqeT0y5S%P_AByBalmg@P)|0op$ji6uM#bl?t3gG8 z=ESiBOov&=;Im;!R7+{1le8-|E!F|$7Te}PeT!{#q^uUquYzp}s@6P=N%}c@?0o#t zCMWN@D0Z{i7jKW^6R>LtnlPeWz*B`*Y{l?fE5##LsB>@JIZgWgK0R~;juty)lw28! zJ~~Z@{Z2AKTFw(>=}FnE{2#d3PWxSWm64YR`5Gt4ZEXF=divPfwY1A`7Mmfn5zH7B zcm{T16`m1>tI3YTd<|3_c)Cc51Ix96~4{j48QCC_hXLsVj79 zo+g3;Qm^Litw|a*?$`2XcvCbH7UivKLpL2%S>sa{GLHP=6f{TpW>$`Fo42h2riSQe#ua7Mk#0uLW3tjiP}DCjp15daU; z2*NxOEM+oYYo(PAyYP%Ukk10>!UrGKf~JV|QmL|_!w5iJ<}HPkEyDKU--Kv~A3}dY zj~1;L>fLprs(~OmoK$_RWyylhxrm!x5?+WTX%I-oUJ&t9EYhwMfa-;M=McI`N(4U6 zPy<4}@}git*C3Y=EekJ%(;GqUHdrbYe-52}ZD0Z_R|@s6LE|1QTHt9sYO`-;$%4)y z+8|!?+z=v$Fmj6ih1Mj+!*fVc;Z3fW>fJ-=G)$FnJorV5@*4u_sy}P(WNGgyd4s?bPw_8u_Sg*0Dy8*bFi=QpYo#3;? zqd^`L1c-+mnNUtewNkyiY;*vRheTAMX<#GLy!v<>sx0U*sUa+zj1a_*C`{-SKnn#^ zC*q#<=UXo|I!sC@7~ELqV@OoM6<-?GD>M&-{B^@S30^)QcR3$<}QZuxpUTAa|V+k}} zHhh|s6!Mg+l0}`vl)a>LkoABTxsTC?5Wkpo$+d@BFVs8d!VOMiwfX?_?CYcsoSjFhT*KDqD_fDLSs) zlZVM4mUC+TK1;21W=%88N+;(jI2wqB&kzDXIkVsu_ya7PVSjX*Se^7!LSgZX+(W{L z9cFEWz}?ry1(zk5{gAe;z4LPzMr?3*c8F49dnt8xJR5bAXfmGDxP`_;`AGo?`1e%0~1KK!lj49sM1^k<{EJEQVW( zE+Vr^J}NrdXgcmg6(YmUrK6-h&~6)cn9_4r@Sbzxh!)MAM{ZG0T${*H4RwlGCSA?A z62!!Hz(v9Pgpor1FVcYQv`_Az^m|JhULGX1R{P6yB-CzuPU6|Q^g%LV9gVHBlg*ydZRmEtKY0P?aSsr}1QzhlQFu&g1D zz46I?9GMF;UMG`eJV>l|UxIZu$@L{2h6m>MEspUIo$js%u;BX=cERDe?rnEQF|2SoU_=n+*nd;+j;8u@kQEvSAU1-33ll zIFezYhw6%M{c0%18&k`khDIbth_~OA#5Io>qJ)p?t#%#S1RI^XQy?3ZXsaYBnB!5_ zzFfE;e?6O$c_Z~QYm)R5Y~aZT)0vH@sWlrW6D#=y;>K*8OzxMksSCFB=mmECi8TU< zvPOxu|AL0#CKp)rUoxY-An6;DcO)PPprB z*V~hEHXC)1m+zx*r|`?NZ8Qw)@Z2Jk7vLHw4sf0y?fmYx;raJF}_fA})}5>1nac7#G!)ig2ow>buosx?g$qwdvMZpC6`N}5#g%9SKduC7yJ zr0nv@T>|kF{$Epl1WQLUCYqZx7#W-82aa=9IAkbZUB&7d08!I;@0cpOHq{z zw{pW4v|6cXulAMP`4hTzcZz5m{i|lQvmpSr9sjg47W3V9{F?@_zk2-V>)WbJv9iD~ zTZtwuS*|0!*l7j|7C2l+!Y|EfmY0$lN1SFyGtW+;UE|pa;)y)F0~HqJ1q4SU zih7773Lrs2M_RB>d3K0cB-p8;yn?^5W8paJz!Qy9q?m(r4C>%_;PhbGLiQCog5St` z$?doJEFV!$s%EHI#1(ns#E^t2^?r+W(8#S`+=4vqvpbwz$oL8TA48*VM& zv(!`MUbS!}@Y;MYs)u})a2e%kkxO#35%FqF8y3=>;*`4u8w+KTsk4}hg=%imyfY1B zXoF=I#9lBok@?3#f+}zvXQL?oI- zOfUW`9|?apzI-ZZ6A9O}AuNN-XaPeN4sb7H zfqU5DYmu9P#uojR-!dsmpSwJD^40w|>V$22BOvOc4}MRcp-l0Kywe3&kL?8Kg6%Kw ztJXtlNB?xX+*=tT<;R3K%kMGdF`)nU<|*Y{FJ%GtO-O$+KFf*Sy{d1llO61Uy(`jc0XxP!L6>uK^4bURzGe@RpP z2IS|LdLeA&FhkHJ82d<|jUW&g>Rzt%tA1O#p{opK=5*7oPXDD zj?LN7U_{cy+p%}Q%HCzs2IV(`p7Tk40U{bUus?w%!w3j8_a|IR-6%SRD?vCh%OBLWdX<+&P5q`iBAqFv(g9JERl2keUqRgo zmgrpIt3m96yQJ<1ODNr^McXR$0~jsWhfsIHP3hXYBCDy3cN)?YTwh8FeV|SLE8TGU zU5ToMekZv+1=W|k(w~Sf9V&OO^xHb}KDpD5G%atB-=xRKuX1_msK@6j@l)wEKOOa$ z0b0;G0GQ&P%S)Z7mOQ`L9Q^PlqP^ei<_@We5U~xGsm_lo4`Q;=JqGxMyB(*CF#fZ7n@fvAsVW2%v1 z%F~F{k(o;kM;?MQ;M5VWDc>LMh%ik_3rqHw`4=r8gnV~O*@XwSTw5jtqX*o&e9Q8M z>eHY4y7?1P->=NUGRw+zB@>5gnlGkLC8FOex2VLo<$>{+C{u>~l_|z^t$$eQ=gewP zCszLqT%J*9YF#9kpYUfiN*siZrO9MAPA!BFU8GhrnT|87)BkKuda*VAJheuU8>B+Y zj5a!Hj&29PYIcs-xZWoFJq_{;#2)5rnkAe{gU}>qmYq=5*#EN*@2Gl{N<7a8TM9WHOf1^meoTib>yD9LUzGjct*6W}T(E?3T? z@gRRt-4tNHe?IMJBg8!)-Jw3nRSYny?TLNa5K7t?>57hL4M^N@+R+{B(ffBi2HAIa zl4bW#21h&o#pR3N4`{;NC<28{6xo=@h>PnMCTb`(R3Rwd5E+b2Q#SNeu*<4KP-qK; zINv~^#Hrs;JB&a9Kb>9Qzu5@17R23z6jzRfUkniofOVY+RN;{v7On+BYY!)fss;qD z)iNMxt+W|}R=Qg}BK^@X$Dj}w1O5y`lD(jS_@kYX{qb<`=`WA31B0%5wRJIQF?+Jb z9Q+9qcBCch+lqrPNM_TEG@TMM?I9NHcnchhrD$v9y{$%GnB4@>2xwp^BQ|CM8Oo-m z*}Bxmh#`&`3MGS;wj}!A%(l`L_Z|eBB%#(OIgFm>T(9T`5|niV<_4 z@@Ht*YVjMGXP4LW-Whg$t)qr%T`nVrKCyA~qfrk_ZbO8sbyKBoL{&ygZUU}z%#A@% zL-vQ!;ji6?Q^#jVV}Ei!NG^F|Io6PU?80}Z6I@S|$>}sXq5RD#0cj`yMPj)_s+o<{ zA6!y-v|rFj~(cKYxiidV5Af` zt9C>WAV~QKX=2)4>%m~0b;tdaSvvXSv(*{&r@(A>&l~qA|D}BjhB4C(?|Kxqd^3!h z>Pwho&6Ed0YRC{phW0)fa)gqOWxASX%AJ7-D&dOXX*hw0NqI&=J9i-S>VX_hdiidE zz;9L;)7n-w^Ooz?D=nL}fMJq=7n+TKpFz_SHK{d%6ddeTZ%cps^-`Q)d|Zl*(2arQ z^WQB{0rKB*No458+Hs=}KzClh6!wZeVpc4hd74f)ux$9&?Oi)4Et^}O zZZeoav9!_J_-(3%I1y%y7`X*JhxDN-Tw&+;=cn2JkI6OgbXULHn#RV4$&jp$A+kjv z^n%8D&FUy_sLJ7+=J>wxvF_z!Ue58|WHe1CJVS%g*zF>304)y)5+*Ht7cxmS1mT5= zi6T_4ef3g0Z`4IPB(hgP>B~V>Z#CLphY)2S`qBQC!vnZVLKCx$_C)s66Mhiz_ZYHV7>vtMSswE0irxwl;Um`JR{}N zPyJ7Qyu;J`Rb2JJcP)uTqT`2Z6 z%^>Ii7UXO}sbgGT%!O>Gl&&Dhzb|Q%>ll_p z?upn874J~k&8pLEp;S*%nG5Ome6ghR<5MnUts#SWDeh*rAJ(m>`6ancv6#wL_^-4q z%td)6>LqD*Q{w@FTTPwgXOI)X`E6*=TNyO@h|nQbP5t@F)8$S(I1c4!z74rC8A!@+ zEv9C!pA*yNj(HeJ@6;uwQNBxMfJG-LQ8Nbhx|;+@#f`Im7vAo1I{4QSwA@9bojvT( z)_-@=CuJC`myNAGRTq=Zx1hd{d{7SZiukaY6jQTX9K7L_o@^51$T3Te!bQ)p-Y+o*%fBP3J1Bz=WN z&EUVvf6VJOePL!30Y1s130h2r$(>R>ZT_3W&GX+BHlF{cSjXZw>QKu*5_D)s5s#SH z(4P+^H?YL228zP}b=TG$uP=IOU#XKvWc1TQmqkB;b z`mC^QB8Zz*h~ACqCZMW7(WX_JO$5PQpgV(RLqVfNvvI)eM$U0(Hz-+}YZ1Mb_2v4c zq955HD(~ZI=Uc?@foJn=?tGz>*@B%iMjA87WV%SBb+q%#vp31W|Cn3{&1TiBt-13p zmuA}V#*QBcSido1D?8h7QI0y?bz8bY%V2uD(4wZ)8wfe^!%6}z+W#1(G@s@oGC9)S zZgmu$RqYP2BIS(d?N$eT6yS$En)0h`b!badqllcp@lVE+SIOQ-4B?xy+qpFiZ<6am z<~O7pjgYGVi3H39aP?9=WVQYInQbJ1hs8+Xy|oeIoLF7j#9%^^jjU{HZN&7Q39ck` zpOOL$pdSBrDb};#I`P)I9?dH|cYP162Q{i4J&#Vc$Cux{yH}x?yjy1d0&@dbvHak6 zwH@3;Nba9yXLqf4rwQz9P~%*%k$Y-gWTU@Nt9jjI+^+g$!ZxPZ;DgE3uGDZvsyC#c8IRnVh1=?A+@2tfznflv?c+4e+^XcjIUn zAc6du6+p}h$<4^eM%uX>5`U(Ohe|nSQ1KVwN4l?hBAWYtT}@!1dpLl9ON%&NB;?D--?_=#F*vux}0$cD#y~U z%;OsxK!1I{VSV45Ke?O2H(Wwz!;6?V@2>cQ6tRZX?^bz17*ADRWz{8JN73Kdra?$x ztM8pkXFm9j!cpCd0`@zdE^A2Bp138(fh_r}Wtv(diDoGL+Dtd$VRh1kds zxSZ|KMM2`k--tq_q%%=hG(}r{gy@eLJaBtKdloK82wQs=&W5@zmTUawVxoE}8nW8qEy6Q}BjeL!GGIBSLlyv!OavifD z>R)_jSu)KizkAZRei$NGHUgx^Nf%k-`T2_WujtK^XEgtW<(WaR7Xv3uiNoNhHu@4n zQic?213K=)X7|I}w{%ur7dPZyw%Y1$fciv)99Q7}$-9X?noKqsR$ot3NBXX&X`_*L z3f5GEWurwY?hM1mTNpK-%?4!Hu%lhi-NjSb8aBS)9CbVcQ;4G*;yZwi>%_nR&f$!XR7yC9n4?faj;2{k=zthY9ZeMENpM)Dw8VNup1kaz4X}n>IV&^5X@wyS+G2i$PbW_$3Z(afT+Q`F00JqhnBnW;=j?Rd)lmy+L&kbNckg+b` zsC>cNw=0vReCuk~<2vfwRbM<^JR*SJOKskcM4n{MAvR#(1qt`UrueeZMG|xL327nl ztzzqiK827opkqkAfiEcR+6>D9>Ct=?Q!*1|shTqs zB3jiQgQqN>rU;iuA&*sODDsXY!%bz=K;F<7@DRH#i!>agT|X{24P@h8HKl~m5U!6L z_|V-y+}}gTzBO{7Jh=R>3=OV_4@ntIyXRb6tDTLz&=<)iTyL2SY~X_$?Lih4dy|0T z;*NGc9qf1A3`$O;Y&=W`N{u9cC||NaaaX?Dx{lX;og#N@sh)&@X6dbWm!D{zQ7b)FPfhMXurasIuy<}$ zSwXPus35Tpj`VsW(WW-%46BtLg}Yc6#}(^K-3X$>Hn~&s_+&RAh807^PP)uD++Y;o z>#DIPqfBm;|LVIPy-oFCG^2pA7tvNH)q)H{z7~ef^8|+f1ju`o0cP;SS9TB?II`W4 zmnN@K9ytG1vVJmr_}=8@t3GkRWq+ietb9Ryt{QrBgs+mbU!@^w6g$g;kgvMT-xi;y z%s+hGuwwB|^T(MpYi-U4wMj1uk=JVbq8NuvN6MU_DR!Dv_eF3QxlO(l%Cv`91!3X< z|6e1ClAh{K+ZG)LhpodaY>UwK98EVxRTYN+=6aH_LIKyj9b>EV&4w|#Fj2r-(Xx@I z7)*qtohP#yY#rg{HEdO)XvM3oxmBqz#vIP%Ibj&K8uu)>TEAmCMRdE*$Wucxf{@3t zbsHGFPhPAAe5|z3z=A64t*pJge5!Dg8d}h^Y$?Pd=!MT!ZX{l?rKZ9>(vDy%{!7dV zav%x7;9}7Mt~YAnU`7GbIRzNFy9(k6z>teMxu?dAwxP=!Q906m0YTC_|5lJiZcBiU zbXM+2A$_sXDs0O=Q3#%Y*CCz29)+o6>-FA`#!F);!_Q|0*N%(<{RsokRyQI1wE5F9 z-iW+$zHNDgy7gLW=AGrf3D$$pkdrG4B$nNUoG4F1O;$1tWN3*^T1?}zmrobn)*0lUNugy4)!0Wa*r*OK6WN=GlOpUGi)`p?mFo>&MN0-a&K;`EPrm}wx(qBuM z%0X)bOl3||P5BxP^R`HTfxQwSg>~4%#HY;L56Dk_axm_9oX~nO&L-)gKMSFk0G%e+ zdhk3=VJvTfT^m~u9=(3_?w9?Km`#Ir59?=)|KK%EU654+wugls2Wuj>DwxDu?H(p^ z_*tnmj<71U+9CX~LA#41y`D`{Fw#&Mwox_c#aQ3;lY%l!5-D5QPfLueldGRX@`ag_PFvFW#)CW?0nKs?GR2ogi4~g zv`H@IV}k*}qmvK~eEdKtf!6_y0*xFHDFz>+IPMb00V#k+utPFQQKaOhh)Sgz;-L+p z2^yF1C87!BT|`wpO71-kdE2H~$IbfYUAwN_0~SQh2-O&=++qpZsu2pVLW+Ji2)*x4 zXVzibMnYE0_IEvq_ml~;*t52(5$f8Nd$v}nL3)gD@wjSR%Y|Tan_1z>UMXi4LKIqV zh|q};F&Casc~f70KKc1F9rUij3U%eHRXo`Fy;ct2G{^Rh4`?%Icr`=wL0P>CGhk}t zu(0~#K?O^e7ER;TZP0kFU=dtO^+6RB*9ODE51{ho*`PW=aKXerwNg5-(3f?vO+!pt zO<-2TJ>`XQ5R*UHB3LuRj(`PXjN-5uf>Pa%+AX)1uh!;s*1x%og(d0OW|VOuH=yYmN^x zo=K^$l^XZqdv^mU|6=>R*&ii!O16n?W5= z>%I;?1%Dt1LSy6~=X)*^B-H7<1<0LSv@Y@2j-j1QpPd^x0a!wzP9A`UT(uO>xWKN; z>)548Jp5*Qtc_Qe$Hvjuj&XrEOQ?{Kr;i8OxKH1IL1N`|=|yupj-9fkEr3}%I#!pG zlLG9}Vq}>>7goT)dM6Pyh)>_MyZFTM7qgdVcQ8reIF9ssBqoX@d_2JnCr9$*R7Yaasj;uyf4N=eBx^yKOk_-8&Y=>xHE5fq1q6UN~hL2Qg*jQD!ONclo8GCcPG)t{#AXB(8v`#7hQ6nO3d#xf=!5|vjzGFKqO%SzW(tk5xV5O8MJJF2T5ai1pEft3r?Acp5h#bwd`70A^ z8MZ}bKh$zwK@cC-vcQk^mslReyHkkl4dA<#_4=KDLP@5KX78BT9>rIMYto^Ou7yay zLMkQ|l|zc~uMpUfQLte-M3gl}{8X1ir2HZI!`gC4y5I1jlgJ@H7cH`fKOYiDeTXX3xXC7K{LuB;A7La^-w{yk4k_8k@+9i2j6>irh zy}YYXAc&caLCHqZj|E9XqLv{uGhjfSIdpPe-fPi^3&c=2S1&D zQEu#pcB+BIQ$!CR?HJ~x97{;m^M2>Pk($=1do{9%xfWIuWSh8Y^#s`ve#hE|9M$r} zOi~S;wN{4gC5`GSvN<*OCQ@W?2N@+=l4$nrAOqe)Eb2hzc3(Qkaz5B1iIp|;J0m6E zpP17o$D%qTF!0Wjig%?HXaezVB#_$T#S4UL@1mguNpqGg&bi)T>T*VDSMF zEnv(omZQ5`rkW`$-1!r((K|)lf=-8J7_%Mr$6IJL7jIp4)hdSJ%^LRCyN&{m&R4AU zH^2jpq7WX;>JBC6fvQqW51JEtFPdB)vw)dPPA1zWOJFA)DGjZI{TV`(s#{7Gm81vWK1qNoBmd)SxVq0nVY!D1mX7h*Z6%=LN{ zU`ir?b2lH>aCvJSM;=@Eg`%8x2PlO9j;n}Qf{+1TiGw?&7! zQxAU1vL7L%<5ElDCE{@gZh|`zXE8__=yBi^(}W(a*$lY5(#!++@!XhQaentHe23y| z#k0h#B&mU7K>+iMo|4x|tkZGYL%vhel-fPoIodHcdEM+{G{}z)y%fUANrnh6b`p+_?S51D*W|tJoTpEt? z?(k+Y8u6Fg_uDAv#Yn8ib=|M(2O77C4I8jO9TK$cD;6&?@m9JB%t3FVRCAVO@6LxHNLs|4Ypl~R0d1!MxyprHg(H7G3r!1y<#{~iD^f&zG- zL4i4dK1_CNF#u9jL#7w{0=%`e9Q)yP-2e2L&LY_3h7_U}bSY;*tJmgT3xOZMN0}M@ zl#Wmx^vk%_JG@82(FznJQZ67911C}Nr`CEsqA&J(l#Ky@x&JIQ%6q1`9q?Vy6)Cd= z-x2j8^#H~KoA`6(x&Hvdx0$eMdv0p|6J@DQ*WkbCD+NP|HNq5!O@_%I*$7gGM?0r~ z9PM26C#M9hj&=qUWSV;){_@Oy_w<*K?#FlkjeWrJzb*#Hlhb6JqPaSVw8IClkTqfK z92bBvwObEHgW-&Jd-o9x zxgC}>QpiKhIb#c?>xG3*7BDQo-EKgNf`wg?g487n0Y*|@-V@(?z^H-F(iT3OgUQJD z`2(FIK@amSs@4xAXcL0WPO32o{%EknTGSL`t^g$hj>>O!2t*(_qc{mg+bE+#8D7- zr@ox!_f~14h2$gf#+EJ2yK(cpNN#GeEKAj&oLgqmQ~^=}BL!+Qcq71>+xQ4x=7U9` zU0E7JR-H<+TG+ zkppIz<9k=}!t-|Cfeb99EPP_KmfH`KdnQvrS!z5_$|UE+F?M|`akcC>FFX4`@9phn zf4mHoa(K}|A#7M;^GZe|u6nd{&~9p{itB}|)%XcZ)f!jJsCzZ877riO{a&$=tU=KN zV^JttFjo<^Z_@t*bIqMze@fJn^(a~{;?kv2nCa_{Hu@nwoMB%5&sCh=mGQG3UD{%@ z9bIAb=)wRuI8w9|HOQ?2rhR4Z%+vcB0DeA$mc~@fpgD+;%NkC^S_6eMQk&b6ab%vv*DYO<@iJDAUT@;&$=-jF zW*HKrVR+5kwiv^Xr^xfcWj@6C-@KpEj^ykI?aZa+~RYF+mB{r6|P&rS=*R-p&S7sGcQ*WY+DE ztOxJX(OEL-ANP_`JG&%1YnwvZ?KZi+5y}l4aq*kw5z=N%R( z_He9XBGQBGavz08Qv{ttKLOX%&nWy7&^NU>-hQO7EgH+;ykno`Bbjfl+YTlf_O#hAc&F=T7ljF{5 zHk~Hz(=;8T-0(nPQ1LX%BB}Y1ERT(6A{F}?;EL+A^r_D1!LkIPx|#b}QRO><`xTr&UC)Ng58J~8>+xTdr!pXLm+F=}J}WzJQak|`YkbZX ze~Igb01a0l=(w%W$JV>*F7d;9yeMUYdzpPgu&Cv$z3rUF*WoW&^=j+-OX^FBkaVCA0lq7S`Ryd>-1v=0ZTVNr z_y6<$`MW23FTMM(j%E&Q`Sh2C@?B!uIwHc<4X{MZaXNyxMEisSchEl-wkeKLokC@f z%-s$4Srd2S4ArN%)@y6edSkuA-xxQr8caFps81q_GyEx3}<+=ZU7L!xL{ z{q?3zJYKI|Y1c%yl^27$IxZ zeGN5R+~xhr?0lRKlg{P+&!cD)*Q9H)G#Jz=i?WWZUsi9ja(qGSqyn)~ zS`XDbsT5wZ zDo#}d?CVQe497M8L(g)KcU2u&1+f+JA4z*clh^P>Tp!wyAzz$l@BmYqATrOBr$yr=`IC(uo*rZmASB6~Jq*jUvtU-dg!n9vc zgBd|6aA(@CKgb#{9XMKsM z;|c@U3+XH!42M+*{-33ftF$*3#Q#EJV37mloH*R`V=9B_i!o;imZ*n4Db|$YUQ5hb zlu|b&aNyumIU^SIi^8cJwP|5A(cJX3-#HtllgaTQB};gT z=@ij@3@wgkg5U{GmiyH+yP#~&dofrwHu4a*=eL)TA#6)^AfsTaLj7Y!@ zdWWvR3#M8@U}m!Uv~!&GdK2P7%HyEV?-*orFgoPGNtzv}qoh3`H^Oxjm=E?39vu)k z&)2a{;2e)}&Pl(wq}_(q^(r`LN%PBb4%E#MfU0#a$1fF}v+wP@kDBn1fpZSL19$&G zYt6tpp@%Fd6iAjo;AWg7?>Vni;E$D3O(0qu+jy-0fr|}2Hk=4QPJey-r2Ff~9OnFX zu4B*QG7T?cIm}u0YHPxrVsavM1HTq7>#-Lw?IrL@9;;_otan><1H~b5tToo@LUcpc znag22lYDy{02S#7qkTT8K+Mgb|LfmP)8>GtO*`CmNh!9NHVv8B+&_&^%JC`Mq*%i= zP~oUj>V|*{$`8DY8G1)~jZ)-M6txjqU(=qm{-97=gM~LAW^i%6e+U{m| zt+xa>Y&(su7wKp+O9xguwVvZF_`%0rD;*)soGYYxz1;uzkj!Y7ZC{un6n z?NnaSIj9K)J4{d_mEKPC(h`*1dHCwh{=we+UvSP`zwMK3I813+oFotwNbjB}lSw}t zHb)F6S)Bf+X^zI;?STCeM-F`B;YYkFsa-NN+G;Hv@lMkn1a{vDQXMazy z@*`=5^$BvIsr8SC*%)a}W}p9gp9bzG{gH^Csrw@Xq-C$7AIiT`H}qi6rIZd;{1f}g zI9&l*=-7yslFBHcv{zm7ReK+z5eQiO(v9WFo+8`K_ zhsGla6b1;2Qz-CWeh6&8){4 zNjJF*|K5v3 zi@!Rk0BAty`81u4C)PgzPDg)FVFE)PYm`mLsr6gd2CT?HzyvHAaAUUhw}0NZ4yG2f zktY_y#V6V3A%qL6Ma_WX5nwPWP00V$gdu>|>j(l`1ujvY8YTdkP{3(&cWqcvf&xO= zWU07uzM{hrv?kr0@I7FNR`BTIqlb7Mxy@pZ4dsykYp7@1P#zxO|7o=h8w#S)@c*>Z z>$0J=K%5N%^3?~P+d>+e;ArQ?bZ_#;|G8CShz&uA)i1Utg2<;z00_DzKel7i=%^rt z@63Qgo^NQQ=@CLWv`)B&X+UWbH^%5i3K5Vppm-5;O6lC~BUTHe6mO>V`ykpWj&Ok@ zCUSv6undOmg@gVtCwqH8?p?ZH5>(mHPPzJty+VR24I@X6b);%7sM4r=wNNU3$u#D8s!2#tsEknCZZOFCKO^l#;^6A6b-uuIgv&%0ULG^aNff01|fuX-`PIQKGf#jyu zcU@WI7S=R|FA-mxhp#MgISgAwVrh!+Rcxk3{FdJMRs{I!_y!-wGJ6J1**T_fwuqDf z68Sf3#G%bXR3x!;z|`Mj)Y859{b1DGda10rMfQyfF0WTpv^UxR7#JQOKt20LZvHb% zR3rJfuu|rY^%3)Sz{A9Ow4+a2+X4R<4R`~7tqb7S9nf{;U6?7B6N)rzMZKMG0Dl<} zh}`eZ5a=&Q@WpIfP(H|RK}rzDtX!{lJu|p5px50`tUI9V8-eu(biJK#U_jrB4MlyJ z{-vWC@vDWE`FgfDm_O!+jJjXWv~%570ELqJZk|F$;l7hnU^b0u*H4B}+<;^@@*=%{ zVZF$hTsP={f@c!c{=ubn3I{F8dV+?za16*?IMTgOnxC&`V^HmqE+kjp zB&V~1^%x2*r+tgPjiZT$f58W;8kLsbCT;61wdOSVfSaLKsP-2n^+6k7Q7fiSzAL6` znS<*Uu8p1Nz&3~+%3&NprNV)B4W`=CC*M$7ZfD=(srAqIlTT$pc1q#mMEobr4<~Rs zI#zpMsXRHO>L}El^e<+^cGACVJwzxB?q6=&TNVy3@YfbjY0XUXT8lRbv+KCC)Oj71<3}D z3f#l|^2P<|JJ}?$&vl)8RD*+Wvg(zG38v2L^CH%Duk!jl$8J@f^6zG;b%=0l8|i?e z-5}cayuvBJ6=qKcleBb%e;3(+qR{t9W5k7}H;iof$@jd&y_Y|xosIj+fks!o+M15? zhWU#_D&3X~3i9n0W?z+b7wKV375=>!AUg?yOFYmlbv~BbGw7u>+X?CJhM)jm_iGIO z-~CtZLHm4k_#-Vt3&?n1FVCMu1L>stV##k}+!uvk9S;Nmf$-|+!G!rCJWREizhf{+ z57$`ZOPGivZH0f9^v`GG$_ig5T6{Xf?VgO2p$fFm!6q6p0lvb^W)zq)&*i%AK-o`<1_0&=;&ra zfX|e>aX%e(p?8Vx|KO?Kp;CuL=SX>giO;{qLaSK;(Xv6VTc9 zOAxRYGw-^C8*3T^-vcqvi60%dNK<18F|YP;asr=0%&XNhhzqVH|!{a zE1I!IU z2QWI6HUd~0jz&LxNZ%%d*))BUCQ!e9l8!UPR$`r*lljAM!3{cxLRe(&nA2wIvIUZY zrZ@6*pfT7DNtT>`EfI+>tQ)6&^iZw5y5$5%$#G1HG9nPuXp2{n>C1yut5BUl=>J`$ z9H^*`9LthZ`>`T??F&TLR;u> zFF+&-0#2=7@<%#^ILZk=q!b2J(x*O821A64lqpk$q5JwLd-Cz*Rrg|>%v{NA7`rQd zV^R-4f^-zwIUJHyg;xymcgCx%m*Q2bk^VY4>5me}^)7MD-2WIw+WzNZDuF%m@UHc6 z42^Y)9OntC$;`>yAq$3f{X!ewU(|z#eIzKYZZDc`j1I(T8@3k^es!7&PaKn%*Ks zlTBW)(wT2D_%_%@3f5pw@U4T%5V6(}(G4*Z-~%t9BrR-}(sE(DhP5n5Y`iapZ$)L{ zL%uW2I%^v9zF+^Ii$L3!9|5swue3Z^u4{0ZMWOdR`vhCp>vRnHuMv4~ajdq^DI|mA z29Tyae8vW{}pKWTR6E>@g^EtPx~55VH%n-CZl4 zL6T!^rJvK``2Zp=T&L$-(3Qh^51%6uw zImOA4BvS{)Nr;3eR?-nO+lDQ92)<|tvJGHzudcOG<6;Zr0p(<_+VULuFG$q2w%YEU zk1ab!q%ACYyPjRHwxq9g?~3@HTFaG0*I6vEVKUOw9Yv zby*@6iqs+YctI3E9xe7ssqBwQ@Fcx>ds> zcT-xseE)8^daUh>-~SkX*xP&Y@%YmDk|?BxcEkv^a3`OmAE{c4wKaDASF@9;)!Jlq zR{-6H!)%FUSwyBqr1Al-Y^#M8%ZxU|%BBZ4QsT#Xq*^_uX+et;fK*tg^%2R;PtMq! zM@Ux2js}nv=GDjDc4QsxK>b{jrirKaI~!RVlbJprg#RNO194?AW7c*{TInW6&iM69 z%t?E{4xj(>MRQWUo!dPp8S{sa#bMt3adonD^W;Ue$m-`SwK=Edcv#YG z@_IJzv&+(rjGS?dk#jKgvi6saoLW1#d*tXZ&!^onMovTmiq*G#^^v0wMxT@v=M6?< zh%LByedL6Y!GpQlZ=vmWa1+iuu; z6b-Br$k))$_wwl)Q}luj$2DwWax*u$pCqVRgkV?pZ1rkd0AHmpMy(lWZH{xZOf{p8 z`9CS_>Do6{+p$hj*x%Qtsu-6yXRKGiy0($i7KGd>rj;5i!$^8NYBcerVkM|o#1yIK(HlFK*V}lmJC&qFitr%I55j9x(_9UGTvN}mAFp*-wKIH^}=_QD+&p_+|=Oa6_lJ2KugxIgt z%6gPeb~c&bwN6t+S0NB<%Ax{;?1BZ;p^%QOD??CcGel-VaR8Ja>-_X`0wOK`d0$#Z z!nzASoqar z0udA(d$AwcNGR^Z%};WpwMn)e^aqpEc?|_S3;Mu?U2v_pBEO&q;;6I?a;E|0N;>XYEvcSxEde z8iA*~^ZwD#e|$W7(mS1GKlfg|yVp7RDe8AmNFv;fMtpkG&a9&y>-88Uq(Ra-vyc>^ z|EWKj;^31EVTBmf$QEopc=P$qdDiV9((u?iOeUuo!pLUu9ASl%=_RERqJ-?(_@4Fl z_49*+clWKQk9Qwb?r*aMV4Da?Ju|@2L5NC~l?4ApU1cCZMbs3On-&3zt`D+Ug_CrM zgz_&CD(SeaG7#=MTJ}T=i&~YFv<)y^ISKXh?1_g#>tK)g*g3B!4;-vyPpt8^@_IKl zz%YXJKeRJK?AlsEVEacqvwPW_U;KNAIVowA=@VDI+M1-K`h?5mB+m_EB*ElqYMV#V zNCNjiyKl?8dS6V2}*k$D=8t z=3jD9l1KSyoHTLzR_AoaKjV>giimm(DSc58jZi?pI`Qb#8lVb7nS4^907X7kv6F{`Rx`T^7*e62mD(qC0wZ^ zmGok%B&S7uLX4^arG#cXl-Sx0+xSh2SN|!{Y8v4@gw;L6rjDViM^M%Fb^1Ij=G-)>x{X!) zl32Bh?>tAoC8IqGTxj71t`~(t;G z&=(COFGk8W9V6JaQaVo)r?~6Z=rr)EU4_N8NDtr>>1Qm^1Nd@LRmx(ICOqi9@VG7-9xx zxHyPss@q&SzLB(8@pAdE3TfY^{@YHafY;GV6!VX~zG6Id)`G5e`^C@Q9M{K}M>KNFJRw0e0LO4_+C_ z^0~P3LGb6Iea zXjC_?x&Z*J306Tku7g!5FyiDISoM9E_K!xiCs|fsb*7w`-QV?NUEtB7klj_`>a}Yt z)U1aaiS`66YoFXd>67Ac4&s^`ZlFC@hdUiwZV@~j>t68f*XDjn|} z?a*R~3iK|jq2qsT)J0gsXs=!33aiW@C35YPeR z8(HlIG%M(=!E1#b7&Q>$0W1~}TteASLXI&L{piRYg(U*QE6Gi9gvyKGv7_19Tmok; zLivP~%ikVQdvP2X-31!>2LT-~1Oj52`}@W}@Bb4QuR6Ak08Q@bIBB<`W3R{m`jLlk z+4W#=4m|ji{QsW(i_>wN9&UOdI((G9fj~yJL(qVgE3k`knk}GH9;~*zEB%=HR1NF`E~I zGEsqS?^tbjK$82|@%fP4FQ;1PM#^Db4`gG+W~xDK60t3AemP+KX1(EI6geK0fB@P; zh~7b><6)9Kx%cGo)inS%INevf+PZ+Pz77y_144mw%%`uag3xXYa=Vg6lN=*PY#o-! zicl-1btr9UxmK$KJ&jFx#haqtfCU5JgUvh+LZ8wSSCOJ#iWgK`0KL}T&;e9Ry4}7I z{f>oa+aL8ZazqX>%}@J(B<%sR8S{=6_Q8URyFjr9W;aL`AAjn1)9jd8s~9EZK{fjd z9gNx0PJeiUZ^OyPKTkdkviLIWjyJl^T2W9PO$1tenz*DN$WJ8R)SCb9V zL5_|cYFPhhho35*C*I}s@*HpG4j}6^!#^o@71n7;C9lP0XClLuf~fPj4F2D01h>w( zUr=!+?U2gA*$(nA{jhG|Am?F^K`xU@K1C&yb65-BjiI#1IW&ak z!iD#N{)Z<*Xw_$*1#=zFJKDiV@frVjwDU9_esmt~zx}iyJpbdTx6kh5V*d?-_P-n( z;oe#w!T$KVUBvZ1|pY~b?@CMsdlmQUX%0l0a)xbK8|)i-y5E%C-2{om!$ehtox47iPn5_FnH;n9A>}u zfBxP6{OFzO#ys!}de0wpz6H3-kg&!k=34}~ z#oVvC=aj`tUnA<(^7;R3ttZzD!eLUHo5@E7%phraO8aJTA31)uh**I3dK^1G)ZqOV zsRbBh#9AS3xJaB>u&xVNH@ms3(G;qHb_dNpvpo+6_B((6`M>^cQg(=8{tS7~ZO5|V zY6oSf^#4?HvT`nV$44p#+ku_HFrZ3^HIj5nDY~7<@(?>N{R0`+2PX-X@yY$$iIG!% z(n-ePG@&@N5N(Ikhd?dbJFoUQhLk4_V0PRQ`ut+(Co1O?_*$62>OI>jLAX8kVN zXiN9?k{k|)OejdB%sTH=2tNjrg+TpI)}P>iD18+&HJl*unOfbbi#A5Uc{TonVvS8o z9XWBKN^WLDb!L5)$jpLRHZ05D+5} z7>Rtk0htm*Pyr8u=TZ0B;MZ$$n67xWH65n)xr6+rvF%2Xv~Kj5ZUaEpi5{*-V$^1P zHDB+^qHRI7f>&+i!{HT~qAUzYwY2XASm@BmMu8hB)HUShk*u|e+!|$OLdGGEzt%YR6&>%4ScAjDzId$`|F&rq3pw14o($fL#k)lN3Zu!Xu)d%pOvkE_o( z&+(Y|r+7g6gI^LSj07OaJQpF!9z^+}#^56iB7Od7aM*suMnKR`0-(N|kUR!~RRR!}i;|akQEqeTf?&@oswmj7f>8v8a*1j8G zL%QwXXz6J-g~dK}IQ=h&d#)S1psZuvXo7orDRo1^y|<_RlNsVAlFtBq#02HPjLdvI z)u`<_NDw)LQ4lL^*rpXW3R57u7WRAiYtt$Ww1#yuiV~a=BY`HjArKMkz275s4!k9S zi?EI47?02Zj`P)s2#@2uzDK;83z!s4NFX!l*f=)~LVnv)N@3N#o=rhhPW2SUDCxE zlo5)ZL>F080pfTd2_U3=r=H!$V1o`2P5@Bk;U}PxHX}IB@t_0_HNx0=+8>?V2c7n4 z=eOi&hdTpy>}cAbUVcJn%uo<8uh3BJ;d6@GE{3X~#Va9ip0>s2$&N-QJ2_S{<~zl+nh3?0MLs>shY18+f~JSU~o* z)OM~te0ghZy=PKwZ!O z1i$cAJmQNCsLK#_@a2fw@%2{pOwZ;BC@91*)Ds`+3)T zk&aVncn?x|h&!p(pID=83g=(ewbD^GJ2|~?eMqfQ3YYgZvt;UrWH(g3huP%(l+K1a zGP16x=TmNhM7yEy-5w{J{Va2>BQ>xp}R2wGPzI6Ac%z}f84r8(XZ=_ zzEL67t{=n7O>zy$J?Xk|aA||wsh8p@tB>aeR2n;H$){wJoFqef`gp17=-4cVZ4dvy zMas@4m|fylezNxz)i;V?>h59;^eccDk5Uv@UBw8Hu7H=^-sm`%6C;WjbH*0DjJn6m zV{4uLJ?D^28M)1Ejt%gUy?=19)CRbZN{2KPj%5GZGL|21;uKH zFkoy7T{rL@sO0K?9P(9FN@)%f>(O7Zw#k}S(@gWA#{4pSJL~_B-Zff+Ap5t}K_~%Y zNgH%8;#N+l=~$@E_-wkf&Xlx{F4A<`O*>}`WJXW}^q4bZmCaQbkpb3d-MO=SlUBko z3LOY#>-Z->gh#CwwH)x^h`HM31r*zs!4%NQdO#QH zp-z_r$#5FI(uvg@gQb;U8yQuGlXL*!1^hMXpN!B9vQAKc)>?n%b=MToXm{ffZ?R%S zs4c?CN7kr=2FpTq6^@}aRau5f0u4!HEGQN^9!a2iK`{@oD|1!1pWMy>oX32yQl#Fv z-<}~m$K*I2QQUdSia>;Z`P2$#M7|bj-lp6u_0Vn|MEZKA)oFA-UB&B%jq-I0pe<>B z`I2u#4K#j)5tcugIZJ-cWtZ18+z6s4|)SqS$h@8|G?rG0`-g-ZLF{7=`yoH-p!R8n{pqLWkEnA zf5H<4>IX%U(=Xz%52@K~qz+9khwUscXSECtI9D(UG#5J#k+BO@7eAu3^}27PT59$H zLf`ucDMu9YEXRrA-an;k7$wc)^VNJI$>*9U}rzG?k8ISQ-os4;#JG{F_H8YqmO+VqcmrOh8t z{|hI5Jj<(?q}NnRfPg(NFmDAnULwz8e-X!~oTD-M);D;x80I@bZHmed8OxrB=Y9=D z8qNf;fJpP@yy*Zoe;09MhK4xCrkMSUQd*q^eURCIkfoRHqJx z%xQL$aHg$A*2n03ip0D*Ain@UEPX*RfzjwC~CQhXO%X8?dP z2Ch!5(Jl1K2(Lw$LCM!^A-Rd?Bfte#%a%^Bsuz}D&C#qA5ILglcR@>N$Hb3PN3iin zJ=0jy8>b14p^#8d=@q#1;-kEKoZapA;FnXe4TyTMP_qi;E|9UaKf;W9GERnKzcN}j z`6~Z&2xrq?r}KZU|6?7%PEd@jP8eJNu|8r54HGThMK5r17>=*9W|zcA^LUF^c!WG= zBWn}?Va!6>74ES;SYcj`(!023C+K4Icbry-{$bN2ZAUSE5J3ReM<u+)*o=s_X z$hD!qK>rlUD{7V+WE-eDOvtBFkFIkX*dFlS#~4e7&6ZpyoL$MM>`ERIfyRZc#vzl) zsFG4ZtBrDa~|(Lv9M{5V%n@_jG#ihEGyI zd-WJyUKfdMHdgD_cw$;uEJ9GeV9oLof(kKiZBWz;@qiV0FBj(k;Gx|@N>Qwn5do7< z0w$>X&1qutWjye#=)OZNRs}S>4)K0Wz@h_Q6l##AxFpP}X#N*|3vgMX7vVPn9!^hwznvcKboQe0r2ES+E$0RLV*;H= zPxfAZJb3lodboG+#n~4&2h1vPiih zJuwO*MU8Gi^w^F~QD$X3_{|Tu>$T}tSdX5$uc(DsG$uKBn0lNtp*5w{o1oBhaf^$E zLLV_@)~728aTw{D(O2+lnHnn*m2GK>+Hkr=kjG#2(CIp;B)<`{jE9Ne5q1IeKU~+r zi8eEV%UUS6-57674xtf#Nkpxss(7Xw1ysnH)(Cf?%8qt|DJJi5393y6}ZNyFsZbvKFqoZXa09&9ey~n6o%EZISEcDlWDq%`f+X z!5#YO;33A@MtoqkM@Wk#j9n{*zA5~icc^$oE5yE$vPujZ83?-}c)Q4x6F~d))#K^t z(M9VTK-|h#TNA|97aam|Q4D8w#MHKmGp%s_R*rugPPUr@y_CTR3PztheWbGCgf~;- zMnH!Kfg7E(zp4Mg7$u&>MK*@9=^}xWJWNo@;3B!iLFg}A?X*&Ofh?Rj>+j%M5KD$E zO}xA#&ZgUqh|!VTYWY*kRgvtwpishqywtQzM&`v9T3Aq5w3I61#2C*aH*|lAmD_-@ z>sz_&2tI#uYg@TPI5{P5amh2hnw5JI@%omSvMT^}^LUty;q_T?87?nf6EKj6U*A9; z#2etnBUTfRaGe)M&YPSUuSy9q%Et7;FmvL!KLUCpKQB-${sR3%D+tguKSGI}aD(TEt=uRT%cf7CJc>3uv6w&8s=eE?1e$}i3%Opim{u3W>H6OH@P`@or@R2S9F`P@^N`|hf`(o@x6zJ@B` z-l<09rbsy>pDv7vwskl9l5p3rOZb=Z0#dMx)KK8U2Sh42?D9-&h0dU^g97i0b^;!R zeg;o`OxJKg{zax??3S@dKoT#TMyh7>S2gpN>(whQn>$|_%&0=mGy2E0L;DKC|T4;|YTEM5~EOec`$xB=M2tIe!Lh2gn}*J)!(DV2bPZoDlz%$HM=% zJNy*YL#veVx%}ALflckm6Xyo_3ixF_VycoKlFMOEtaN}P=Qg@$kK#IAHK0(|%8tFs z34(XnAXF37gI>uA!Z~|ga8`HS)~;I;RynWT)(KK>kqzybZ!~TMf>sDlXt|+dw<4ep zI6>|`i9YUs?!Lc<{qxFKTXX;1uq3b-gzN=@PqAT@jdCu;Q#pKF_RlMB++YUG**N*e z4Tsj*LslgQi{-mmev7q8AS7TGW_aG!_Q+8`A5s+9WDbHTB?N_ks!>d_mCf(3g}#AElG&uC@2(!1{1t zy#XuzqjmV|jdk$)>DyQP2UhUk_^;+BieLJ(%l^aur!%y>uAAuVg4vrCv*SQr3u&_) zBW5)}h9{~<{ksGaF!X?R`pX?Ydi-wFHJSL{X z{W#?R8tNIbGtu57@8CeI#e3}BZ{Y1Cl!X@O{94!pUW;TNaUlZFhNTd`-Vpmr2(j** zeR`6e~?>W_4}GQzi?ZL6v-wjhHHFhomkM1V);8j~>>vbDZ^h6AFlv zQA1Gq_z&bp&JUa*%W)B|dfjVtjd{3{wfTm-^r2A;={UTNe7T&nG}j(ZUe8#XZ;X;f zh(#>I#~(uLuZ1M9-qFs>lV2Y>)7S5=VQF6ZYU?h|#neTH2=J|95R-^?T(LM5o72ki zZ^9CRc}A`2x!D0TSH4rv4>{O0**=%8+v&;8Xy01wzqfh`RSW4h%s$yr(CTBFG!r2vAfR!h*Yh`m>PAj z=54PmrDkwh1w^eJXzJQN1_M%H$`3Qz6U*q8v$q>TsSYw!aI5m8+UjHoc`p^ex**g0 zD#51Jaiqf+L89AHRvW##B+Pd{%JS)Ok>T>&4wI}508GSlp+4{Lij`IW)QW@VJbu@y z>5p%ArNDkA3breSX{P>;RtkWS<{Zpb3q-YA&X=fLD=>mp<&uO^4DV`L-b_(pwT#Oq z2GT#MC2QAq+-EQqRHm-&xUXB@zV5ic-qjSabXjIt!>*#s23{fkDtv&R?YWdDvNVC= zJ5;gqc`4=xWv_hj+vjXH=%?1>y?1+i@84Tb-adQ#mi|21dxrmqn)$AP4evRNIKJz- zw0+{43tJnpAtu)8IPJmWcsiY)vr)_ti0Eb)qd^7<{GG4s4-|5n*J{BbgSagZEEhbV zCW~f17xE)BPGC^bkj(*(fiMJz3S_BPNI7%^aB}pvE9AC%^-9a8EmFN6)X~{(J}wr4 zd7*RKiesp&(9cCS>etPiINCYAKrPLx$u4$^9Pc<1)6TzbIN;%Wm*3C|)Ba_a6tx^j zk<#QG5GafDiBXrgMWy-g-aYLf?8O`BIUjfgbuQa1{fG{mR#XhR&_}G*2odf-Xl=~7 z0SAQD{G!Hhy_5E?u)3&&-A>ufE+tD)XPduO>^+}Ai5QAXUA-kl+2sZ8RC`3-j<6f# zdc<*%$I|T(i7fv;zp1}bLKsAY(R16V+1MwCb;WAT8B0v7F}@?3FU?Rezlm9fJS3Qu zjJfUYY$Rb|Eu^pAwf$Y&E5qxFWx1-&_}atk_xHl+K)0W6!f1^6qwDnNE@^(LJGXYv zxh;4tcWxX%dgPHmAh$r9?%V_4cZepLmtLPcm)1m={zG~kM0UsS(azfpnGN5ig%sj% z=cF;3ccln6TdwtHoC2X*S$|f#+L|`wVi92O8eF-4zyXyN)9^QKGRB-#F&V2_N{>3i zp>-_ARq|T7JgX&F79|WD@gp~3gMy_t2MUxdwbnI+qx7ff+_hp*@IpN(WRL)T0|qTh zsT(CrZJ3@MXQPwkY_o|hoX^-RFQHNu>RgyD49-=kW!QU978QXoxGv zx{R~5nxS?rJqJ=xR&9bhxeyQY7U7YC5D!u{b#RGR5`pcG45p}zX1Q@YeSvZaOz=e{ zpPC9eM`M!W3Ekl{B*l~5C_N0c5fKciw;OgzULfe&d<}}S!Ah>hg*;b(cwoEwiJsXuPQm7kf{MUt2mjTs|3eOpnREnDs0p31ObH*L3^1m zDY=Z*6UAFa5M*SwrB0hx$*W~dfqa*&u|9AZV{qtytCKGH;CxZl{*Bg+RLKGA&8YXK3)F`+k zO#tjP@+N!`B2iE!P9T4Y9T7M-VK4$LoNiLDV40?J;J6Y@H;WbUDywQ@XtIlOO6y?~D1H7UGM{hEU8(ZOE0H0-Wp&*~{0A_)1eS*VQ z*o(W3c2n{Zeq6d4n#6~DC{#2sgu=H$k*$r<(l+@=1nI&4sM4I z@U+5y)Z!8*mxa+Kzrn0OtJ9sthmGX~*KAFIv1j=?R!sN0+2F>OQ{cTW;Jg#M3 z=5la@pJDHf=ftK7i#B=Z+4=g~wfew!C8-8hC2@ntG!#zX7dLH)0&}t&ZA65PhyZOZ+trrU*={>sxArx> zdUyQ!qW@6Lq#^J{AhhsoPh`5^h^whNW+8n&*ATOJG4Gt3Nw zQkE5K)hEMJlE)Po2(^Yjh$r~@n@{meZ`26hus}I6z+7CIRZAKU!vNDBPi(?GZI((R zM*0videgoHub?V@h*YYM2eR{+x`DSIHk%-8MB2+P5||;A=4IS(B@`ggLulJL{z~Z= zg}wFf#lk_b(js--UJMOjdr~md{_@e*rrgt(4rq{hK zzMp2XC%2od*LKgMoX5jVDnBA#wG?Ke)UB95!rCJT0OXRV;y+o`Xd%HdjVVSG?^2{P zV`uP_k9PX*LPTyh#`wbVW|4Nza32SF)y)Yz(Z7Uv3Nru#Pmp*Xc7p407>E5aC9eDL z2lW`1h%sI{%)6|V(s5#GNUaj03%voULW-ls3Mq!S*XB~O{9fkF0$g{oZTtEc*ex!$ zU%T$0J-PYZbDcMd^V!+L&HjxXB!3t>vK~oY^x;EB5AfZ(=-T=h%S9Ix_vH)!Uu zBplbAz0bui7J5W_f*1j^_|7i2zdY#;qBd-RYaGBq7c9Vk)%)Ya#`EWw?|tvQhdx;o zzjEA=Ec-jZTi71-FCK9%_4YoGMwg!(=F#U-?Zck~W#q8BKN!VdLGghF=vxDzSjweE z;-4%ZOJ}j7oDz_F&tj_Z1jU-mL`oHxzzPNSX2DjrwA(j{;u5r3unTL?X2ISp*voIm zzxINSZDuZk95jUmA3Rb#!c$oG!L#8hKSO2$*uuKR0af(^rC5{}Ly4vQlmb&r7EvII z)H=m)tHdvq8i0B}(emhAZ94wKNh$X0%t@{!iI4h-YK)8Sn?P;$C;+kX-4GrySJB+&! zT*sqMf&`f1m}a;UjI`bqKVgsCqIi<@6JfsFsTKs8ZS8Lf30kiKSE$+LKpcs-NCX7I zTNEs~o@>CtkaR40cu!VW4G5Aiz&OZPh%lcAJD|iy!qUKkmy{1TSR~&dy@J%XsF}4~ zs$OWhU}Vi+7OGKGIpcL;JCLmqjpPM^XIY-&fD3JPp%m9=E`DjKNx%)9(1G^vohRSX>^xL&rf0xa5Q`$2uGEf${3mY2$hO)ELva4c7UBml5>Qma!d9OQ z$>p9cy;XjJy<{}jFe{_x+If;Jc*>V-W0r+?S8({NvW{n@7V$3&C|5YH&Io#-As=!YJ+~ z8q=0&;|OWiu^&E0LiBE=osP9Zi0VC7(HLnUq7GrP?~>I zFF*^B;HPqj2kfi|noz`PQlMt;=J()a64i;#@-=_2U*ZN`J`>9wbhXcuNIMucl6Is` z(}S&(q>*TQIERbZ8qdIUWRRU9TjNz8B6+G0$ z>RLI}!bk2+5AlldO~8<-l*AwxDU{Mu$CWWZ+18tMo*U8YPiLp-VWrjwzArkFm)#E{ zn2P;VO>0)JaA#YyXPU^N<59Y%Y(vh@VYmf)$9~d}X-91th;0N(G{}13Pk;*Q#;ia% zlvZra*q~7`b3=aH*BTvc+8V+huj0qqtsZOP1bRTiYW*>lPqp4;s>SF9Y`*Y&h>*+_ z;koFd)rfl0KmI{_APU(%iH0By)|zmOX@mfHA=03B;(9#PIuR~TfCCZoEglXT(6j-Z z_j@rez#X8I9BJK2JH$=s3_?5OeiHVgAsSn|`p1OZXbBx4EUpL+D9=TS@Wdn@(vtRv z(psq4fC4=yT@4A&kVy;rAKlEc0gH$I*|0&02fAwKmx*oHkeGTN&&C>>|6PHIiKLg< zkw8KMg*3!u+r8NSvvc0O^lwhq4@8!~+loLWTQLcKASo$wb=j31+4ll|#Bl7HB*=}- z7n6*pWv;Cin&*P8)ugn51!F74h23gwt>Vm>DK{)_pf8%3^ON|Cr&;QU7DuicX!uCq2Awug!^a`FnEP zb4d|Wg~oW*k`v`Kx$a`-aRosrew0PHTIEMM=p#PtQJdXZyM$v-lnO2v_$EgL<)z z`+af9jrp(9>0d8j#_`8@hhH1}<4Eyzy?Zz2d0lkmE%dEdJqv&P(@cGH} zcS;u(XWaX<9Hz+O4xEzUm;5)@%~eni^BW%DMA10S`XThRvPT(Ct%7liRy1WN#J7H{ zC{9o)L!#2gkeySu1GpC5MgCEq*Kww65z@N-{lETOl}Kf6)&lIpFjG@@c*dTaTk%rc zQdWwGm269|DY?}sJdK*o>E(|0EF4a?4*)ZugwVSw)Q;{}~$1pG<7^K5~B5nglynY+1gd3#T zN?Opy_qDxvJcVH*B(l`o`N6&)&BS~!rLr>W2O5%)|IoFO@mZSPloOGi%RiB4QLjfH zZmMnyXTR=+-Ro#bws1v`?_=c(C^9i zg$$qi;UT71YV(@G_O&0l?yk&do7Ph@rS00@-hrG5mU&M8fF!0^qmjVW)@{pmKq2nK z8g=v84o=^X?jjU_>AS6{1j-g7QUckqag!Yk@2=!zBtDkH`JXKU|5hWEl*H8VK^F+h zuWS)fj8O1^(M@=R+Ss^#8y4)$(m*0>p_JBAZiM<_doXXM7scCp6E} z#6v^j!!j8wiBPxk6*ni1F}Ro+gltBQtBcv$Q2{ddsRm3M;JK{Mi^}zhm0lj{ZdO@1 z%aP;G>`aCYem~PRWII!K4&y1nt$=A7!)Y|kFkM9bOSP1PIF#ml_=C6o82-50OMtFs z(nHxy1Jha5k73|BOp(=Kyp^_(>oR=NP%YOO!YWYvta^ zi~*@72=VDuqZI-Xz#EN~z+g>;l8Om3WP%3;WC1^|BCYeFC; z3lrs6FB+Jzf>%+AlJ^N{3=BXEYX_X7|AO?6WXEuLqj&SOS(x?%qh``to$j;0iiL?! zh4iTunxDg(dB~ROv^2#G1x9^(ah2Ix6>PD8VC&)FGNH@T$1~+&Z zmtYXE!My=&@zr_u)qC})o*J0eMkR_g?91M5Rcw(ipM)(uMAkS~aF4OYEZqw>{yFSE zKMW8spOL9vvH8$W9a@&&(}MF)k;Bs<&RP?IS}KjbYU zGvA;)%{CR<(F=w@3&^)@`ZF!V#pzpnHH*+ifnST+l1h=1m2fj1L~aIHA=J`)3|`3f zg<4`=g{3QJZ>-QbeX>ACjwQ!?t*hf_E<0W=vq+EaK%jgh$w z;b{>(p{Cky$hA?UZWIl+wzghHV=P7RZ;HqG6(T^PkAF*2%xNbH-nw5#;S1bvo)oju zKMKX565R-La>oO#FnoBhz>gNX{#=@c2B1)iF&1+l36K!0Xu39{S$O)8Tz|+U2PQ^{ zI2L}4hzCd%?AUM8+fsy~!hArvBo6Jtg)VF-P+Y&#n5t4oNG07C;!bzctH%^ujh6a(h#R;tt~0lv5DkW~!1jlYOAn$d4(5hmeUt7%Ls4Gmxe7Gc9OV zv*K3WWwg9n3|G7%WHqSGFSa#;bf;4C7NE%U+=g6tBB+VIyqm#Q~QKB;} zANe5)8e<}NCz(!1dI)>RvUV)VYjBl_0xp?!*|x?%G%`Rli8+DH8uf{>EI0lEG3KL} zc_gKxizKe4>dZ|*K{vX*Gck)?O#%Zf{+4BeY1Fo1C_j1eWdHru$7w2Nt&vT2>AS5e zW|dcP=FtaB4}$bbbx}~--jj&YTXnew6D-uZxj{(L^2 zb7jpm5Vl;s(DH5ggYKO2KwwqyU4R`^jsVS1qGf@LUI6D#%Enq1{-U}BZqAZkbkl*` zKV)NOncWMA33OZGNJGV$ir*US^CS55!xE3&vynCm#;r+6rXftL?n?EXK% z7TVjjhp%4kXJ{bOiTFR?gi!>*L(NI;XlUaJwqQ`8l1GA zs*l}2I5;F1^b{hoMKT3PU=+cBQt90R1p^*x(87{A6tEJKE$$$^Yn-YsB`;_h{KA5GCRg`~NmlV|>`>puGiG@n`g~O>^HGWV!vQ0}OO)mPv zm4)5)iPcOCRo?t$Xj3l5x07q85Ps+~{hy>et%^~=PVd80!v`^u3DREq4jc!_rOF*( zkMkJ!Z-h-a%R2*CvVQb=f?e=E=!mpuM`U8j=$7S(s6l-s^ZD|(#+`=+wevw;Z+|)X z{MUzC?L%TVmk&EwxXq%Pfm3}ddZ`rI@v5b;0;TR^s$dmQsBr}a)C%`xiXy;=jnNeR z%%!-3#lr;@ms?7^*m*3E+*P>0QGa;dn2&D_j5zaA7r zUX=Ov)6LDWk6=Xr&&zg9o``cp0+0{Xx3JzSp8pwFI98-W6aZ@sY|o3)giA}n#ddK) zp*H?joYC*kH*}ZI==Z~-qpqKrW<~LhcF`OaNn2Zf5?AGWCagI~EROhz+oZ~$X`_eF zxpl`oJ@^uX%}yT;xuHQ>c{e-#%$+_p<%y8pONMF73b}H%RQXe`i?5ZN2T6-OWG&{O za$hROkX}1d4poQu(_Jv zmL`hgc_JPRZm1^mOBDDQ&!Y}Zakyrt9R|h~kTlKyi&2s8FCIvAj6Wm)tMr(i5+58J zQ~q(K!_0@HR4E;DxcC!wl=zs^TM{P}e2Cd!)+7F{@xc=<^V`V16H9=}uVzaCe@(@! z*&XF8MFsKd*n1Q&n18aA2RQ4a)FJ*9Vrfwty(PYqrZdqlm#yH%cX;uY|oiuuld$hS3HyeGUCzX>HO#)ICbW1cXG^?Uc2YEe9LMRB$)`+OnS7;iW9jz%Wn27=Xqb1Hoq`vP)KxJ= zYy?5CMI~vM=~yh-%J;^{gAHf;QR3TKtCi6pdeDNKkm*AKsahGK{3%-*O(KRX+8E)j zSgu zuorf>7LUVN=lu$u`Bgl4)M!kEblbU+U-974EgkkseAw`5vR(N3DIPq9Qd*0+PcmEP zq}7}6izxtMYuzt^dgnPkBlcsMfEwmD08Al-HBt9h0l}UF_axg!U?41*m?MR0Q!K@_ z5g?dLPWY`tNJ*eVS?=2-aWN=Xm>yE}UM2l2;OWb7h~zEJ%RvV$J;*RbR=T1ya=N>x zyD#6dIU?hzg6-a7+vm87gU%Ezv$_Jeu~{$b^8H?bJE3WFJO{zwz#G%w>yFqPnhtaprAWdcpu@E!<2OOC~Z9%qhY(YUMb`bC5wY*s}qp#s!mDYmQp-NL5KQ(mt9?O{`#t(1me z!?#kJfe$(G-Jk%g(rk%oDecYtZ9e@RehvpVLw6nJ-2dCPrqw`Ejj0h+Bm=IDt zqM+nUx*iGT`$%kpgz^^U7s6z}7a47Y>$~kcmI>iSN7{IL`M0J^lrLb@CBGuGYz98? zEnKO9rgUU3w0&Z!k(w42g_*NlyU?x*cBOZPdel_vC65e=lz0sZ8WBzt_&zbXXX047 z5Lb(jW)NOQoU@@;6D7Qur)ag7w!QMN@4CW^=V8N4BxJ}S_3eaAF+j38$qAi2Se%AL+(cleyiLffduhj^tFqL8a49#7F6CK2j>S(y5bwP#^Bb@K5AGbSWMCW(6cvL@FcUs+eh*Uiq zl(9D#+mW_oX-$yr+(g|5!Uw4V#QtdrIr#_8u_O3BYxOB+VmKN{-7}=2oYW^PGqBBy z*+<+5Ab!yn7hqXZ(GO5gg)I)Ok`?d=nZ6w~TEaiz4n(-0W&}Hw0?^Cx4@HIjboi&l zGA4!eXSsOfh?as$h}9ut#SU^mrT-ci+x2$A%v9BGmNhVU{9i-!x99kA;ZfqsL3(K@ z-%1p0r6FXpUK;XODuoEv+cf8I4FnFlZis0Men?c>^U?d;(3>v$NWzH-9{66Ql$jt(AB@VyQ%H=-+| zaHOy?o{XOM&_+K*jDJLt0l0$42IecaRG}62>&cfbn6$Q0v1l9=j6^M>M6&b{T4f7K z1?g{{{cW)G5rCzI4zTm=iCT?+UQFDU30fx4vGCWa*U77?`su^1tF3SgmCPpE^BNvT z2$>cMs2XHGbWE}gLMDxXZ=HZ}tYbtdfN>SdyA0*RH3);ry5zFb{21w!(Wp~*;Ln7H z&@>*6u^lebt`^V8b-(VJt?i18bWGP=fQ$38>Sb#}~$qhY^4+CNBJb*(y_ z;QW?O&a0+V$6Lm`!HpcNfdV`UI$%d;Q^TWRL@_iNQPwFjmVx&0ee#hP*I99sO_dfy z@_g9q?3fRD0~b&uycT~IycTE{`At}#M3c(+{K%8r_NnRevUbTON_DsUU9 zNux#eU3RTQ1WIeQx`I^QB!kNkX*!!Z1F&g8uc<;!0|Y)*Xw!07_V$vp4FUobcnaD) z9LDhY1yIMc%w>iM~OpyI4@C1Qu29hJskJX6^<#q{|_Z2HXvx@1R*ZLWapa_KQX= zqhttlUK1&*DyqtS5D*}-nW9lsC)1zW>QeL#g(*}PzBUuAG;!t7$=`E=A^bj#6vhNX zE*^xjAvd$mFP0P}3q_Ikh7A;Y$+;Nk)))f!4^P`iCY^ zTDqPIG|q; zP6mDFIo`pD^rHy#zZG{P$jSsm3ZNgR;|xc|qJy2Zi{T4fRzF8uK`wYmA8>wB`EIM+ z?W$vl0@Ypdo;5WC9|32V1)zKFjO$va&Tuov$Db&T}<$C{|(IlVQ#WK-MUSTOl zlg*T8WB{DRM?qd2d2gn?mrX&gbj|2K3e&h;?e3<08Q-lSAgs8Gmw`RgOI%_*tT5OY zjM9<{qukE#gztW=fKZ;R5IM@$;BvL^Ii*vfL_$2Gk26JqbYs+JVjPk3Ir#4jlO<$f>`Ub9-_u} zw37rZ1pT)5P&x}ALO5Y*=QnYeIS8H&!r@gHX#++`YtFuA-;}B<1MW=8j#;fTN^bx2^45s+Fmz=CZZ8`k;%6n z4dLz$57aH~Rf2R6lzytwiAE#spIT$mMNUUdAk3%!bPMy7*s(fM*xUkJcamJQ%MOur zg3?554^I)#Gwi=<;kgdMG?4VPhuPt4WJ79Dl8LqY2q2QbBV{o<`wWrU$H}z@UL>RmM=Coj=Y}y>@kPDs>=PwV$zR1epy1o7#{C3) zhyE|;iVDs)4{;XAKmo!(q&2e*_hS2P_tV+)tN!2)+yj=r+luZ1<%N;C2^dKE6Bw0& zKfj7i3lcwofBDC5$|guBd0t$%B2{80XbezvF8)|!)6Zn$ABY& zPt;Iu`apu9D9aIu=*^JQ`sR+Sc)N{c*xYKjK~wR*xp(%G+*?(z)iY9jmP zN!X4EZdCPYHkf@g8EVfGG{duJA#*iCU=T`NhRC6If^c!l!KfV)b_Tc+$+n(_J)jyH zW}~-j1tvDd^A=neol;UuSR>0ZGL?)mhUKBM;<89i?S~*L1t_6(mccitvlQ#=CZA;* zZ*tGlQ~@L)GB^Oo70lNGyzMTw85Di%8e3clkZ?#QLoZw0@> zd%}h{%bgILIQ*TF3Wa4NAzz2qq}yer7g%z?|K^^n2u2;!E2L@#oTrULIhAh+A|cY= zUq{hFOaDZn6SEKiOLaLTq&YJ{tJ_iT+2}tuk+vQxegS@DQUvY;zO$!SRVLWIDreK^BE>@LOk<*EeZMo1_SgX9*JB?OVCd$ReOLm zTL#Osm8oTAC-7M&2EExoIf5(7QDT1m7FU$2cB=Kdr&V)mLzDW~CH8>Yg z^+Mj3iUGnS$|pDz1>{VrIuijzbdZ@`I4B`EHLJ+}%_=sA$)HHJ57+P6ewnh*)0}%R zprH@xx+?HPLvC@iV_`gfR zc6^Icgw(~BmiS`(CaTX}RnY5nyW>lmGTllIWXkt^zd|-;Y&{-#o+3d~$}FkwA9yM0 z5YOXwThaRrY#X@2EU>cVY`}V$b~T+1IJN5D4;}&f3S1Tk;v7dAte99$kNy1F@1bSFv`Bc zjR6eu2HvAY9jB;-ZAfC@87aJ@HPc3Z@rO3VlL=b=04R{Rk$T9*kEL5T)Rh)(i*rX#{N{&ey~4 z6)TTN$rk>*=pUm-f6`|cbhUZ{#Z)tidUOG%7C7a437YD|FqT|(Q1{bKf{D3}|Mf}c zVRq3`cpZVdV;DuPtz<}Ou|?H~`tzsRDb;X@(K{7)APyT5N9*rWus8lo7v5Q^{OT_gKEZ6X|Dy~oDt?Q^()L=Cx!#J4$Opdochah#6fh?-4u%wwx$OKh~ zL(QiNAzo0!b*GOXYhibkXmIBmYedYf0jy307)Bdw#SSi|N-9{R$oEC{k`b~xo^71F z!`QYDKv9GZ1`N|^vIHB*Nk%s{N}%NJfJ-C&43Km!?p6x!ArRqa&hWW&FPgpbmI? zg(}01et7=o8v?JZ+9_fM zu4b-$fY1xsyH>MeSw0X1MebLzizrsC)G!7fnX(B@F}5)l z0E+Ri;2~Mc;#HVM6>^3&12=Pwb4e>!i{EyRK?ASYIWWr=iYNA)d26IbjVgktIrL8X zYCg+KrHGh|EeRzfQ>8eaZa#b|Qr>r|flNc5FY$%}e+Rg5vJn6Kv<=~%rk1E*%4n8P zB|5t0jVNC<*1V$Z%pgD+?#gRtxl~y@VbJ0hI#mq+QHUgU0>aoxkYvHR@C_C=_uF$4 zAb!lk7ffED1z#Ytuw|tfeW4)(!D)-gJ@dck1b+L`jrJ6QL2=Um>Id38exyCoj`m+_ zF9=N|L+D@ttlUm05U6nlw9Tw(Ll%F=yl&-I`7s(OI)6DIzMMF{<3H`s+C#_GPfp*w zeuEgHbAZmgPy86Yo@aI2BsO4Z+(i5brHmq&mCb?tWc~qTv^755r?vf~H#E9RbXoc~ z(o#tac0Os0lKx|@_K~tCw4-C~_|1DH{Mvnebgq4P^Y&DG{r2o=Pdk6Jk1PK{|K$^g zbus@*B*16r*jlw?z?LktpGq|A8}zNI8Zw)lx{&}-RgqD4(4vCO2azXUlcFFaoj57T zkjM!+KJ8$~_}T4hB?su=z^-QO2F?K~eN$WF4361D_FqLk>=Ru8!EOST_k*9ruI72J z0XZK2;Sk|{P>mvaRI7gY_QkaKX6?GTW zsdsKd^&h_d`NlG@4qkiqKh4wV0~%4KmpNBeh%TT4G;};!1x{WG9idN$0?x|!rNJz0iR{DqWo20 zLS)qM2sA;d%{cLW#?SBbnmiK#qnYO{{ha;ZWt{ck_lIj2IQhf6xSm{b;EO{id42)h zlHI^Yb9Pi`&Do8-mTN}n3{p1m26<-~hrLJ&z^L$DH@^aO{46m6&s)F=^CO7|Urt*f z-w7EZSi3%d7&ZjC=q>$xq=20IE^rSf!ck@8-} z`&UuiPA=pAIBLVTJ4Jbp_@#KxDxW&t{0|)#JnXNysar<|kZp7%FflEWL<8T1=OA$% z&ZTw)y|wr1?JmTRgEOSxI6lW!FdU$E!$JBpb}fH8d`V8-#~13^A^&p41g2-FTod^E z`S$>j!9fIj0U8fF9(*p;wU7p(O#!4!oN zvCXTph(p{JMPUSn6^Js5ERtrIMP54H+R+^(Maqoz%s=O+^YJ@$4*kO#A5%c=a^f!{ zRl~^NQX`{XtFCu@!n0r0SdS$M*%EzBYj}3(ps~5w9`?Kbt9Nc=?E(T@0GGSliV{M_ zQp+9ih&)=Sye=gNJnnK?IQ_Hbq7WG+5GBgLbqe26zi+rXzrg>)br-?mj=f{%`D|qW zI}dJmI(8!%Uv0ShU`>&>DX&yB~*VH+FFFWx^t`t6dm2 zl?V87jmJBCnakVZe?3|6%IvLgtXaU9Yu~efI=556m?SMRbv7f&;Ew=WhnQ+$43$- zGAdO9NDlD0fJ~a#gLqBTq z$6gSZ<#K5Yy?l zS|owsFUGUgBJhr5^o6DgE-*YcI`oFq+ z#&OxgbtyL=eCqJmY0ygsE+M7tT9LNL(#EN3e~)zhs@5NlvSp>e%@|&aNE^5b9xcjF z%yWf9ocxw0T}vMQh9O0pIUQv&$9Sj^P3gI5sJ^t5jYxV)5%*+AvSLn?WO7$}95R zwQISFtZa889XmaEQtKtT;KTIzwEMYp*gK+ceuiA&dN8rfaBa@0eAujEACXXVuVKK@ zC&{>lz1pzA3Xn4h&^_;REYe`$7NFY>n=6*voTdn%2R*7znI!p_b7 zu~2Za-3uFY_^G34j9QGE=rt<-5O!+Ba>L+f1RgmENu)pQgUTUcR*NCQOCA9;PA*m5 zu<{;JlNur86nzYiV?w(pyCI1)s3$%~IGiJI1dTj&Xi@*Ncv{jlzmB?fIO6srI4iU0 z%WfRf4Ve)*_eG}`x0=YEi!#+$`8;ZL`f$^1PoKcY3DO9B-eIpttUkqbc44UB(ugQ} z^caLdh#`gHzTbX~aL#0$w1?rKGkuKU(pD=%55NR4+R~oZMx&^Q44;SS7|NconMdt( zFef@OabmAkfoa3OS#lL{C^x&R!32kCl@1gBlL+E3>M(8a>K1gE=bedG8w`;{Ml(z# z`rfgPhvF7hU8A(-4iND7MtSLr zx&uL{NOtgjUs*0{<{o~TZ_Yy5y>H{@48EP4)9{&&uNsWIX({HX0OJVJdwLf2&y%DV zO?Nb;@Y447PB@kxAafYw(TKE%&y#-pCH^1$iiEkrzBr1A840YM-SB1$7jmM`Q0pYa zAu^wX1PQK=4DDiH1%!`~%M8>TjFp30yTbqD)7Ib)84(uiWm4}vBt9=I-Fh>`hy`9o z$s9e+Nb`~v&PwzmFm{!jhyuTtr_KVE22^i0Xb<4&W?7~Wzt9}amrL>b#p?E^A8B2l zBW4h{+f(@d3u2JSDl>JS<5i8q_n`Vge5z@pnH;0v;#*u}k{Ad(hf^(o`a^0jxkL689pQ zwgQ8sV8tVV;s9%tHrO;ZV#xx(X6}1y5PJLR(DcS7;7(lvDA|J<=(-6N2|gqE9c}xe zZ6r@V$9Lcou=L$lbqOdhGUO8A;zO}q^B4BLmANn79n;(~s(o+aeWxSGhwfSA31Sxv zaK+MmfBya(@(f+sU%~ajbNk~dDPHF7(03XZ#O2|8uqT@)LMcrs!K9ryJBJYz`!Apu zMUrSI(1CIcMx(3f!!mS+lFIFtrDo%}h@rTx5j7fZwUcXHqUyo2ylm$@C5A@y+{il2 za9ZApE$#VagrxwvUk^jTH)yrqZk=suX?KZU{Uqx5N7L?g*pEZv4_Ik~78Zw6SPUp9 ziF2~BL&JX^AC9W-GCf-ufz;p_lQOUscW8WnIIC>Xuc1I@>0Sd3U$15E=y`@lgYkim z8<)`hK|7ryH&Ulxex;xvL37%Ul1q#^KrbW%{z!M2Sgja~{;S043&5S~+7xA1iS&Dh zvTLzGeNvn$s4gOmeCzuo#d{S}UVg^

RFK{3n!p0U#Ym--U}a4p68VtJUd5ZnDS5>6f%^v4 z+~0WLqf5^I$ALqp)Ke${*Q~0YKkdQXZF9eX#izZD7z;B#hj?PHXKS@I*|T%XY=?tF ziQpAOmd@C37pK@YkrFd9LCwdbiLI!3^95ch&`f9ARCwh%RkBPRkDC0AJ;ZoLQ}V&OnMCqi^)M0v%GN`5ny4S>6#68$V7KT| zz^RN!G!0HPFzfD|RDwdK09#mZZ5EKkBlvlIq{D}VwMo9bEU0SYwd!b};(Bm*hm(LH zts2^qeV4%|pBqs*m+lKYvag=mj=7kNMlY!&TwL+Od!P-Y?jIM3(&y3l7Y~^0MU+ZF z?~zuyHWV;u)Pr3pWFL#$}lO^gYN=7U`s@W_wU|u%U%; zK3vbj3-Jx%`aowNyxfWyAA>&g(s{p5j1>G!;>W2|zZ;#uXI*FC33H@J7#=+~4-@2% zp*K&@sa8245CH`U79ds%@h!tDj&I*@qL(b76Ian9#D~#`cuVatW(-@bKWt#L&_Jqh zVokFnHu^dRj!h zMFIL2THx*DQTbbBOBnAQ|4UJlPJ{lU4y&KH!ygRqo?VsB0IEsDq()Q71Mf9yYS>YO z%|^MsjC4fw%X1^N?IM(t-JkuHhmMa2jf7jLCv#dY7CtN3cGzg~+1q6EpOX&pu?*RZ z;G{zoO^N4Vbwt1eJLdZdJv}B98zGRlQBBqp@uje_p{wM#k&X>-P42U9(&TbmI=jEP zGkfdQM^MGF^I{=qZHuefXo+u*O<&@Ka_SU+>o-N;*|nS#ia#04%(ap#NUX7CiWJub z;wor@DDqO(15u*hvZaO+4VBeEpuzI(qNU~#nb2N96o3L3jEx2u8>N&cwEX>amYQFd zv2mUz=oh7Af&Rw^YolRcX2O6QxEI^cj(q+3vvD(51+7UF1VXrF@3yKYC_^JGKonUC zU2q-XTPmr7xYo3A`ra|INY)_J*l~g#$H;sqWhNFMyqs{<0MV0U*}fwi35cz)P>S&q zk_pz|jV!{Yr5`Peq{@wghRY>)v>#?3yFYRYGLZ&=nqbW{5f<;kIb2delne0*WO57O z5Q8RMDB!?FJ%ie+tddIeD9GiMTRc_>cadNqi>{ob(Qef_?%~~F2#wI75E~eFwp%tK z;*j_;0>6n;EbIi@LsAYv=)xJ-=jhHbj^E&yP2xKj(>Y3Z8v>Ov94wJo`ClVIGbTu8I#I_h|Q2Gm&OR6dO?Hrhci@iN)Lwq zi5w72lJsPrO(H%7QQj+@=RyCm%7w(fq#X35|A(jg{9zfB497T#uG{&spCDLYK}c^tW zAhN{AW>z%2nhx{<1-mPAj6#?J7*Y&Ql_C+OlQaKM2e&tlQFrJV1-C`Rw4mn$^^=bL zp9VMy{m;Fxc5k?T((>hRwj%y5Z)9W=MQ}slBJc*2ya0Gp?}5VUpA82yb4W@@$lPIp zTD<@-(1dFww2efNB^=zPYTPmQK}B7Rgwke-;!pzChQ4FzJ4SVi;c_Q(KiF~?Y3bxC zh96?jT0lh&nk?*JPQ8V78(JrQ!OO6$(-znwHJBe1wlPpYyI^Ey4dO32+_8M0sh{meJ4+%_=GgXx{kDj+t|c$KyS@Nlh||b zf!i0z{GdnnT{A!!9;H&$Wj%aFDIQb++r6Y7b}6fL0lLBjK5e9BOQ2U7fpBE%MkY-| zrpk7ZW|_2%b_>d2dFpe>%B#{8$Bdk5uOZa!v@AYfS<~|8A%i5YbY*H<37)3&o&y?U zFythD9Gv0qG`~Y^1(U!7$)A|Peyhb`T$6npREHh|{cMnSUeEvr(4Y`N%zz-M@bE1B zjB7CSUEN`INZ}3Vo2))^yW+WoA{;q*=vdgca9)M!96ECfK>^FyXmGN8g3e_BVzYUj ztkI13K=odSt+J>Rt@K@YvCmc^G*YrEj&a0gM{%G!SY=HpGIp$jE@K1Ga=$FaS?w29 zQDS8?@%@;NV|h^Hf#`j^K4^a34IT3iRFq5KZAFMyu_lwQ(uX8zU=6^hTL5GC1fwnF z%7T?ZaZF;aElSj(DHajMDG7VpZ7xabrZQ^_$?+7%IrsHKE@73ZOWvSnwKWf9DXoKJ z*ED59tyzvy3KE*>7)3d3TKzsKt|;aCIBs+)hJ5cY=5eX}G=T&&8ux>mRW}QN%f|u1 z{jPBSDR{cAS>@HmjSY%Z7pD0wUjh^u zsjpM&oo7G^GUG$Q3la`nOO21nYp}L4674N8PV)B$P6&UEes^n}cY)=qAY(T>%;dM42Ln2V_7; zAcIveMzCoQY_HQ^0?Md=!*C_IiW)Wr>Mv)j%4l#(=-Y^!53j*=9cC<6cq?k3M1{Zp`U}@pS}rAJVAAhS zTd9RT&$8W(u}&=#Wn>vgX3z1s)vk}LVF!) zZ~EHNvmGt#6FRh3k1KHJlmi@G#4V^14q{VKX)cK1r)mFV$}C)cVN!*mc84`B35b95}^k$TiW;6 zgAc+qUH(nAUT#qhywnav|Qm*zC_H{T9nx&LI~Z$eS&zj&jviDT9Y7 z$+!CldoUEj%?mV9kF~dFl-0XvFdW9sX54FZlW_+iaBI{_2B@WO`XDhGu`~e^>}&bj z86fFaCyuuo42J}8#-=>&+Ed=@yj0QZ5`VHhgKj5^_2?=3DDG`v9SGs9Hzn1BTflSnx3Z#P^}Qt!4&Da5B~mE zjTnf4APs?3Ljpw8K|vp`WToPx{$ac)-D9fc!j8{1EF(ip>WjeRT4sBtyNbsw3G z_Nm4^Ef~iGGhFuhxMG6n_&=;OP&Q)JrF=x0%SA4l;6*4G;8Eo(vY7M)e9{bKyKXbfbtaJ6xHaa+2k)bB+sOBDXtf*#HS>h;@6xTNpRXU z{-q^JV@qrI#!uWWjYwMKe(Mv_?iGPGG4zA&4`CErJ!B;s$$eMhwX0*=cDvzl^lzZ- zZa2|}aijAX6OZ%ykB7+=v%3qQzxI|kr9??F!_NWqYs3V&G6ihvfymTt&?lzObv9mG zAq>%lzuf{lF}(gV2nMGMA`{CV?!?UwmASP8dUid}bhP1S{~c9gzTw#}@&LFnSlBS> zUTmL!xViG%FYh4t5Vn>j@3tZcu2>M6hdUNYgesBchfL*(R^d}L_3&=7z z;)nYB|_7ZTi-XH-6ED|8yCX!Wbgi}~r#vN!NfCmyR$gq)7 zZbM2_r`6HYXfz>fNLp~i3Yf92pJ@H4twsH4*q+X%Wz_-WSAoE|Og4%Fqs{$}C{|!dFi4V0|Di}iv0#G235HjzSQ%R( zrpGwC#H(?EhPfKYI? zmrS!lfud?-4I3J@3Lpw1L`XXd5~c8V(#Gm8Y}cfXMY^EWdBb4zl|NY$0&*D z%e5pSK(5F7U8dZ;rG1`qa~|$JtbyV8Kzi&{sat1hhO>N`rQu2IDazkcEk*fTDuu9F zk#n=q-OLYGL|U;0@RrN~B#ncS!U_T#oJ{ZoL1_4P{45?Eo|&)jLi@Yy-Btv?*`mha zF3i6O&8Ls41b4+47S8`{tQ)tQfMt>lBqBoc-f9;x4~bPNyQ=MiSQLwtv=1-v?0Hu!p81R}XC@PYGvHDHeQYTIBpGD8| z1cV|w=8(^Gd*CiSA3McmCJFSG&}i0%l#HmWF|8tbRUt)pObIT{vDwQ_DvY_u(bxP1dR}jg9qz(oA{YROS%Cm`yR@uwB zBPJZJEGU+CbbrUr6AH0-@}G`3A*dbL2c$Y)>t@dO@E7d&E$mwx;al-_4AR%-L}1BH{uhD}errC>pNcMQ>h= z&A0hJzP5xy%inFqjXYiGu#ejoo)9#Q_jn&yqZ`tQW)JTzFd>$~@M`LiFpAYudj)8$ z3Ux8>l*;oWnu*FqN1nn=ypVmviwzfmLHV`<7@Q$_Fyymt)P}rH@_aAqVeb*SeZwI< zYDuL9o7clrSQUB+#D7No)B}&e2%hB3TYQjoW2nlIYYqPlVIw>5M3krq8(AmB1@Oty z>S41@wub@Nv6L)f?%V0epX~32jJQMaF0hYLP8m+xuAf6|FxN^}ff0Tf{D98D_K~hk zo)Y2|rk{Wx=ixB^3~y@){dnD(rl3i=NQBm)*O}xAaO9`yl=RV%YT-I02$LqS-tC5fA+;Vn#v1@jQumau zY7Y;0Yh+zZp0Z9x$dw!sG|`@DM>GYEpvRP0H_5Qe)FDPN(jFSxX^5G;8=^L}C`XvK z@mL#%4Jfci+CMcYCKJqTXfAqBoq!QXRQCmSJoU5GLA{nD* zcd2*}YrolwcqmDAW8;EV0F>I=M(wh!!heE2}?OthvL05xj9XJZx8hX)h<2LZ!fK<)B ze0ct>H?NNNKb`kZ>qj&}ssRyFkIpv#W$EcsM^)i7#__D>x|qkh9Uv2`j4{fk%KcIN7(9`q2+>DBSQmHTtCJ(xgU+G}5I z+aE@sTfO@E>&`f7?E@)%8qm_h`vaBA?Muqq?paje%$W_m>L6~YE@8^dSK`FN*qFJ#}x<&c0I=1lO|pt;&aI3K<2%N`6wF(#RT zb)LL_t^p-sKc{K;-w*0BAQSm1ng(JW$g!`Hz_#AUYn5(!vE3VxBA_^8tRFNay*Af* z$?s;)Ou%&)+n%X^F-)-TZtuQmeYN9h|3q(}9=K1=w{W+AkHhvK@IU_hF@kv2E`|86 z0ZHlMF1Eisfq>M0d-`gP-5Yc-w%=MWB5UyWZSSl*-R*q*vemODzTBy7I>R>!O;$KL{sDs(x7$W9{E&;6`q#Rb$@`1cj6O8EO z)hueZ;xSc(t6m!yvWl2(yh+Tsa!p!(COuKrB`-L-JKk%t>fe7;r?YKW_6OOl*^=B@ zfPwP5@J;(faRu6}+23{P*{sJlLp%AC6kx*g(kRKp72RYs^`ia zWuhVvN@1Q-97u7RB8mf(w-~bpmVVkDW=zxn%kVr+SURF0qnQFJ`kJ==4LQuwiu$%T zcAbq~M|KR!JV&ZrL$bo}?pBhj5@nss#;&um>l7VbHg=sJDf!^0L@|;u+-3h7^F2}K z*%Q`aHoDzsbz|3o({03^IZ5MtyAoD0IN^0KZ$#SIb-EP@P*@)`1ru`$B)wE?((PXU zIA4;DU57k!fblnW9mxu{vFniY3VdC#-$@QS8@tZNuCszcY@;Lmu{y$yU1wv92b2~5 z3$oI5V~gL|;!CiHIWcV3>>s;kr#j`75ocr9nO5bqD;b7_5ki({*3Q4MvFmURVVVt9 zvFjAAkL>q%BH@$68|OwvGS<1dL6x~Uj0_MzfNgILL>;b4`LJCVPCsePD$zU(QoF1a zvVUsUJ=dN#jDFM$;WoLWJ$z1FU}qG1(2j-&Cnk~Bz|ocQ*@1KcQ40UaqbFLFq??Et z=nK>l50O$uAc-~{TF)o_h=PDXJ^OGkrhB}>i3WUE`(Vcz^X4c81Y#zglh-Un2&z`1 z%%O;(Xb_J;(@%_ftXJAu_MxvD%!6Rp-!%0@5I9EQd7$BN49`bM84>WMMBwE@Oc3!g z9fv(Zfy_uiYzo9v1B~PG=ol`Hmx+fWDVPUe=A;hiMtXttT2DH0Hy#Wy5whSO9`U5c z?VP+qOtAKRLiCEn{(;2xG+i+x5v+_XRH-v(9}^Y@P~MK=Q61&BT+l8tCYj71z7!1u zA}oq=QDb4i5nGIf5#ADoKp5iA#~85CtUNN|%|&sdioo7hDIWZUr_Q%6A6#rC;d4=-ECdo{rvow^q7<$S+~vL@KY5(Wt^6Bu3MPBUab!M8Im{OHC%WYk*Gizn)?OshDwNV< z2a^dLZ+^AI$HaqRaLseyQL^k*ehz~?~)C~yG)il6`=tI8Hq861m~ zB|K1$i#_j5SX``OX*%+?;Q#y_4lsJOC0W1j91fH?seXsR*u&bcZyaEDrG5c30%J>q zlh*r5Nxvx9i0&FLOaBCHfRqpM;j@!={ayVIz{k>eTM>Me*Fypyfn^1rbD!D2O8&G9 zc~kt9kRg-bg<9bsCG_yo58}PC<-VfGx74MZP!IwEp@@K6FVcouCeXtS|KBWXwWz;9 zd%0Q%jd%!o$|`4@n^McMAG`Lq#<()f2T}puwE%>TLX2y%RJp%lLH*=V*kcw*DqOm& zqE=a*i|vED{_6cz{q5^o?cn|MkIP4*&TXeUQmOK-@v5chQl)O{sV)WNR(PvtkVSDp zD1vFYZFG{c8d}QEU`(@kJUPnFApkxr%5L$%JvkoFKu!Z5WrsvtKsz|kx(H`zs&^OOfZCzQp9J`%ETt@-0ai_Wo9U((4h#_AF&qC3NTXS z&&Af4k{wBWHj<={&nDfCeu)61S)k%BtvT6lJ9ld~oX6PTZ{3^0HRJ`i?9IhAP1!zz zw^=19lVPZgBBk}TT&OIkKgL)DI-PIMfaC|ZgET2nTW6+#+*kzCJzk236$&Y6RbWD* zzc%tNW`8}X>G6YR87l2VbZNzq2Q!p!~XDm}RlAjVCAu`f= zDkX-Qln}O@5&kze2iYL0Bz{>RVRfDxD4DyE3Z+n(e>pV%78`Sg((UYdt#N6o!#4JvT#KTRj~V+lteboZBi(TVy~4AIxbN; zOZHmDpi}CEx__!&q=k(eblB#_c22%|S$t~*{XVU#1}kRAHaoWXCd>9!u_dz2*Rpol zZm4XBl)NOy3)!JDx8Ng%b$&X_sFu z|KO0oHuDD)O>xA)QGzW!y*R@=f(-T4uKf5?M^Z$p*$;6AFW`Uq@K}6!_$g^7 z$U`h<1%G@hsLTQR{v`Nw6uzimeZu@og_NSoY(axtebQ=)*pFF;wp~^QBk&8@*I;jHVD0*h^<0qVl1!X&}GHbf&SHBUk0KfPZdcy0LsVIyp?iRf^{n$0v0FjF`6R4J7Xh_&s zAZHnhi#~7^$}u&+oC6b+==)Xd&)>4G0M=3^S#%N5mvu*mK^1ZZ?;rm8Ox_jkRGYjjhj@m`TdBL4NlHfhs@1!oaahRey|^)3wvW4%;R*(7 zix7Fylr3Am=T=l~-g9a$!$G^5V2V%d_GW@*EXT9w?20CDE{w`NnQ#uBVernE%~{rI zMOm}hQcz``E-V7lDu2(d(@5p}cR@M32>qpN_3dc-^2Lbcl5|*S(_b#Uy5<90q`xdQ zAp+9S_mX9u3&D43SkI>&9o;qOCC|1TBN%WT_6CZQON~IW9p=*5}-o$Z?e4#I+{yAP^?QO0VZ8T{36?1dkA1C$6CuBfD zQLYnxa29GO$f7bHPH~>CiB*PYF!(tdQj3qS*#EY0+lz_Ow1BmZrVTic&@85Qr@uxs z1p3B#UP}1hj0mDr{1$%bb%NqAl2HdCD#+y!HSvpbc=fezd<6q>Z_2Zwvbct$XPBR( z@yf9aVltG17ocZFKxaYCiZnr`kVb>5mDH?;rF%eqMU%JXI6hiX)vSnVBsHt_03%tM&uzX)^zZpK8wvVH)Ygc|^J6QH^ zD{4P0)-oPfVA=tkYq7kP^q_plFP#3_jOt4THM%>dzhmU-j8z*r!aHF*^*z(JOro1F zY}!gG#!GD4cf3u{Pqv;-K>+z9byn`&&SoN>C}Yo zHD#Z0ut}%Kro>9&U?We@Tg`5Q%x)TiKy+G#7b&Pjfdno9P5xtq?fX`IaX!TUPsA6* zdhjY4iaehn7t#=0In+8)I2^?Ol=BqiCnP zkH<~&z7zZKT^y%l!rY}yi08Oh#WCLCssJxWz+gYRnC1JH(i^Z8_9v|BJU_UN!e*ne zxs`(I?i4nNXxJ!hl*A8M8B%97$;IyWOd1=74cte+EtSSbVe_CJA(1gxLpKr#gFp&n z$`Z>@4Z2|i{03=xe>{xq6R_N66-tIBvArmkC5q5h^8c~l3Iiv6X1SoI*fhOGteW4$ zyfp5vWC2u6G`Cv-bGt#&yP}9eg&wJB|0^^=x(E}2nCr6t;cbvCEx)1#;;f5?EUb6A zYYp75q5P6Ew$HLrR}STThK9muj3=CN9DW)zhO?5+H?!ricOr&VegxNg?7RFC3Rccy zsIP3{!zc(<6rI&l!ygR}0)7op;lPtI2rSb#ecNO%D!E)zE>#BAG9KHUB1Fn`k6Dnu zyqt_O)3;`9lq)g`SA8H{MF)0Qb-W7_)JELeh+9jDXB%;g78&}q_@*oy-kFz0x=NIb z6)iF^R<-FR3pg!CJO?CbnoFhm61K6AV<-Lw#@~&&HRM=c5`a<;-9zWlKO~po3Lz=W zW{BaxrAC%#ue#nw+}ha3Cs9ew{j=G}H{w?RimboY?(dZ{0+xo(cs!^e*jidX_|D>% zApcmWxCQ&BCu9{KNwL&W{uJUCB5olaEhKMAhVpx9_FvVoz-_@*({WpH8*v*3JUM;S#d!<9E$lDOTe=Q+5`L|#8!*PU6vAe~X_Wr95L0I95&DLnvoriE-hu}s7g`Q9 z?I9gB+0QzAw20l*`pHDn;bBiJqG$Z9i* zuhr5Gt>gmzBAmXte$_htbAZvFXI|s0|IN+KRxch7!*P7gQi;wosl67J$bohyVf#HY zq&egP303y3g-@Yj7x|J2#^YqALtvWdZusS@am@haV>Ri~j>+@FV& z>EyDHoGU1i9F(X%+SU_^W|`Cp zT{8UV5N{&>#j?F!Z*R3@B1xGis%C;{j!M_jJ{Mr;a9enX`@6d|%hJxJeDHNxeV|fG zvA%inBO zZ>)4}CLb&xgl%ApC%(Rt50==13g_>o&AWmvRED2&6s-YYwTNKxEvHfrQ z=MT2ZJhEz!%tQQN5dq;;H@=SW&zFAfG&=kIRgeDs{PFtDU!zy9VE56k{^-DA2j-&#mwpD` zqXYkuA3Um|4*Fy~3Y_!+b#O+fle~ypcIiyOyMlXVH^l=u)Q+DO;o1C0f&OSWJCk;d zZp!bK-c($L5lJ80x6_--w_sO^N5*sUI_9HY-c{?-E+0OOBc2qM%|`(*Ld$7oaSP+o z?m^m(;GcM1>g;GV``J4te2i&9FEAM*+1(Q?^i7;#n;-sdz==HRU9uqlK62>bRA{#X zyB@U~UQ~A--)ovJ1K}5XBaE7+ABLtC293II!rEhjj0nY1y^i!bP2X!F6<2gQ>V(4x zLlZU1PH|B?vKcT2x%^~n@-f%VlcH$ z+MUN*aurVht#!f<;#apQVsM`PRgPbI_>YV2oa#6pgv|vV98vKXk;+UYxxs4b+n!^( z2Bo}FURkLWW1LB9_=YH?q(L%Dc8rn9jv>vY#|95X;q@W$q~}FSRDK>}RCXRxOL}ZT zPRfo=_L>!0ub1?aE}V~tVCqBvBysQL&8rVj4qhF7ti9Slcyjjc{OI@?HFC`@n-j-p zwXMB3IIrgJ!bEiBM{^;~rQHamNA|HV)0CA;iRdz|>O=XE9xIngBjroSa-D!^chZY} zY1n*HRb&OyIQu++hm9v`O0qiAcm`4(@bBhzHR zEVIwCC_j@Yn<_u!n=Oma7AVTg(zE4z#qc5$o?nMU4NSgZg|zU0x$wLRR~F(;Dq{W? z;mSU!Uy)Q8id7#UJ>mwpKiCJu<4>=%ENBBpE07LKW&*F!D-2Ys4Tg%7kO2}|htiEvm%xgG@gzuqxW&A_ghO=sxT*bbtrdR(j|oJ9VH7qgGrFqlKdbHP z$Is3W4hY*x=ub1zA;v!PzY#bKtpL*m50y-=4MYcnRT_T=VTTkjc-pWJYLa*yx51l( z2}qMJ!X4wmI8OSh&)Zt{9mgCeO|lD8UZfL-fiVAzO;Vv2x7;=lU6b_ z-wQ^Pg*>hx1SraD*wln)Jxn5#KC-1_rAWQ+(^FVeZe{GdHMXUi=rr&UKH&IHU?Ngo z*<|?EFO_12&2``UW7EPdS+Y$@$IK-40+`GbayEh&ACTDdXxIMp@|##r%kAXK0^zM<~UUtNhuXF&jL!j(>#__6X}w9!cK$WM6s&E zoGP+_T0XWYwc_7Myhs#<8=1w^3jKg-yp}ioR_JW~fh{hemk)Ru0M}+gc$qhz=SSs)%>f?Bh-9!I&y9f9= z;Q}zzEl-qEH@Mvc@`=kS?9=R3+C{%8|Gu4meDP{8tjEuA;XDh+hGdfkE}FK7jVOSp zqqHyNCozbbNDDrHXmiti4I*Wy$gKtSFWeg;qoSBqBKAvMpri}qSF?gNp8>rj#Xb>t zqGA2go}fUIPCrG>X4ui5XkoV#_L^aT8g@{l2se7+FiBclqwr$egIgu#l~knB2dq!5 z1HAjZmm&=Sr|sv-0@6eTP;sXZELi{f(}_%L{2@VI=~#g6V59dmuy?Y%un`e?xY)7C zXeN>oz(|lbEJl)N|FT8$06w4)?oo>mt4J#bod8Cfx6Yr0H>D&w&1-A}xcV{U`6XG&uYc^_}zaV670%44UW1E$zQb*Ne0fyaMkFe+D@-0?+0q&ah__ zJ>tjWsrWdszx=jxIs=mr8NW_$JI#WoJq^3SPnbI{<%nFXHo^!PHCe&#V3zF;My27w za0AF5MYlbl+anaWJxV568w|mcsu{+PW9*nYQL16n$FQW^9+&3$a$n{P0HzG<-6-Gt zAy?%Cmq#DZjsq+CrAe`3nG#&=-HUteF{eFMqh8f+g>ro%r@i^j-yU?fzi%Gc^fpr6 z{Wm%7m4;Ap+AEd9PvWvT7^OE=IrIftG3`97%M1@Dl5k*kBe!IooXC-h^u=iMWDs4q z??4B#^xam}f#mxfsXiQHE!K@n)yMsC+*3>uEasRKo+EtJ>Q2#vsf*Q7$&C^ zr7kA2Qu&q?r>0k;Z_YRc)P^OnQPD+Oq;t%t?m%qQ;oE}Pk@@Z_PVw!T|0RufnnxVA z=M2Ay&71FE(9cuBjNMDX7_rZ@sys3R*RX9_-iU1bQo0N87T>wZiD&~xlS>8bl_T2o z?@^i``AH1o28_xuN2JKwfKhh?MoDRXXJ8bX$_*IJLPxR?lKE&cLs}a!+AM@PC;;;o z!-Z0J14hB_c0a)ALWVe~I~7A5`@NPZlH~zfZdQXo0aBeyBxkpp_12(qT`_4@-2U*V z_1=Cxz5+bDT_v9C*9r=86`iA{84B^j~3`b zy)OR}zQ%t?8~z_3-moGK*DXZfM=er42ZZPvkX|48$N@pzLt4s|OOJe#hU~ zDEzyc+kPW=@d_=a^BMIx4)dcIrC3{kwBF7sp|@-uFz`*=mv}t zLfxaU(S`Y$3x757Iq!Ep=hf>M5BGj}@tR{ZPBEx>do5SWx-ISOFW-B8j^XJ&y*|g` z?0cde!F3(&AE&-2+OuC@H``I|t7l{FmO-^l2Cs-hiYF&WKTO7xDW!(p;OYH?$@$p% z=)6cKd_BFGY&Z`klDKF&V4943-Qi@>lQ=3|1*YR}ax$K>3?yXexIY_Bh`PB?HNGbw zFYbH({SWth@x}e5hxglLe1A0fNV~SV;Rk{&CesUN=}ixAhXW6S<@;5;F( z9~J`-zdoFCgWMU{6R+m$>d;dpc6Unjhwkp)-Y$}5D7dKKHFmRaN{L<*_zrVQyDf&A z>^EPcL*G~btEh)vdzgo|@6v($W*EJtL;qrd@5-@{bYpGFXc#mbf$tt}ew;?h)9r5S z4piv45nk_UH&&si8--}lB}E5L)8DWLJ?Ga&CmusUyh`C6;UJtZA+|rihsQVpAtB)* zZiSPBRmvnz(a&c}&x2$ISU(XLd4}lM3e1*7qYMcz|1k0?aa*82;HX27KJY4lPtWY`r!!O;a*& z*&DRl$4jZZ0zZ=5i~BPU6SARO+w=3u*m*Je;IJ$E!B!Y<1M zGXd4UE^1ZFlaJvycC2im?HEe7fq^>TC$bIR4e|T@3 z2&(%hlaXu(GdLAaNa#$O6epf`S>KcHd@)TnoH*5gsMO;Jlj(#s2MZyWHzKoqQfRyaaZE!o4ro<-R|?oTm^Ea93BNc zLGI2KfZ|SFEA`86vZ(8=i8C5AtR4-Ze` z9yvYP(79%-HQ@0esi-bl4IT{6T;UArEG|Y#+@19MNuo6x2Mh1j#G3$7-3MJ9#xx&XMPQe=^;jbZ5F695Rz>vbC&qxcTEN zXOD^Y{p%RGB8s?Mg(S@Y&Xf`+VF}>GZ-Ww?daBe0Lp1`_hGRsi2| zn_J?0*T@mSD=WZtl?He1c=xB(8Kl~l40u@d0{ia69i`dJN;9}oV9P6YPFBx?C0~!y zphYBZli7-sHl(giV2Rg$>$i9B@7#sb;JQz{AuP$(K?O?4_sy@97wH=XC6^gG^YAQX z&lF_QU!a)!V>f=CMfw;@T>!&;;&5IY^gstm@=1k_Pe2p#{KWa&bZ~q!|9jcG5==h6 zd-?434^JdOTXKl2+ZTnK*M`Vc9r~ z5cNvolEZi^TBhIbo#T9z#Xz>j)@R2nvi+YdMxCr?M1>RXkbm=v-Ra~^g2JjoAVKy< z&Hi966=j4LqpZ!Er~fVlk0wr6GbCtw7)dUhKf3hx=Fi*h?SH)-O+I`6*zJ7l^XKZb z!WwMeeKmoag5mNBluiqCPS#eu;zYYftnGSr>I(1j9P6b%bgWmW)obH=b&4<2z4$BJ z-M*F8iM>!Mu`;3x;q#Ss@oadJT_V;Ns8ry%XtYqA>C>^9v&Qq9VFR_rO8j=2+m)q7 zx*8@K9B!VIrC?JZ$T=5O>dZ2@skc3Ar71O_HC*-vju}q0SIV)kkopr;_{CD55L=xM znPlq|JLAz=0%@e0geX@4Sk>Aa&A8rBq4 zaLt&1>Tk=hvtr_1g<$1cont`!Tj-#)wJUUzdFnv_6a`=9lrs_WT7AA&pJ!}srgnU- zK3}WPE4=R3>T{i>@SAeETdU94>T?Xsg;1>iWC-yOy5DU1*B6ea$*+rKHb0ti49JSc zllh=eo-lEE#x+iiso=rmWRg|rPRLCJEhx(6?9$mV?xG^phSx4;>*=7gz%ePWP=b@` zJYB2L=XjP&NB(y2W;9T1_4(35u~wfK^`w?YwN81zR#2@KR26#mwfg*Uv(`vht;Mla z;l^XI)$5C1M#l6pQbd#KtX82lmiLp)JjR6M7ay&hmGUqcCMMS_;)o?xE zO~xOHONZmm?Rd`a!|dBLJEA^SicS{u2czV0b1;kJt~^NjxZBR>#4lW^eB6U6k!?Rv zTbLdg=SDYj-VoWIjO5RZc!0pL-2pD`Q>IH^{5`~;%*w9V{ymL6Lx3GF@D@!uVR&oeO#J$}L zjypJ6sMvKEw&500C?@~wSLuenx4ZXnPuzMlkLflMq)Jztw1~ z)Wu0ZUJU0)>9qY;LgTI#xgKqWZL(M9aV&P^I*!#15JE(1Lo(eOI7tBoyHplbP1X7b zU;+&YN>OZHXZ!hwopvx17WqF6D||D7Y5~zy33Y6F0Gi8EWAolT~ayqb*Fa5_2hF%S_V9Jz2dIZvj9O+0YE+rf}@ z`fTtanLUri!_@ii`SZu`obO%?-i~8p(wy&Jp6yEP!XCJh_;~(=5$O0S{}0cNA$%DC zkUo2XwhO^|=_TaS;5`TNVDmCAX!{sX$ad=n`#r(KB^UtAE~0>d{9#o%AcR|typu&7 z5cm*M1K}(nAVlzQi174667_9G!4`-h%P9GElGv1XyIVm5P>^?y9~=+*df3nyl-bWO z(ZXGc3VucVtGVF4op!W$g-H2Q_hg{K3Q61Bm;a34s={)+lOdc}Tlc;sk4>{31(L6< z5wtx#@SBI5j~@?yIr;Hpa0g%j)N#G1-4G1q3!cQJHR0*;qwpJ!Nwb35GbZ?^+w#12 z3+ElelH0X`^37shCZ5%%Sf#CH>CIy7JmLT0)^dX_VS+DY#@6nJrG{;IQSsOmUTizZ ziwg&akSV!f&>Kw2S~Q)p7AE6kPI{aPd(L-SA#pf99RI)tfd^iD=A4Tfg3Uw)I!hC> zv-IM5EcyrDpdl;%Os=EAkb57`vpf5w`aZ{3K~x!+fQ|N#2m6cnC)Mc{R~DP-l7T0?Mqj6}1e!?a<>};Xd@)#R3M}fuMEIy|#M1Mo z^9%TLC|E+3dv8f(E1N4G*|=SlNdm<^dp>=Tpr+_e^dqTd1Dfy9T5R`5gYky*-497T zoS!^Iopq3OPd0?|kCZAS=gnlu+}&`#f0_)t$;F!!s2cb;n{+3O8H-qN!`TPSNWB4S z7aYtJz!6GADl;EBFgr+`TxnJH-IgI3m48TMg<2;&|DI^ILFC& zps-sR{(y}{F>({^14oi1uqh}&nl9Q7I}RT%tSaCWkUqd%OO=Ck*Y3wyud#EZULVt|enV#&4xjZ3BuI`p&7B%b!RN(zt3ukpVw ze%}4*K)A>*!yu7FzKflu!m04LOo^ib6s;`7=7YvYm^hw!;$MH=*^K+`226`j*~_~@q}yW5XnPR4_qhh0~;Q|*~i zISRUJ$+O3*o5ga~+gmftVg|{T(DV(W_Cq#W3C1V)bUB#%hCUo&%bi?m3pCOIN4am4h-KzPMmEP9;8NJ=x-ad|opFVT`6x;d6=g-wAg*DP* zDXSjo0+&NmqEcl_R?Z&VEiIkB)GWCu{(g-l*~Jy8dVXcA*KN(O_4*{_;b$ik#Wyz_ zeA3?Tl_R`fpA<2D=5ea_rQ*7hZ;)5_JgjVIy*?GUEa};tYvvZ$r_7Npvn62rRO^@# zI^Axnec2A4CJ%`myJ2rKBljcpmY5N41LR%ioM@@Lt_EC6q4^o$TWTU%P&3%7=bja0 zU1~t9;_OxU+m7vj(tOe0f75*RLDH6I-I~JOiqQ2dVZmjm(va1mh1=8aWG*E+3R9GA{*CR!1>FuO&hzy_%ghC@*(n0Z?L0QI#%+lOovCNg+=Yq zlGB?-E`kBH#4Q5Pa0SByj!oiuc*z2_x;rh){?)p(w0QVrxh?rHWTXElTzxLJyIA%d zlO?%ys${lVmDk3fP0iG2?S|PLDy|?a`I6Vh9|DP4N}=SDnWfycGeeJY5X*IyOvknH zXKnn+onF?)pW~@;c(jTsJ{2{qYva$KQ2lxZV&_UD!B=Dav6TPHNyjiws=qDMBv*=NNiMC_ z_>1npK*?PscqoUNRYazV@-UC(FqDV+HEa2W;h}v)h6j%Mgqa-;2}2k%R&XfNvi2qB z*Yf>NX&f3p;i_KCFF4Y*@UFuXCNC8+babZBI7|q%o8jW#Ah+$7^NLVA(zA_*gV%#k zc#~sBJ5ESJzu}z3&*nH>)toYP}xXEK{Ta3oyf<=KKd z&vsrp@%z|$_tNPtdW%!+aARWmdsKhVlX^*qAPjn}vp6NmyzlzTD<5JBVTZgYoB}~N zc21^A{~w1&GP}{FLtvMcRpK78A44jQ$y_Jgm52K7yXZ!3-H+i_p(`Z{q z{x8OmR~oT=m0S&98Ic1+8iftUwG}l8xeD97PKNVasgyBK--|~v<2vAPJNQ9gcB7!x zY&9eq=rRPNK3Y?k-fH|nh`T}|c%Y5l(2gBQmz0ILI^0mZNZN#FN` znd~+lFW8DC@;MJ1P+`y4P2YFtVFUZoe#`Qpza(sck;`<4RW)+WV|BgN%3N=E+uJv@ z8)qK+1yBBOETx`$VM6>!iEZbFa*6ZYJL*1tI`rP(fg}HwpH}j?*RB_Z%U>IL@3mM5 zLW$ammN~sG)&Y-5&!!hasM7MT* zgM4diu(`RWS#~d@L9#ZebQ3bH!R;XMTM_Xo?VuInTi!Icz;YqIpYDJ!mxxEs-PU97 znQ(~0uAmlxh@ zv*a8m>xE(l2;nT*@*7()mt?*hCIl!pY(s!zsjQv)XltEoo!;pw^nycSeL0P1gA<~s z`xEChQRm&)uQwbg=Jbk_Xe=vyd=@*Y>m>2goO(I0cGw9{9d>|0ym-KEz4(OV@hl$2 z&MYCUY~Uz<0KZWJ01tl|$J0@#@uTPh?gRjE^q!Ki_i)u2n0)Ncc3M{*tuHwT-x(V~ z`R!pi$7Gcd<klAlxPCw3O20YGHRWRA1?`{L~b1Jb1LeE;yG zlT2U12*6|?#zzJR0z3s7b7C(ME|Xiu8qFX;`BXT&do2mgvrq5hE?j{%+{gu%6wwAF zhi9^lH-!0cH|aWoOYHwv*xU-eeEsmgQs1w3d*?4kedxM{-+YO-;2UE*f(WnGLQxR7 zEgxQ>Hq?h+@6kKoou1zTwz>AxZisF26;U8f!y}tI1dnRT`X;##OJOUij;=m#Q9@c48#a7eOA*uNY%7bBGJ4@95-rgy?6QJSX(&cn%vLGM1qCmNlc4yF&B zHwwQf%pxEL=pw2Rl zwq|x~y76d9X5>c=j5W7m&@qn9!Z}fWW*37_86Fteuv|!6OAEWZ;c4~)Bc`Q-pg_UP zD2Q5-hl-(Lk`6&oP%O2cz`C&Ew_G|UkH1WfceMlsZ=N>Ke~f+0 zhN~j_uDBK$JB?n8tWd^M1G(*Zxx|yO{L;MMMJIwkY|ml%GQS9Rv=pG^=9Byi7W&zs z!x@>B2-xZgWX+4}JPrmoODX=u-5y`O3g7?hXGS3#(RA~w2s@S|ZikbxKIYc0@XJx9 zIcr1y*GfX4!4I5R8Yfs zk-;`~$AQ*Oy~nQtpaxh2xux2v@882Jcn)fV;BP+Q>R1$2Q`A($lZm={hhz#|Iwgh1 zba9@{rTvMnBkdf60MBAtI{>m(#5?DsT>^d=Km54;^v8owKkO|Bt_b`BS1`^>;CE># zbpCW>uGNP^7pBUl))sJPNfDW$u{6Z4SVamAW$q`-B?L->4f%b)%PP3S4{7N$!$VWh zx(g!LZZxWt0>Q0egfiUs#iJp&X6DzVP9BdPZg|mue%~mrR(e7#AM=C>;=CL5COt7S zDPN~XqupwHO*7F)kX^GLfmLn}FMMNLOq9=zOsyLX;4bycJ0>tNyHG+IwGV+CGM zGNFUJe3AhdEVop}kYl)|3d??qtukgEQA}q5EW74=ESopp&@6Y4tBqI*&8Ew#;rPOG znkil?HnLI!p9zQWZZBgCFf=Xr_DSb-A?#>%y2b8XpF_DBsa^XZ^{kW33? zYOuUS8M(G>t}UBgtg>?!owa4th_Jm)PsEt}KH**(rzrjBN9*%Wo3WyrXU>SJv_ z$1(6^G3uo1Juk}4kV=Olq}=pl_bw+{QJYg_KweumuVR)kRn91pzMMDBm)GrFC8oa0 z+GH&Vbw>#H}YN`Fve*maZ_e zvsMm9BxA#q?r0<%j2E?W8VlQG#%@ISe*b@d!|j`N^Q@WJ@<=2{%C(fz6fcEVh<{m| z%$a^0s~v7*hGDH-%1fLpomxR4Zm4e5!k0bpeDQ`#3w2E0y>bl!1=Z5JL5QFAz28OL zQfSwc#n5p*$8Bu+?Jd4kg=%Cx1NeJ~K54vA^Nk>%cYbVPdCGSz^&p71MU31IrUVbf*VSlVi{Ys=k0^|LRZ z{jPkvxHBIv8hyWWxFGC@PdI2GOlT%yPsV}Ck&KVhtz;W#)$uqPy5e(h#6%n}+EGlS zjiwsBrM*=R6&|U_@d?fm4~VNdCjaPze%F@>iNj5rv124`Q37%RngWTyWSix)(U8v; z;)c*lIN#a#Eq}u0C8`zTBXnKeZX@cn37=5CGNveW=_7dnGc|6h{_69{!0ov5*$vcp zA+vxYU?|>rx|(u>U)~IZ{l!z;Qnhz?rC%aHB$N~>vyE3 zt~3xla1)-4EN#^~?^Kl(Cygh=pC3MY^7svXeY@cw82XFRA5FJJ-5~yw<=_uL5E2Xh zDlHiuLV|m_Y%NH_-mnj!haX6?Yw|uuOZJAZGxpVMg|cDCGg&Fv z29Jt;d~eIv=jJM8v*QaV!c6v@hZk>OGL+z_5X$x*HloJi=8s7~JUH2D-hS-I^`CY_ z4XKxH3;0om654`8=ASOpb=V*fIdt2S$vcz;I#!5kZsxm(g}7O$%fz!P3x2cKq~~O0 zRPn4Nq7C7**m8qPkW_5g+?p#Nn}Q^^GuCyryCiDYh}kb#@6}4*Q~OX)?ErmMC+14j zWhY-W+dc1a{2j=u5haoLO71GuRC|%j-Hk?H;joC>Y^Z84kv*p6ceoP8U430@`XN#@ zdxW1j>Y}T*G~X4t0C;0B_9MN$HS(JT$f;5h&g-k@^{ySNA0vA=k5xaTq4snEgtSJ@ zcB@C7Fwy(F-^*&s{;a<1*7SF3Mt!|GH+7r(y80B^*xS`pJS6G00$S^JLNbrHbd5XSY%P1}%z5HEHFk_@Z^RJoM?PoXP=O$>)jsmzDRy>{y&r~ zGwoIwux~(XCbPkc2u7Ypr>WlIMvH&)KbUt>?5WpN-jGJ8t7mBbo`i{h_ZrOjmfPWH zM5yg#MmJ@)syW+IE#xy-9kKKqXwTp8q$h5=+BhP1Q}C>yT1O5Oz-N)aBl0xdo_fR5 z_meBxQlmkwELzm(_I}@OJ>A}Y`E>i;HW-{gdA)!Y+#a?1R0|Upf=va_wXj{D!Vs$N z>vvT_q{gNN83lR)vszFCpl6C+Y4wBkA}0gJ z0Y!cC14Ob`SC-MJ>2qV)52ZV}Y8|dR9kn7gsKN1qQ4JH_6S{}=s@PHw)@zrHa3l6v z8=E~p>jvpXH5UNG%X5cJ!U8XiHILPK>sHSIcog#n7DK>&)VGBQaE=A#LqJQ^2BkBA zq=&M0+bsGUx&_bM3SAP_=DV#YxMj^Yv&7w&-@>1iZALbM5Td^}oM7w(4HB`(pFFoS zKl^z5s9bF-*LvDbQF(d$lGyBKGl1qZkh4{R2`)S06fS>l1aX%+Fj&g(qWdoEMZtSl z9Cb4X(#(6;6OxZhym#}Q7P3m;ba$iEmk}CR+dZYfPEk5b@?|L7x{vS)KaDH-jP&W8 zQ84RK+}e>F*9pid&Z6>hbv?)GgMR+1a}1+i>TRFTA@oWuAonG6C@sCEb?VdG^X;GA zkBcXtxlR?^`D?9HS6+Osv7lXJohr{EPH)iEh?!w&(b8%&S%I$}{6mHpS1HLqctY^1fi7&Q__c^p@t3|MvOz_NT^i_?dI4*v_9}4qbT^ zF;iaI8SNS?(G};g#!J~j;|?O#-pAsVVc8dsuHwxU1l#9}M>~&yWv*s`H??I|@HAW< zh|pZBqNX*=BdX8_N`6V%fbgXgoi?>u2Ra2rzmPZ6f6yXbj5-sQhn*8zeom|#zB)G_ z#^^G~8Rg~^*1GxlXr_JN#YX`y!!YnHDyecIkIJ2VLTrE>?!*Xg`}X7obuK-Gskqvu z=Q0nZAcChCg$W$5!lK=fgG0WzmJ3bMVEA=mhlN6*AX;8fCz=@I>P2ftZ7;+HicWJI zLFjoL?Q_w}g{EjiINU;|E4-0*w=GL*g!7{)N?oBY?YF>fVl{QqJT)5;VfQ8`2}*^g zaFHbJXi03nEn0THMzA2@-p}{Zidxu7{I-t>d4~#%gZ9zNOA$I%Y={l8sTA|tDkuu- zOm7}36Wh{wr)%QWrqt0b)Zby^@~Lfx+Otc0&}4NQT05`Q;Mx?}P-X$Gc@M+C{oPP$ zYv=xlkB@)de*5fSgC7v`5`&r_$TCoqO{UIm>@RCja~mGu2*zjJ9Ga3|#sW&^Qu;qP zs4=*DlBfY~>ItBxM^jlwWip7H8TZ)6H4J4ql0hyLqsSAU+-q3Z#I$zu>2ULWFgp<& z!{O#oQVss_^5sv@8vlCn@Ky8aKQXl({o{OiG&_l>lJ&btdNNn7=`PhLlLrS>k< zmVe#0zMj_6va6b6ti~Bmb+>*GtIT}nTcz_PX&*dBW1PV5I5-^~y+8Zs>>n83l#@WC z`GNPir+or@%CZ*efmJ;tJ^{jsDxa9~)Gby&skd57Yxc1=`Rn##)t>HSsTs9>eo>bY$E zm@v0%BU2u5(?K%s<*ADTpAEz|k+Bn(Eo4EH%Owk$yrh4e2$im4jjC(%I>soDC)OldPqF+2w4P#`sjh0)ehK|uoo*;X%~2Bq z)?`=4eBZ$5x*?HM5p2Bq(pfGP=q6m0VqD|CHqm228=IP~N?4ZEC==i)hu~iYd|jsH zKyAfAknBHbU__tNDg+aiN=@-R&qI~KPiT8?8;2E@rPP+QqDqax0Y7$y&IqP$zu3Ph-sbKNkX;h z>CBOwKy&`ud$sNS?R9cYUK+>$JIM%~J+vy5@rLu_33&ymK})gIPe?IC;v>14Tr<)Z zB#@GG<|LV>>fp7`WuaJF4Eiri*-=hq*ur}aH7D8@uz3{nudxJ_3mN-d`AKy9tJz_P6UHnseY2$ezW^+po7|@Lr75 z{9i$v6oR>a%gi6xf_N`t$gy$3TP;8-$5vwSD3r=dr}GE?0XwPQK24phqA_ay@^kBP z^wa#~4C(5u8QSJMnsf%kCLiYxnwb zq-B*qdJ=*T642iR3g`rUk^6~Dz)4op7qfy~l8Gj9Cg~kS*(`r{H+Hj6G+!3W{8xwv;jr3>NDz+O00t?1 zx(+vgnV)xGeAxTtE;jq?KJBKPeQ9lz%^uAGJ_u1|xVB!l3p((xEgZ!5e9F9rF+xXZ z9Eh|J$U1@RN3mMUaU;>%g}SU@R^?K(_JM&O{H>t1KmpYuivEI$Q!+5}sAjj`z zJfC)c2!9zTyT6`(n(y;VFL8couALv!L#+i`0FgO)P5B^B{5qkULEBWp<=5!z6e`Rn zXD5UB0T*n1psx>OM*Vm!ZUzQUc#)htPZBhD7yopAPT~`}yJ%;Sm3cmWK#tzcUl)Te zxr2Hd!t8E}ZUgF(dUO zDYv^!ETW^F{~m7s_wew)PBMN!5!@Gh5Atg$hj6Ii{=e;`92$%f1yl9RQIpz1Gv$iv63{w5ydlBn%kJFq zh#!rFt-*NzZ^R&@*V!SoMwu!_dU`tOEXdnDJ0fXQXP7K~4{?`V_FA6HE8CzA;*yNu zMAVT*6OIH+t!ADZ3sSKY^VfK9Ib zv>RfRe3=wT5_lnQrEbK8Ents7ll>j?KD3UO2|jEQs$K|F#jHcM%zr!&RP{KYUK*;3 zHJr^s2%Y6yA@|ryTf6`h9Qc~>u(ZUaRE8~PDTB1M#W%x^w%Fq1UXMhBlW{ye(q{R= zxIYmbHG*bj&Q`;EmM&bsU(Iqa34`b7M;v;S$&vEsXQpWEW=ex?dvUlq7#&mk)BdAZ zZw8Itc=F4KM)%2GQtU4uOghhdJu(a<_CRWZsxb5EpVEy31l^+J#gwvYl^$M?PPErkAU?=Kh_?b zf9Z*?ralv0FD=YI;t3$n)U8%NW1Ueym`@h z`{w!lV*BmOo&5)+VA1;LJU;#hkW@%=I|D$Yu*fzDffSJoQl?A5=uCL>Zc8#6J>2AZ z9~*6yD6e-WN00XI|I~OCJ$;&V-ap`Bl_H{g{lX$P+>da79B%$AJpS?ZkKH#1Plg<> zPk*{pM4!NoFDoO<-bWlN=*Pu&@0VxcyPuwZoIiO#iXK@)O1#oHxoBq?XL-=c?KsOr z&dqWeCpXJw-1#`$8-IHK?6qu!*{)j?Bc~4b*#W+G$ECf?g5>N*R>jD%bmIq}^Z-Xm zLaM0W?)C8>Ps{0WB6-vFC` zcL-#=5AKgd+to{OLeY*UodoBQA&w$l<)EQ5I*i!gGm6zQ@o$d-vO zo2B*MqNSXVmB!4h2lZxP+5|UC4QTTt-M%~w)a6j8qi;9Dvg!?A>w?$1U@`ya+^kw< zYie{;PrXd-R#Bww1m!y|-R)Wzyw(M;b-`!ouj_TpN6HpuA*fqWHES~QwJvzA3&wZp zYco%mxGmCoYGKEdCvB zbnA&;cJrw+dMF(z7)JejwAKZ$b-`<0@E>BpTkC=)n_0nxyGj>qK}uyA{X%?cOaw=8XtuOsJM8YPrx94f@+A&eAX#6=hqN)CdQ4*UDjr?J@G2&=)ov9{=ICV-9n< z)Dq8aL6t{1xENg}jyu8y#O_1b1dCN_1&*Q>1MLy)M^+%DZSqEwH$hU9EaSh*rKZ3H z5D}6e5k6!~nO(0HjVv|wuM9aH@cs6210>yA8(tn}R4!S$&=icAdmj%%y0IwQvZPKp z$OcySTceF@jV~elO#%!WRusKQ^kkvb5-)0y7LRwjwAw&zke1YlN9I_!{rL=sXRpuGrIVNL7gS zbUMLJV^y4=#QJZib?ajNQvH@`O3^;migZHx?2%b2Xa_-)C7)z^OkFeBT}5o>CL5Jb zuWjEzN0@h{FBrZ!M6I$g$h%-PXup41okANneo=!-4Y^fJ>e`UV`s)%Hhj*ka2>@*_ z3t64*Qtr>=>|MV_+?>~w=sDLHYB|0`XXo+?6FmnFE{t(pBCJj>8CPJ*ld3dki#Qn1 zC(Btwu@NjVi?OfICeC<3fZ%i>KV(XfzRE79NGm0;)5T;lUvv`f0JfZ$uTH3L2lA^> zKji2o0U>~d{i*MgIx7E;^x5|S>9g-d)h_>#B*0lgz;DmLL${ZIm*~Un4<9bQ{JZ%y zIx)YV#xZJ4ZN^KoX0 zDXee&JgeDg!?^v+4uv_WYWS-rfg!gq<55w^lG>;fglX5v7$mIVw;UG&eM~7XL})VX zun=~{a##qw!R!LXs1O);*xIeA5Qz%gKASpz%WX38D=>Ru(JpB#-f6WBRP{4r47i z+$^pv5+d^ZErc1=%=|lhneNuGRBJi#YApvX4>caiINRvxq1g^Cm0r1!>5=O_g#Hs{ zC4?pU#dLdLmelIZBC;OW5_uAB9B`ZqP%T1Mf&$Ofgq90U;n3va-j3$CC|h1oFWRa! zFAWSEm<(WPA+0UYK-}&U#?Os zaOAF-tB8v7l$VQF@`uHEM+A9NAjgyDf4c5upym zXGx-)*zuMx7E%;SO>tRiF(vpaG`a9Tbi+%9brJs1pq%--@0i-x%f>sGnQdO|eCK=2BiRG(`*8WC&oQyNj|dOX`Hf z_PoM?8InzmtuKHazNJzKnTvV3&=d~45>`d{qD!0LvZPKpY<#H+-O}n^lYbdwT!WWh zvau{@<1#f?R*K-6;ixy6jE{Sma*jq9z4(IjZI@J6`t!j9QH#di4(jp_D*{)Wp(|CQ z5F0|L-Qjb2Nr?F3uGkeiOg;5SQ~qEbk@ni*Kh>=r(ND92@kd9u!_FdYHl?+8*X6MY zSZQlwwnTzDZ)$A%l6kSRvpFvy{AQSIWonp~q1-H$DMQpi1tb~HZ=fH{J95ZHmx9y8 zFdXr!LclDE<11?BEtjenTDEm9g-JnOd9q=O;s-$%!hda&JD{GhrW-+&b}9Ggrl@X9 z<^ zfpMWf84f4s2}DY1rjyVuTLW0Bi{=22=~A`_u>NtqXb+GS8(g-Q4lxKk=Z$tF$8Bvz zjV-sC(@H9m*|kC7FJ=&Eak^u5X?wi0Tb_qXe{W+UMd=pR)v?uAS zxQSUmsZ`3tjJ=$5wxmwr2;+)}nrJuD(tQ6eRjI*}ev`)PPlxDgz_?^20qpza8nKSD z!pl^KX*6=oMwIre;D5_5w0 z2$crkCG#lnQD0Ed>d)z%*1iRlToKuBdtj9-4^lEZ;&Sxc1u zZj}tRr|RgGtY!*Yo$hs6+G?jsSa}xC)hZpWPRMHhvZ&3mHEI$7(R$f<3!+I)6{aAn3KS>^dqY|R`63HFC{O%zD&VGs!4^PY4Fouiu77ibQ zI1UvkYB{YJ4SO8SQ55WqfXc+Ss3e2 zQgYV7U=_6`Ob58`TPDkjQd8VE435}58XhM1kfWa*8|}U=*QgN}BN(dAdW$eNGKsz; zTuL1&?SpEm&=eM{CzA+|vVw@&^*X^gVy-&&q006NX=~7vL{c@d0~@2xpcP9^(I~{8 z=9fbR^MX3zAl_WjYjP@=)N4dkF*niBo3fP)P0=LAgbhKIx%SBB1vR2^5)WWcty4Xy z#Rt6Kz-@ytOjOZ#MH+&Y3r)eA*a%s`(N2(S6u*CBP9;JY6ONr?h1s4M!-lNW z7=AgF1-8dVxzH4j6`rhyXp+j#1$jx0Z~-xhD}-bD#qv+|6MGIDwItUyx~`(s6pweE zDT2dCaSG-I^}u8fP%8j<=LQ6E%QY02C?Zm83>UynU81Cwq z6r5Fv;-$;f8!lO04Z+HVreK^98zJG`WesLlP$wG64p;aG;f6#UCL0)mcEmO?<4oBq zN-g0yq!SXsS}a>Nk+l_y)^e3PfpM-`fn}Iu60eO0P+k$-soIvKNV(J!*k=TA31PQ( z!#0bRc$wJCAx>JwDs=+mM82YjLzicfd75nA#53bMVlLU`LQ7nH#oFQ~QhH<91JaUO z;h1stGn2SYqU^!3Pzm7H#v5(S%yOYA9J->2;V=8JSn@p`(Pa;AUQj0*-i<4v74we}H2ehADFC~e(QT9zOHJVb|D-}80v03! zy#a#@z_7)vs8S~|fvPKB8Z1B<00?sh&WQJnfTw(AEEk&M;^qdXM%1E^c|nb6Fh8tH zDUXvs>1iX>ogRi%(3a_GQ!X?`V+yeJGIxujcM8gkw zl?x|xiJ@hLvG~v=UXv#3uPC*I^Tm9Hb5=y~Cx&V0VoG9`tJDY#olAR#k3PfA358<{ zueU%5Yi}+Wn&OtdbBofJYt#wLTCoCqGf5@26ppkUr&wBx{=218Q(W}dtYZK#oP0g; z{w*)*6;3p&=ig(h5ee@Qj{$(s28ohqTj>`Q&@?Z?QB1&iL6srMX4zqr&Ia| zbppgc?2TU83%*>VR$SKJ`h^|ZE}%E>ItLpb2_QBW_Hv;qtOr~ozN00!JbP(hmedI+ zr@eaNI1;csg6~6lsL1CYQdDQ zqSO>mbsT!!(khu3)Cm_b=j(e8O@a&p`!1UbfjGW-2rL(xqH)TXpop^mFfXVRjs15; zsDOkh`QkFpBI4rvVCfIbg{Ek&kjW-=O2rA67u1O+StM8W8ap_{$!NOhuz+U9e1n6l zP-u#lZ^A{<@`5_iK!Pho6TLotg;38n2Y@%(D zEwo}*z)HE$6i%q$Bn+ptgX9HuqBZbguNO^*0^KVo95E~*nVdt6p)Qr0!r`ul^yr*$ z(Z?W6gi;wVmaEhX3^b`17|;#r8BTiWxEOD?cthTDp((C-&+%djbhEdRyr51robFa^ zZvZC_N%&Zy^<+ z^xG1Bza08j|5biGWds?y?rqa-k_Kn>|iiFmV)R%nRy7 z6Y{ru(O6UDjD=cWElG;kGavbKp(UCaFF630bhRi`bwQnIa^9;K4a2tdS>PU!@y*O4>sS_BB)QZJM`^sX!V=ab$$yS<`3r%r-6$A}-VtKNq zT$a=cCwoABZ}N(=bYrzAMqV6V>`ky#XbQ)H6*~v&wW4f!L7iwA8CRTCn9DHT0%);C z0U&Loc)8FN4V;NiNQ{_vuVvTk1jAUrVg{npKsyb1AUGM_Q$d4MDl`RCw~DQw-G}W5 z^>b0ga*aA+nNcZXw7g2oe)oRvRa*AD7mUGj$ChTUkMf#H*4$+&KP2_L#~DC;ha_mI z&UZeg9XcOCq&n3Mmc6raXEMK(#PqB)rgXuZ-96mIcbG2{Jvg5(#w6x7f(6YO&OAP} zDm`3Yc$u$zIzy`NwIcV$>mb=FUV# zC@{Rq_$aNQJCPffPVuRe{?y_RJs!Y0)zKY4pG>BtTvy%CowmRHfg~k$&eU_q`ni8w z65J-^2j_#+!CBH9#1AG@a<2z8~Hc>?& zdkab1U^+TZq}__#OSjPc6>`G0AMSIMTZW*{2EC*cPmem2X%AP??qoP2GUZ;WbFzB$ z)bR3JilX__Hdp~loxDtPoQ^`?MKwfl*@&sSv%^iaGA?95&`z=&@P#On%_x5N^ypb{ z|JMDyul}$SCuHh@U%38FCU0Y5xJ^<>iTu234&tSMcU{tKCMx^n(J=Xt$TChaye#nq z`tt{`xU8_3&Wc5R$ zv9fkNqNTW(ARub0*3x!}gUZ_RRH=Tt@EYgOT7^>7N@-W`}U^;MJwSG*#)OWYs(Lp@FJoEzB*co zhwkp)-Y$}5o7G3!3(S){Lgq+HN4=o zlPu-A3A*4Yk9!Ep5%FwLB!!S#H{Lye87zJn-8#Iu!P9OSMz&mmBz%l;5WMyIbQsUIcH)z7L5c*$J)UQ zrSw<2gR5O~yqrzf@J&OoE;kdPO&oi#A4rY}8wr?@k-|zHMglXBB7E?1IH8~>iQanDwr53keZenpi|$gOH=5nJ4ZoPvMY&R>J~K~7|6wM zy#P#|Hq@C3fZAIKv_!03`K3kzx)nMjbmAQo#`5zU7%UU{Uz2)0~rD_SlqH`09Q z)m9%>^w0^b@pRBx5R5-NO2%<#Xe#e!N6Vfo9j6V{k)6x;qy5(YHXWXA(ncL~e_T-` zS7B7wyMiv_&j}VtnTDm0*gZHMg+IRmSa{<7`tIjSqj^0KG+u3Ws3aL3@+6wtvp0vfKAsD8DsLkuKKwL}(nj$!a-q*u2d zcs@T7yT>j4SXowCFmEBizL>|Ou$g_yT~)=%m$r5T`juY*CjF8jjKr*W!4Ss4)hVi0 zFofYJhwWcNlrRLhaF8<<5amK1mD_Ky6ykn@MYm*-T$a??(lB$U9tGBBX=3okmfzlT zYi()h$;2m30tYM%u5D>~sq~T7qWutG#B;C>X6dp1=Tk3sM#=Of9wqbPz!{8Z@t8~$ zv2%evNN6+151f~B#hDF04Fst{dG%?Q3@Pql7H|f$c|vxQWSo^HQz`q9ILE4TLmtFE zLnj`N;+cMG`f%rjnxEoPJeIZwa^nN%k*+_EF9ZvUq{zS8@;5u1Oi!orP+OobHTu2$ zG$ak)CyX&?@|-_`>5oV(;ouiP=;b8HuF-YrGV$YJdQ2aem1VtnI7>!7K-2M%>Q{!y zJ6Ku=P7gz!43pO%IAo2Kc{9dl!hi~T_M$#hXQanb|E;IkMpk^PW?C;XFX)QN>0oGI zKatr>d&78Rw3yEppEexKwDP;g4wEQ*v`qI~#mXg|d^PnQ15(u8Np!IF!atvko&BT( z8LaTZdf`@JP%L4lpbUz`KMsF-b0aL5@~|b0gEj<*Qn87Pmlbpm zK)|##qj@b?sRMRsz*ApLYn{I)u#?1H;+1W=HZPr|vQp`{jGQ!ZyNXgWL0^R;MtzSI z!(mJ4D^~-kxI^~e2bOJ;^K@^~o#R5FbjQ7rFR>U3$*9xdIRq=9LTomK7qijq({|KJ6u)Ojv5R(4HZm1>K7^Q*YKB_A z=dAqI>dv;;W8$FKSNDj_k=IA&uF)GbRv_Iwas0PAXX9!J*RFTms_6`1wVyT05@xbQ$52q7inL5z3Hm^8midh z1XaI7*k1enhG>|ZL7#3Sxs!2&7N)P>lBGytKT<-rXd+DHllO_c2t7tTRvP_oDdEjJ zO5jFQ`h{2u8kHPRGCq3jg&j6e_w6mTTaV$ek_)P)J>NRfcxSr!SNo=s?FA10-1^Nn&&?PsnTfwq;S9Ojb z=yZLVMzX5m3Vr64oZMElzgn}r*VvI%RZA6XW?n9sp-bJ9)w9gdd;9XA(b^wnhKAZ8 zB83gH*u*-(ZUdU#9q<19u)F;@L$kN2S-$quZir^{Jx9WnMX3udOi?Q3J-Zvv zc?na4U6+6wSa?}>v05K)eoUVHwC%knLS`}@VQ{!b71_0)c2iK1uRF@O4gN-sIF;1s zYgF-dpo)Q5!7f*lrQGP&!g*bt6ABO9C92a@5}22kiWpkw^n5a7WnP?NB%Dm0-ND)H zaC|uS9ykxDaYz0>9QzNP9S-2}aN;~X8FWv<9_o7Vz+O6HRzOLtVp8y;i>5_fN)jqDP=nPQDM+dDfy zOXpg8s9D_UT=W)Puf!Jjmi{-LAwglo!G~lY(uX?8`nlLQ9aJ6^YywCN?OtWY8aWa(R@n6CrHBr8x1>=vIHSO`wxWJ!dAx$haek+f8HlFV|}Y>Pnw1CCP7 zYXeA2Y4=_104-0B;iOtoDa#sSLogDi0h`q!UEEnno`#hGN=S?w)Dh)=B>BV8;A!)O zuA^G6ZikxMgrW>OVi;*Q6J<788d7X|q<$+F!%&v&O&N(w5^4*EkfSMiluacyyr?H? z(@FVWsRId}^bh-@dW!ngRuNaVBfkaHmbRH_>!bQ?sb}dP(8>4YO1bwyDXL~_v(zo< z{#V~KD`xf-0^pR)$|lp0r^E6~-|`#9Y!!-BaEcqUHT1};QPB3$)aIjXL1Ko2S3}j2 z(QHOt`n=JVXam}$eX(}XW?!;Wb46Vbl?=Kcw1gs$mLb1ajh*=Ob6t&5GdD|dMb+0M z9_kU){lF6&9hpD6H9qyP(^sgOncC8RqqGr8K6q`d!r-KSsr1HWk~BJ9c}gU+i)U#Y zm-^Ic`cj$mCzfSku1fnInfc7C4sTQ)MA!5jqlx$oT1Mu3&NLbR8Y?3HS=>!rrw)K&GdAB ziE8s!5j`INB|u-1#=SDAMJV%exG1V5F@Or8v~{$WpcZl4^Of6P=x=#OP>br#ugz_5 zr?uzqnKr*$f%A8a=Ifik0+`>2H_wcL%|QAervrD|-NJgeg zWe?Fd3U=Dzu56_l>i9FDAYo2#!CCE-9STNUc#45Rd!yaaFYkXmx^)#k(f$C%}scU`5N7IT1nP z*9=K<(?a6{eSJ%%I$B2kl}lw6&T?FPr2Ftl4k3xD3Na3ONnL0IK9wnxv$YLe)6C4N zg{-|ckFCx8rb=Z))Syn7-*8y#4Cb?Q0#n~B=C`(oRs_8vx(Eo;1rk7^EuUe>c{rI) z#<9RWULJi-K7G`1Tv7NhD`=@&qS&-sNo)CnM7U*HtVrY1$8zRZ#p>yLH4xgv_|A zU0+a?RUD`$(vDNNWt{{7rf&NCbtfxHtj5v*$)mN<`bVk0$a+uT(b47VQF^>|Cx$MX zzZo=b7<|>k)V*O-{lTGDp08x0qWVKMXLWiyGUimEk`U7{eb!igF}$YBfG_EaN%<>o*q6uQ}uVRdnEVi z(btn2k1J_O@IJ4Uo=e6HV$S?@HPBV5vlL0DgXQ9|J)WH`h8X0J2R*gD8i$0PcsPtZ z$MN)J!`Y9ggI?T;g%GnwLz=yB=`TXjnZ0OfM;Py-RS8 zCe^PExo!FDRM+VDPcK;Y&)09FUS4$Px*NJ*Rf#FF{vPNT&=Op~tV&kj7$Gbzdgai+ zLW`qkfnE*t_rCG7^%|wu@O0%!S0ud@l$R~NNa>}>*Aq?sVtZ}StCm{x(qYkyY`U80 z0`r&Y)lQ3(t!lcPeMz^O6pReb( zF`C&G{dC>1l7TEzw|r`BO0um2ClvO~;<2$ZPzaMo#j#O9I;PkXU?5P?@gv7=YL7xTWRgo6LO+R66w_1SQxcto#NggZ`^g?5YZG)HF^R|K|8LdSm>H6gA zBr^>ekg#bKNo)$GvesoD8-RTi^QqNv4gdcd@L%J5XX%4FSKg-W{h!~*&!C4S{S3OXb23f(I3VN-S}O$puIRb-t8V%8W(3?> z^iE~r7tbuhS)heGD{NP$M^%22*D~`ZR%dkQRg5B=t2W*?5AYNkXnQvqo`{ydppWmo zKdV&2zgbM-yMbQl!g)nXgk&}wOvW3pCa8J_-GLBEQ4p)Egi^nV=M=|7yv^}2pW&)L zlAp zaX<*}cbW&`$+O$XYuA6;P4Qa3rU^`o;DsUMb7f4_Hv!WwW3yyS#L^87xT16s#d04# zk=iQtp18(Vxhh%C`=)})(thew|6fV9TA|?3J!GgcQoWW^z7%!sY{*hyBXwO-_in1t zG4&%_WdoInz9&^@Yqf~N`{@;>XYx{QzbZ%-DXC6PtGUe6LaK7pij-72tu^CHahOV1 zVJ1jV7^%`mHHlJ3MJ{!RlkqWRgwv5TTXfjL$GF#S_NI$tC+=`?Hap1FhG`8~s{Oyr z7?Ej)l(H%9U#(bEpQ%Pw8xksXSXzISDo|6sl9ESjSu@?E7EyPU{#139iv`vMRjgHU z%1a})Wn}gORh<8hbj(uanZDJK-fQSfYV1<&eyZM7qf(qVMXdic{`Fw@#rxI^(aQF! zu|C|aVEtr_lg}WT$uf6B_}M%@=KH^|Ve{L#2e~P%nOpz<4SLOcTVkr>WC~KlA4Ou= zONk5%rHHt+IYQsDfvY^vk9$2N6Kr~<9&j`m_a`Fe2xiCm;FFM;r)=fFr!o{1t`%Z_ z-@_@%H zV#)97nM?Ga=5bVqy2^ppRnG=p%Z+@~s+)W~!{v%iuGFrXAJROO;EbcrUwNOInu)_r z)vJ`rq2c08PLJvef9n3uMla@!(UmhNW6PYud2NH_*o=2?p{|XOM))5Mn1pyZf}gk_ z3qw4f%e4IM9!5^8(Yzlu@K!-_3bj3x_{egju504b88{`bJC5q3^h6B=N zGL?ypzdt%lj^Dn1M!yt${-)BM4MCTm9=)0Q)1OAK8lztOVD?}XELyox|KSCJoLIl* zYU0a#bWRsby62PhUu=!kVc@#c$(fkBA(9onfU3!_S2gJYN!*s!><{Krk=WKDJ-xc3 zyf}-|>YBg*E+XeZ%v_|_i;WRoyk2b9i;cDXtrr{CuXI(q&SJxA^XpS7XoQzYnIJ+2iCPVcHeVO%1U4G582|CEm@ZxBejAtOd z4{SB5of4(arKvYjh4O&Q&hFMAs-5bLCv?fl3p=>_TOI0+E19UqrIlm%pVU3f=z?M< z3KKce`PCX=1($QB%4OQ)qU!$M?`kPXG0Q2?qL-$mJZo^x05{k!uD#Y8~p`xoq5xuYY|&V$RG_ zad5s8dr5ust3CaFzqNCF#IB_-ROZwyOd zD3W5hw}+@#vz(ZHG^(kmXfH&~oqq4OF<)eLT*W6`rAH-~g5QvCyrIv;myLUrj)MQ$ zZ8dOS4uzwf&8-rUWVs#6H_)@@9ci3|=zSY5K?|*>kT(GJpu?-InYUc3UTE0{7Rynj znmkmrNSVIbgU}jgNFPrCNJH?l%u$Q}#a1czn?Bj7@_YS?NuaKyd;jl$oo;gKK*l#t zdd|tk0y_rR`W@#&j4DhKj10YAv+UCGK8}-qe=4z*OB{5MTukrrw|E~X(w6*~X*C_~ zz3*N;JaE2yvc31i&u?Ei-#vT#aQlt(-OjVeFP`$x0pB0JJZL)io$p?~JUG~U{lnh& zu84LbY=ZIW!Wk`weI{5c^!#i9YM#x97tXjlSHDZrm~|FJDV`;>_?&-_M{pev2I(E@ z*I|R`LA#Hjw+hM{<@igXYqlX-r5~*)s>mWD^Ib;$(jy8vRpFCclI<%eyXyvDrW^I$ zA+og-?cF-jw#YogdInP!w1k+Y4mY2Nr@ypb&Ys@^Mq;78-qUV~kt!BA0ZVQ8c!LPi z(+aTkO?zj2H5wjkLrOQ4vlw>IG5f-MlKf;qLK$xRJ!K&7_v9$3x2@9tXKh=hQn_#3 z8+65rBd(gBQpMdee4EKKe|e&*)-Av{_{-g}jmw|4N2dun;b2syB8G-<65Pm-?k+mX zmh)~h?TK^Z`~))G_+ra>Hs~azv~*s_C=TWioLz$`^#mFLYO_oFPv*k$ zKfO37w`KyL>f`}WyD5GfoSk5fz7lQ4;FYTSCt67kf*XYmmh1eb@AD39W zC;5_6Ox^33NhzfE5&sK?wBCSB`;+Nh#(qw`*}+egT?(0Y_un|;*qlr^9PG$2OH41w zw2C%49&eyuVkS&w0`*ZhZ#Yl{Al>`UxF#U>F ziM*ZN(+$#k_Qgv~wI=1l+B+SfgPc8Z4yFsVm9ovq)VfvJ6p+@Uq|6E%E!=8t=meH3 zYqNq7kX`|vqR2-bX2B=nt`Gt*)L1^stfooF`*r3jD*1@s3VB7n#wpj<{80)m?@yU` z6ISk4MDV^#c4H2z9PJJ_pLSloK6vvsx_zv6{iod!tK|!%LTLOxtcu?*LQAco_z>yC zm>kHhCBz7GVhM5Gt#FyT*A8=v(6W<00*4==IxRp0*DjS6RYQX{8W2cm@ZD$eF<(9e zrcK@+V${r$OD{28DHeF#O`M0oIIh^LofO`2-PrS=O=b{%9Ygam+i)IkQ>YEsxlTO2 zOB}f@PV|d%%j%Vh(yN!*wtv2F2BavFsytXXq z$&guJgr*Fy0cICk=KXy!_b|({(kwuwK&0ul=p8Ye7VBAoEI#bazDY=B8J5V9CP~tk zZQPk*DTHW(gn>?uz zH?{Kir}RIyTjo;JgU`>MXnN}N3`C!=>5+U$Ex2;34Rv`bubrXk=?;^N`N#2=gB}gd z%y{M`P7F6`G9%yO#F-DyCR;*1+Kp$ia|RbEERtAZHmRRpAa}Hon-(J?wimyaw)x>&U>ZtA<(7Qm7LpXf7JI5 z(JuM-9x5Uf6GRvF6Spq|1Oi%S{hq}qA5Tx;bKSs)J7niXx3q|5Krht?iPo>p1No$N zK3#d1*K0sM#$Uon!#|k%_)M!g`Y)`%T%#J5C8~`-Iu3kZ@~6H@-`l!yj@4B%0Z@mK zAZpN5{*3O$|D|{9hP50RovEE)p|-x?v-?7I6195$P!5VWg0PplMTNlif>%_7Mg9tE zq>y<~#;%xsdDkuMDks3_@l-NXy8c$!Cg=6iDQZn*LvF8RiQ+6g`RCN@;BOE%Wcg^2 z?E^C7Ce*Fo{r9a-`0DnV*-=nm_h~nUljZeKYj~iAfX#*%w}dRe2^-vXG(0+?eYDKk zW@>oo1?Z7w+*@oPUcvZNER}^$Yk2My1!NBK>TI>-KxuqE6e#J+T?qlcE{|c)51=|A z1*}JE0=EHek?n+2DrJmQd@xHEPCObMPaFmQCY0FQe~JmYZfyVC*lZ>XG3E{WMAs2D z?Isd$?k2&+`G-Sr=U~_qRZ~%GG)7r@JdH;>08q6(7c6Wmkksj1aim~eOh#CaK4I#? zOJ;^r2|Le}YrO=gptvW0F9sl0wO1=&Ahx^1O+cx-BK|Yoq$$QG^mCI3>CgH8coYwY zsrKqt_jQlJNTEq7?CqYAmfM@V5Z9HBL@+y`gDN`MgpqhmTf4h(tf<``nmXKTT3f*~ z$IFd8Z1keIcCg8mN+pX{rc|<&MPCc+3EsMLJn%@lf`1{{xkX&kySE4L4tBz&vgnp{ zMQrZZecDYo_wv%FH+Oul+sab?ZEo&Y(x5=mSI`BfJ1OH)ML9ERYIr_JF!K z38u2$k}C)XG-429d!gcWS(r>`l42>V`%<4mrmB|E3*<5UHGcN04y5^=564fp2S5F5 z`{(w|+q*sqKy5qA4ayAcxAYYmQqfgQf&Et9tAz}vn~)VVScGB3zMo9yLwH3`} zuBM`1ld?Ut&K7dRIO#LR%yFenzPQDgtwLPMM9sE$F5P5nFPce~hr7#%qZR3`u zOW74ugV6W$-qCcBNc=WL2dLJ?UwD~eJ<{LtGstqTOB(flAJ+ns?r;sWJD>O-Kut*r zZD2HjZxX|P|zJhL(p@Y&GAp2h98gu1poXBt7Q z=9kQzG?u9^vn;(W2`l27l4A~HZ($~qAn~OYYXv8cJl%8WNjyT>RB#XX5nO2a5wK7r zc&ioS9>UyJ;_KO*&Tsv*+JAelUx9{X*bDnjHNC^|@6PwzBG!w^2i$1>mUTzi_b{e4 zBfmujj5|lqA8mVhe^Q|xG{1-7NX9pHp zt6EcxWvo5pUY^mwmp92asii07#gx?8c|Fp^;Tco3s1evw^z}YCS>*mQ%`Gn^_fzgK zb0dq|o%?#PW7h9BW*ssgA3kvY54H{#+QZH5c{K3ngF8qh;5V=Pv>Th$N{h5IsR0B6 zxApBtM1A$zw&jkR?ufE%vINWGY%bq=Va}HJKz!pxxl!J~4INy$Q0_7!)?0TFu>M{) z)EbByeRjaQ#0HQqRgu|9q?-hLXn>SLR9z}!ECbeIT_LEr3(NJW)0wDjw26QXyi?e@ z-Na){=9BG|E5ah5_>w9%QN;0yVu?@09hN#ERyPUNr^*kSxT+mpQ(w__6!mr$VMmKM5#fK!ap4LYezY|L5fI(2B`N{WC?sFKLKv@E86to0+bnCbyqK#!fBmer3A z)l>9!-2txZ626;ax~Z6|(}bK!I`Dt~2iV~zDkpIgn7yQ$(8B2l2rkQdJMF{LPg1=9 ziJ%^8>QB|%x@CP=`UCWJ_kS!Iv#z{)!@39bz$)?6elwmA|srd*H)cDs)y2!u=o}vyK`V`U<7=a(>kC8$I5OJcbEMk&^C_B!fJUjDK;=r9{Awa4pBt8$L22yisx{!;Czq574K zKQ0u`#;G&AL{F#4>;F6bUk*BcpE2#UJA4v@s^a`07}0pl`R+N9rEo%VldW;K&utVh zb+z|)-oAC7zI?s6{la;^{p96q$6EoyIBWsjdP#%F=HGey^6wlrvhR^uN0{QW0tq(E zzXxUkr>VTatdx?Hsz-;YRI^HvaYMvuGrw+rUq)pn5|EhHX-Iyy?2pKNn)%)MJ?70( z7d`EuPv2f#K-zyyQ(dox*1Ch0hwpw*z6a1+-1itGH@a^;EZ_i{KiH5ivVNxg*sQk= zDa1gOUpIb!!>H4cPVEqm%x9=1>56{8)9nPvLr{y~mVdAnh*;fLA7Q(o5ILEqx2YTA zZeJ42btCELPAo~|Wco?1Kx_-$c30BZb=#izKTNE$ZhD#=FYq#>>?Vjj02Y-nIYxsb zCYNw*F&WwlYWOB3H$zcFO+=?JH<1#E}NjSaS;Sjus!D9lTrI!71R{PrKnM zAlt?S`nTF4j3i3wT8b=5?Ky?(-wMw?3ex(_x@#rrT1jdtHou&bG*iA`U2$Bm3-^>l zKU4=${gYCaoKoPWN@{iPu(ZD*1r+jt2jnqpYBe{wrJ$tN_1je%X|eaCFIS(b(LlAR z72EoXR*>uO+TfriE*mSf86h=IX!(>(rm^BuReC0+{5JiY)fN2@{iJBDJxS_2HspMLhZafQvrtv@LaQxltr1xVogb?=V@#{Uj1m5huefH>a=>+%g z_@Mjd^`lRpWXHSY1m_3h0Cb5D{BM~SqWXcfy0nbp?6e_`{Z4b}bkl@t%5%m@Fec7M z%N%PJ&g-Hlfzm02R)KU{%|eWQvDB6zXpzOj;}Ksj>25(7Z`O8oD zUp(z!KemN^rZUm*wVzq-XjwTsbk$O9o>lj15mfde)bgb->cVcEb{ACNX1L2acU!Jd zb6Y8Q?=YWj<$y>&+dXf(-=9Q{z1<&wnQRZ{c0OCYdha?9aHV7_4=`l3IuCF&*!9le zZhIddKHV8cr*DdRfR`eoy98o$j(qk@nrhbukT)4BHl1{u@%@w5Hp=mxM`z_ z-Kyw1YzAyWO`|Gunl4u~L%^hiRZTj%So1pJrp0C1&}?ZZ18YfNFWN#FlxJtZWTRLw z+T=+uEH$^87r1iSSufhc_;Q65?v=f#HkS3GZDxo60~YPy3rV2W%3hm4Soxl35g1uy zY>3A6l4AjoD@FL1rGK_;M_0p(rR>d>R?+MfqgTfpq<@Y`|Eyj3>W?P$V%8%|my+BX zoO9~lFsylg@=c_FwlO^W_Kl2F;&Yj41`TA8U9#W+A+x+#IQPl!Bv~LPQiaBczcIHM}7c zq%Ed&LD+Q6a=2XP11-8jLKw;vgkw=U6eKu~ag6W`j-GF%*8P=XKrPH z?RkEAkzQ%gYS#XkLUDFW`}JAa4|^e=>j&-EH(8Pm-Xky2ZU$Ll>Vc~y(LuNf?ztL` z4r44BE^16kYZgR=1o&-1C7{=#ffkS5K+~bZ>g+hi4yeBWKRJDn*~cKZ{hiB%h+wBj z{w~f;_Q%)lhdN#mT2dWJM2n$h5(0fGKC3xPAVU(#>G|0dWJzcn>-tHH?qOW<2%_iYNhCY z#hm_Krf&$yX9K_k%>JZPzlGD8$C2mqZ2dd}gPw)?fvJ;#vtcJ=@Vje>sES< zlyYDhY44Fntuq|5jq%AgIdM=*7eAUp$zG;j%dxZkQ7~PFyEJ#(NU@E8MvD?clnPhX zMqE~bbzc)k%i{4nOrH532WoVSWscWI4alLqn#?y|rkC2se1vR!FF&G$EnWDnPzmpo z^9)aMOnDx)P8@?|G!a=Mi{om*aUYS@59x=rJ-`OwKyi8+J3p;D!PQRk2rMSTdu^IW zzC99BROflyCOpt2=OD5Qw~f7+RZOV?N)k3J6aB#`V>$bDhf)QZV9eXqL{C)WJ2%m< z!$)s1d=#Oh9`f5bAkEv5A$Q@P-S)lyZt!()^<0_74_2EC1udwy`Mz*klzhav+vE+9 zY({Jjtm&ni$jn?BNrc(RK>j;0Td%bokqrr)*wdN5;ROva0E?KMPJ76Oi`5c_-AfvL z!@(JTV6lT%?tD0FbMlx!^^iTKnyF?@UzKMbHkIHOf>!xba<;CD{>?$FELV#Y&=gZn z(N)lDS&9c<;ID_1@H|H)!mqV+C;7p-(3xKUF{Tv z4uou?9^o37(MLoLqKRPKq0g`jzO;Vopt5KkM;As(wlonE{769MkaKz-7TY0fyNtoM zll*ef%fkftYLJZbELkWgbEl|Fh-^7%Mz>Y>pmnSxMlgWrHSus;VVo2^T-yP6Lf(VD z-ds%1@d6tQD4ZSQ4=v`NY^hnc+^Ak_S;b!RGe<=_Y$dV+U#Ss5$J7J`8iL+|8h6tu z#s0;+2cwwlvOrp!c+a|RkvuwK_y`yV$#LQ4mFW;>08?ZaKfRW>>@Gl$*ibT`iPu z!O<)Mxr(9ziklc3#twAgI)?Vi&95M61cqo&k=8&rG{fI9j2&k!z%m^qr`&c`u#bgN z?x#ptgc%J&|4{PvZNm;c(9QQQKN@H0BPjvq>T(*~^k_?(uMD#>fHOlDCNbM)X2BX7 z|B*mC1_%YBv)_};6r@mQI@s7Lh7i(lk~S@a%}|+Q}%7k@O7jyl3x^CY{MNb2ejCU0P$nrpuPdMQuK{(gdr~WSU>QB z=7Q%*J~%tSN?~O|L&}1O;$P8W9ETcW{Mu5-TnD=HIJU(RIZS>gD1)0za!C0WHMl4ed%@tTQBeCpzzt1%yO8#oWdL zT+(7wY#5lQ5H-tQGj>N}&%~vK5T`;d+;=QtkT#WeL|t|ov&uJ*FWP~0Y$Va{7J|>4MP+Ch4CEh zftk(!tSclW;eu((hYl^c?68G-=E?R&Z~P{ne*LiefZUrOZOIiG=nRS1K zDX;bK%V5&kv26UW^c<>S5~fJ_e(}g{Y3OKCXeoW0uyL3gcvnrH4ibZj-w z>zf|M39ez~wAZ5rX{Cv-n6?WZqHIye#Bchp=LaC}`3Tj-5>{G?HB0gGR8X_TG(amR znZ_f_1Ipo0lS~CL7(keOhTfpOj6<^(@GMSqG=naQ12_Z^33AcFd|)kF1yS$T*}}+U zIxcofhzhWm3S0tdQHsbe2&zy42epY|Gkp}77DgPrMP@<`mV0BnCW|f>wQ+Lb&_^JllppQ}qZD4Q0OwnR#cA-$z#BH_$DP(A^k9-VNIUd6EnHlm2Ha0g1AuIw_u{L8#hXF&x%m{e21^L%{yO+BdD&QH~Yl&4(=fV0Q0K5Z<_U$(d` zyJXpaX%*V&y3K0ev71XvTp2L&v&Lt+JXQeHvfaG7a$T;QH{1ECy7~8A0XO^a-S6ho zI=rQuOQkF6=FR2xa^1Yy&QI0Nzwa{rUUc)@Hzj~NF+1SEUqQfu1&LqXt82iBaD<{1 zuce4Vj!5nc!d@DsywA0MI7XrH-YFr%AZH^oTRcbjAc+}*I8*I>ly;%SLjJ(J6(S5r zD`ZoULexU4A@b}h)FD-ft;jE|?7&8^ifmLZ58=plbPJ&!kPkIU#s-c~tX_pMwsD+g@O0j=w1~RM!5v_)mB9z3iLWINOlW}{H za_MA>xZ^IWtLHHtbVF#do#hh>TdD|5c<`X56A?VI^#{_AD)Ci9L9XqUh@2$$mcPq8 zgKkvTNi@0-Ba%y~JkH~buUlUR4<2|qV$V^oO4I!M_|>tuqaC7y8UaoOpJ)-EMu-0R z#`9tuc<~nU8#X`D2eftjR~kh0dSetNwVf%{Mnk4ZrnvhrzyVCY zG=ImojuTze&X=H_Babv`fElKP#Bx@k(EULFtCiv|7q8PGi~oL+_HwBA4gNtlUlHy@ zX_8N1_eP0?$IehW>g;Qe_g}r-?U4Lv9WP}_y&9g4!ak3+LOWX*tQ8eFnS=#RC*+)V z%0CKH5CyhN+xn2&%pkR$_{V^K=_@N9bfBM#v-$ zZ<%sbFU9>TChZ?+mi}4Q@R6U`gtyf3p$iGl8Ipxb55HN6XJ5?FM=*;(w-FM<3@5>c z0=*YXI##wGDwIu))Sh*dH8tjkrFLCu?eoFc;(s;Vx$Mq^ShHxT$xJ2UKG+F28mNnfR; zRz+HV4*(At{A2LTp$?Ebc+Lsebt(n1&?TafGKR+ZiK!U%#E&2={?E!VxfK;OBC$-EpydY5UcZlG9 z3O}(sNro`gNSWM$(}i6!w!1W4G2qP6m1@O0c)P0|yxHHUjOMY)^cUH!ivlK}5JKvZ zj_h_x8WD>Cx-`2BmneNwiO*yn9D1h`$Msn|KFNcz7fy&}A#FFEy2)O;Fz(_C5{qfp zvE#7s+h`6=utNk1`vVO-+D1=`-eNj9+azAny3)h;yce9VFeN!Hh-xwEE&QoYI)btz zogv)F|Ks7r^%xQ;Vy^djn@$L-*tp?F>iECkkz0cgL=b zeQTfS&uHv}`5nF9g8{%spqXQxY#&93!RP+|#Vx>l|XNB!+k?okr!A ze4LGXV|{RtjZ&sViz%PxmL$MTiSIPsC4@!AcN<=#>igedNdiL!f2wWOch-_%0ZGgs zNa|N4p9!2*C)0zdw5Q7N@r?c>QbBQ*clsqt6;%ZB{7Z>C7UIbeqKr_~OM1 zSE-K@8xe@oY0GMww<-pH6$4f!qWJ!Vh+=icn?H zpnB&BRSo$i$$rQ*HYo~5R*+eN4rkf!TIMqmoFZ-m7Dn9i4$ZmqAa@{z^2v5>p5_fg z5w*U3kdKfyx2BF-9_ac<@lD68oy?EoIb;M%XDDqtu>Dbn37yF-LR%5vZBo*P3J*(Yl!TO-$nn?@@R#d#BNAg&r)e3s zp>WYTnv=ST^_4(t{7DYN7)VGxjT~1%$=b=F)5EV(nq!4Czd4LHg_^>QdSxdM8#%8}dD;Ewy(uRmb54|hb}TG_{b;@!SVa6%e|060qX$8pum&?6@!9} zDs_tj3`(o8DYS{D5TYP$E!)H_Ci+We5<8&}ahWgL&8*$L{18^PWu?0i#YSBHBBbqa@H2yaWmEnI;HLm5~ ze@owVZY;*tHZd#{*5B67dV*%5H#22)W!9mOe()8S+~F6Fptb854i#M`LVMdvNr zEEnv}9$v3gG}o+nBQdV^E3lpdcQy*a+(j`~*%j~hE3oV&UGM|0nWg9XFP(4a$suV1 zTv!3wca&vMA!{K@F?2}&%h#jqDn#I0I56!U?Rb(5>bV zXJMP^!#C-9Kb)L~r+s8HE^VNxo}7Ra#d_9dctxG;S43ZCF+wM&=!b(dbiH&W)lW5a z)FC|Wi56XH4?l&I$y8M1wpGTqW669A=AA*J94ob;8ZdJ<0x#EMDI${ZQA=pEQX5UT10LAQ2PGYg$hL5=X z9yZVJWbk4-&Wej;Es_Or!<#$8@l;4RRUmTeN56bT=#~m0I8^D7Qg-YcjaMA!C zV74}q2LgIcY;N(`UTG#mJk1^5*s&y!twY^n5})SX?l_%@_k)D`PQMbhy?9|Dvr&a^ z-nPfK;Q?LKxbCfc*t31lZ~}yz2f81)5S-h7J=h(-O{V^B0G*p2ZAqY04bs3j@Jzx1 zjtj-BbU*RUEVnSgM>=B^sjPtCg zrK5bDLGNks>wXxYzZ#^k5tcr|-v*8eG4A%U=1yZXW3C&5~XpHQJ>#%T5OvPg~ix!z_pik}QgJP2uHQVA3_aK4^V}KW6 z-GIOF#mk1FdZ~H_phENODE$pGDRZc%rDt_cw)dki&wl@M@b?<_Y@J=l55SQ($uFbvNtr^P{&p8H9=H*7XX44%mJI?1+{8PYlSfFHYMg4-C)e84Q z%gv#Cz&#+P;mZu5JF6qCo}h_RRy0#pxQTb7-lx|+UK*~}JswWt=BsfxdpyqSk10Y$ zMGPo=qvr}*(&A7!FvMP156qZ{Xg=XH%fB1yIy|OGS+XzIce0c1ONb}S10_P!6+;Hb z0x@m!QF?MI`O_K=l)eCIax}b6M7HqB5w>rbm1j(exQF6~+>~~)m`4W(n1zUL`0!eu z9X4J2?C2@hZo$9julZ$5fNUrr21Nx{rE#;`h(}jFg*L{-g7^Dd++@KD!u-Glge-Us z4Ha?{u(k10Tx@|}ACBP!gJeSj$4Osk9 zpbG6PoL;6wc+6s0q6E0rM${HAu4$5uK84zo5MGw|S(IN=P^o%^%rFHZ54QFPpYkgWiD_5u!7_-HJIZi5rNRKacd99=VD8qmLk*Ldlyx_j3twZ+mP9c^ zOg$7L>+TMP2~~z}gW?+2wxl8oZbrhb6+|3PvL05XKcCYK*$ND36+)W~Oz;3~d^pX* zr%!gx(?8tR+mdd6v?Xmx)sRNMGPv>s7b_^dNCPaha@UA)WTG2S(xC>`P9OpnayQEI5y;c!%He*%ue;=Kq!A57 zGEDR(oD!SxsEfCOvB5f@2N^c05Q>$AyhOrJN5Ieru#(2vIL@_QP<}e26bi3O z!x4+c(R9cZ{iy@sXN)K~^q3e2c7G2)J-b>xc)Iz~mIO~?03+yxG-rAMElu3d26QUk zmD+ZpS5cI5g*Ghegx8yIz{>ZJDbkNn{7&g1ALPgck;P%*wV^Q%b`rSB#lCTl{sL%$ z#3f=Ayh?HDQI}PbMTIhU2#aU3($i8{aplWABa9*5Or8;m+}p^{4e^`6B;#l`_TBFyqO5Sq*q7Mt*}i>dx_Fi1yx+2l$) z!iAlkVO3e5^Gnc&5^2tVo{zN-{Gz>dEKl@R8pO-X%dL!>kJ0?NXniXmouM6-dDJ`! z?e941Kgh@uLV=;*pAP60v2QrYzOwgCgG-uTJurqzjA4}b@iM{`GtT2I>@mW5iq|Xe zgZM-C7l2B}M~8U^@Phd0U1WajWgx|8oT*I6mRWE>SC%er3#YqMny~`})<7DjLGVL` z8j#*@#Tt}1uWqKXObc5ZyJR*Kmn}d<4QzD$J?}t9Vxpb`Ho%)Y*x!>{ zTLx_KF;1M%a#+LmZO1mDNn!-HOL_55wx2(U_2Fpp*U0mdrak33SbF!Tw z(P+JetxsGhk6{`ihD)P8nihtYYL>u8#vE#v6!Qm@#2sA$Hzzqx=hFU~nzo*Lm;ijH&l2;OoEs>%SfthQH;yzKt*{%i2Qf8^`kh zdsE$RZ&^^$wjl(?wwyHV@HGzv6LV<~;yY`G^^b zikkVjfDU-X!e`6|s)}lb1Qc+v&2W>JR4aB&G*WplDH*Zh`gj~=rN&99JNV`%C{*$e znEtEttD1%q(k7i2N$nRNcKTNH(qfau-J)Kgm_5OYQIu7H%j;PfT2e5e@6Nm#)!EGQ z_bWhpd4GR%$snHHUGeu{P~$b zZqEbhOc!A;#IHK;_el`p8sF^EN7@nSM`KlAU%c`J5uz0DbFp}RbF4jt6Cc?^vv9DZ zoou*UFgioQJ)vw+5i%)|AbgQ0!o?2=$CG?~tm`C-A0YS}OlhY-6gp#cQ* z@I8pBAhBXfRN#=(9c3M;3Y;zuhTX9#@=booD5iu2u^#ht4)`H)CM?|ryOao%J!^`t z9`ir7!9@4t*rcZ77zOMaa8f2aH^*kl4&Yuxd6_w5i_WgKVY(icD}Yyy9g%DUyCb6k z%QfKZxcJEHvIFQW4T0z5WFniTHGr}cTH%i(aj76iU>rW(1pDDe(Ags)2r5O9g&)xL zXFtr8YQ+&t;(%NtI7gfELlpNp!UC914{0>8r^! zl^FG~9hqP7iSA7`BkJ8OZw9@{Xq~kmJ6)8rA)$Q1+2DfQm$Vn9<-ok-}2mD)R)C|u34LK%NlV@ENes+<%9OnKN!>c$S!fv2)&?O-V{ z;fA6dcE*m408F>EcC3Y}FSuL-Uw^JyDNMUUEXKdx*nybARKTuJxx)Z>_VWmY3B`7|0g6rU zKTnXHihD3=DFRkcddx}r+WR!pI>RB?-vNn^m$J!!I$(~K(u&zi6&C)=*1}nUk}ZB| z<=VuWMbWH16@b_RMWH{86$_Rdl9y9#!$p-qG{%w=Pr)TL6}urhij}4}%GD!EBX!a)yCqG4g-8iAXleodSt&N zAj#H<#nKb2Bia#5Q$n7UqOSEQteoRvo`Da2m`+bD)#!>ia1CS1ET^pGB8;G{nD!t* zO6x^HQaU4tWsJiUqaQ&`7`!+$25xEfMu$O8H#%|BkdJi2%HobOcq^8e#mb#B!Xb?v zJEfL8#uL3^CCm;*Eyi7TkQjNSabgMN-RUKzWiyA~F<;IW3t5MqB7qO( zEP&`CCu}8BTnjoO9gN>=hZ^RHym7D5?N$oR@xCDMQNtB!k+nqy=cd`N8Sl4%bHi)N zA_K$s{^5D@UqqC3hC_IFsp6ZF*LktPwMNv!fL`o&mr?^D7%MChi8jVxE3BQuC3%)-TY|F!gDdZ5jrPr zzyO{q-0hmnv7E$f+r{eVuF#$qvdd$*QG6#B+n~tOdsY$M0-|skVTlxkV7xA4orl3* zcIH%8|9Me|^Sls9c6T4V5Hfg~hn)z^q5I9nUbpUtxGsU9xe7l77Iw1)I5FMxQktPF zw)~rS})6q8%6r_gKCfVn~U7 zNymu5PaH%I1&n9Nr@S_XFS`OkJ}k14ASlTW3>P3saauJ?(OHT&>ocvr(qxhocU@hL ze0cfc!`qj?9il*5j0~rfPy-)T8zO3PN=rnx9^uO(b*-^K)jsD_BGILzP34aUu6zh- zu(6hEn1k@&c;#F-Qzu`x8jxsHGIJGQwi*`m2d1>Qz;QLa*U;h;^%T7K z=+XZE0crb+t5WdZf;9IP;5|eP!zBg%5fADu2xo$X-1gP$WO)4e!D#h(@8(Ba7VkAj z5Kcm~V2|`#ksrUAcO3X#YnPSz?s&QN9r$#`;!(ECLXN^kb|wmKKtf-l(<*%TS}FRe zn5yr~|M&N= zpWMRaedD7oIe9mS6q>s|1ggPY`h&;$TclB|m~9*%R_7Y&HSozqQ|DBgmQLAZXIO4Z zf-hg(Vzj7C4UWG_DvK2fSY9$KHDTKJRQ7M%YFgRFGd2V3Ml|3S2}j@+DD`5g2;+g} zI~W;jctt5yj*qy$O~djq*DoUB$n9g4(jz}*I)xy-{T)553 zQ6?ule0J6XW_ZW9;Pu#k0H(&4-$4I5~2MJ(+^zEq(SoU4dmNF zG5QURT)3g4D%wpx^xIS9UA z1gKs!3D$!f#K4S}+7?vgO=v^D(8ea2a}!AYJE0%x>#L74j&7e{_qs?BRd-u z{s;CK){Y2n{AT|G(!U_FlSj$g6%Zg2d=;g18kK)x_xK2++>;&cMLHT?X?y1*1U~db z_zK`3h(d>$4gq-y5}l|e{5$*xY&-Fb=sRaA7X*HAl}}&nzj^cVbnnTV!=vLDouh`6 z;Awic|K9Do$#a~|JW~TB#wgK*37jFl9L$q##9xncgcM#!QlboEouV#F6n6iKO2}Rf z!=&NsNQi?FStxPK&%9ZRyIi~~XAnXflZVDbD2VA=%~1VrZ`$n+KlR5XkW(2UHa=w+ zd491)dm$)ij<=w)C|ElB6@+{;ZH!MQj0rCi3>W{>o@V`Q@E|(5aR!+tDWVD3;dV%F`v?v7y%XakR)s@?N0uxK3&~X z77a06=RFM3l1T&Bo`x^N@Lgaj)frLvk5wi*fzyB1aPIBzN*67h8a3xf zYpz{tEGrfFh{)$08y?&f`We;mO&%G{f_W|%vcrEurSLdpU`A?u6O zD)2x<;nzS2yX^f48ih2pavbsfpu-FohD}l5%fgBbc^L-jLZ~8p&8q$wpaKHPP^(N_%?foV{L97Twsp6bvj@FliaP@?Y7VADY~8ArYRY%7bh zv-6B3<|!4N%0qaP{Q}B4g8^vwmtRn!IlC{W;5W-<;rd`Yw`l9YQdXbiUE3uV0<8$u zd7jDe)Mt4pBU9>KWtKlapMuibWASD#tUH!a>wx{kZ5PwaN+$Vhr1r`w ztmG=tM8++F3(fM^w153He{evPuXNdB`mRm$joTyYDbxJ^0sSYnrcCpHA;$sKZDbBb zzl8e@%zqa$Ex#RK{Pz3B?rr1_z46hOoaM_gi{^F&CxFJ`)4Nog!+$7FEAlrH314xL zxCVkT8ZHvZiGo(vbdtajfiUdry(Ads7c>gsmRFt>*c(m7HUz@~kN0me7btr|Jr&dW zP1I8AZju7JKTzIezgxdgcG|$Fub2{+3l8vfUk{ z*mMLin_(7L>MrJvDS$-fjcFp%lkLJzz4|0u%v}PvNo$fV*(~h|rwtR!R2?ier;SU# z8uITj=Z#u*VP8zM%SiC8S-9@p*S$~osdKaLjz`J|5iumyTs@R35F=PJWxk)mDHBhJ z_1VD^Z!zJS@IpKPb^WAlCg(S@}O{fhAI zWQ#eB;DWE63b;&A1!FRvC)-~h^oQx$+c#uRu0KF_b7c3_Vy89iVW|1^Do(rImxHUf zo&K};)>hw|`f$=nzA~ROC;o*u%5wED5DHx9fC7Q{*a=txdEa}VYo~?3l zq%3GfDV+wEp76t;Op)#sdM=LJMHDp*k`Zl**=GuY97`v?RZEYc=>q-Als-xSrrh48 z??|MKPm%z#bfROS1<8o{{2HFV!A}S?{XB?0WRu7w7fG^Z5Bo z?bk)FkZQRJk&&QR3!$R+`%t0|cK4t_6^GjSDD6U;s=C;WNbs22v|RQE)5zjR zF-8;Dq3?lVeY zBNruZ*W`6sE&e!*af^>_Xxk6yX#C>c#A2ge7i~LXP}}ZDr2OTe@6b_Pc6Z#N7@okdOKqu4yHj<0_%H< z^}+{6`{a1t}U@ikb7s;YIiSuUN7yJ_(tqIOGEwQ87q21o`7`u_wX`y5za zhLiaGU!VW;rT2c%I(_rbpoQOKFBC*9w*x7*lk}`~)Tv zx>{hl5+Su(A2v(z_>1)+g+A0=!`2 zVSVIrh!{JmFkE>M5UGdNur0Bs)d--%8{1$8W8XTuMT|ua+Y&MOML1%_U>(<`K;Wq2Y-o-tFyf?SI?L%B?2WVarp*M7@*j2=SBLiHYA64A!XjI zoa20oYAu|l=~+6$oBI|;v-VW1tVL7|U?=G{Fvq&9!*}k=VOXGlu}yf?#mp=~8RiRy z%21*-yCEu6<Q`*1I;WWAv4CF$R$g_jTs9ttj|$*uYiGw5+@jl zeN^l2`}152``Y2Ze$fsNUp~@ay*Mw6DZ1; z>V~jpvAU9ZU&a3Md5FB=AnCo`>-iV@^W^IXr0sUzy&8Il`K0^h@?W2`^zt+u|L1rb z;f`boE5@m=)XIg_QY52N!vZa#LLjlX7`hNAqysn9;Q;dgH1GgqT06rL$&(tSU5MON zrLF5urr|a^#;ze-eO3`Dbd5Isx5f!w)BN>eLgvAqsRZ`SFm)L|ZqGhCqllss^%PT= z=^z4^u%4);n7Ul{PbzIUb@8!Iv{2q548inbu{ZqMU|<&-ei(%?1^E!a$HL)c`zY$$ z;|E`!tsZ+IK;wo-TM~PSkraRf!OqAL2RbxvXAN20_b4GO-qS$>O(4b;rHO^5&;EWe z0qgg3ZJ6fdL8cWwGEyfQ{$!-#AXyL!Ye1AIS7+03l!(f-L4(DCtPF+ip@TUF&4j&i zjuMEK<9f9>S$wWNpMKFu{Ue$(3j<}Jj?dF@2n`l)XP}hRPg0mHAc=YWWM6xH_~!W$ zg_R1^7JKI^ZB!FdI5R1=#ABVwXswx#{;VZ*hF*li?rlj*Ktk4nCn5N~fsTn~2E(^H zomqB21)I&Z`zh)x*~#`L9C+J&&ceIXibBQR<5*&T0~|NvmRCkN!+`11hMXQ&ZZ`hA9y?a|8q8-HW^n>HH*Q{YSLa$VSCiG5v~TKb zBCd8soE%n)j#UhHWWkK0Nj4785j%2qmW_JI?K}nf1zu551mG3bQw#o9e21SGpYV-h z3)^nPON*ND%!*I)?dvS-so0BeF<5IR{Il2&#EbgyEaN+?Z+PsGHvb_tO(~_yN^A!r z8{j114_8l-oYp!^9$lUaJ2`%$lp0DLn77G~#ZR5mmWh20_|MS-I|rRzYO|(%!NpBs zw}xLBCEda;U>D4fPTu~qP5zzTGTuHBD#W?*`5TXfiSgWc}O#f`J=QRGz)QuixGv{RnaYym^vS9tn54S#l&}IJGze8 zPLIF*E^kLiDYv8Rm*1uKse6g-@^|Ve<#wq))1VONCT*AZmA_LqHqnl9o_Gz)?KqaD zQ;v07d$R4!k=v#X_N!o zbn`EKW$lpJ%XA*-20m;U>d}(Aw#Knt;@xbx<&|Z+$s!1!gOEgyoXcNikdx zcA&E*(g7Ic5nS8Xt99p3U|)ye6^Ss~o`w9xK9mz&1l?N~&p(*K@D|p%H$K{uYux52 z&&=yk&_~#+!v>dbIPFbN3j@K%&W~YUKQ2;5S~|&HAq%slo+q2vJqVwGsb;_vie#Xs zJV9b%Q!mB+Dxf3-IcwB-A5f!Miq;nLF6o}H5#`MgB^Y3?i`17S7-ZQVY=IJ@Y!>2u zDG=q~58iYhA3opzhX%JLuo_r5U=MEtff$JeoQO|wYVnEX2`~-t7e0Z-bUgy1EKm+W z8UtfCndwv)M{QY$@(O=#ygz)o^GJJ6Rw6=lv_4DXgz(!;PQpPr&^mFPPX`n2f3$;X zulL}X=mpQ7K4EAp7;C&${Blxux)wkgeMfz$3A$L!urC4s593iVft6;tg)%WCuP`9C$b0M&LAuCg zjFif#90f6s7<+U#k;Irjt(cBO38V1?f;{wtCATi*ei2eYn|P99FQDciEzI;DuR$egT;)(30x_k)(8CMNpL4NPqPJ$<2`l zPIoEUuqi3CvA%OdWwwGP$)&#ent%i?;Fg*qW#9NT9gu`jMX?57vAS)g61FCHnH?`P5ehPtVp#0 z!^!p^Pd+?7{ym@E1{}TV(Uydx%_)N5D7YR-+oHF^(fh$I{$q*&i3CpmGQ=L}5gZ-l zinft~2)b%EEZQ6OYDIW_OI?haN@eofD8mz2cPtmWX|naJA*ZX90dDgoN()f?VGDp8 z<^3p+dm%=_x=rq^+yC_YOY`cSIsUWpuor@RNSZp8o5ax9=&DHuQ7}uB?EwWp*VN(B zk(VLWM)ZlgeXiTBY;0YMbSvyw_>S7j;WogQNP%M@@{b}39ZkZEl$aPOPqcw{33Nt^ z>b)LZX<%NW@+_3Zub`TwM6Qr;h#E2xbK02Pn{#_RnF6>JJJ%<7$XnW!7D;)1NDldB z{ux{fATn?%b{r{mh~VW-%|CqEe0!7o{-gc9eVGf;HVMm4{Y55si_N|~SxgJCC_%%L zllwKz^T}Q1Cn%PtgAQCcaw=HCl2~eP4?27M9hq~%hAaV|_IdH?Hva@UhB7SaSN-C+PXxQnA2S!X|J&|)G0?^W5$#uuwiUkHV&|e zdMT?z$(nKW;yvY<5uQG^~c1yPWunl%;akl7XgTvU{4-ML~?kt8o0_tvdD7nUVRJ~*cr z^3EkjzskB%667n(#&S8+GREFf7E7l?5V{b5G`OH;Q~MZVUi@~k`#I@!GK_#?Tqy8U zkz!JQ(hH6WXsa+cqba4U)EZzGJGYg0!~qezATg)FWTkM!@Xpuro;rqy`9$1iu^-U% zElo1aGh#t!z_YM=C&goVlBt&}lZ@+c2F6V%naljG>C+_h_GLEuhDoNao%Rl*Fi(UJ zVa6n*)D@kHLhfgGsG7lAnQ7{7rxnjM>a>7I$1;pkNZU12&Ah7e^ktKbe7jWkP3D?< zFIxApk_ol$W7qaEF&5s!lruS}fm@w&h;b?K(c0#RagTg=HIc5rF2Jj>8n*^w zFs^NW@QeUoj!*fzi~A7DrKz$QTig8BHoxuF=NDSr{3xr|+Mt}neFA&QDN!^kR!s>>F%JQ1ym|b8WCx11EnPgC)jZrdd-XchzmQ&1x07t6rDJbKYiYR%rI} zdYz)VX2p#gxbN1lf#Q@|+wC_NwcD57QRO(+yLPkMU;7%|BItnC7qDz*8i`d%a)L#V zcy{vC%ic7<2xu3jbQ+Z;=ka+K$L}-fZT_~SJ$wzx#YuiV?MJN#Z=lav;fi5Ms6v zuV`bBipg0Rmq+kA@~=wZFyFLNvL=g{s|D)XNG?*3#6W+h_p&C0X<3Mg!Fa(;@99|h ztic6$5yPvnHAxKfQ5tJzKtk^b=yTS=y3+w9ePB2)jZ}iTs8A?>NHNU7F%TC89ZUx~ zcPs=*K#wYa5M70*Gn!P76-%H z(IIZ@1Ii3NjuIbTnSfLzIPe3>j8E&QM4 z#HM>m`l|@QMC1{3w6{7^OJvvB0XJXXqg>@c>@|RTErLjOWH5$~9Al?S=b`vgH(ILK zT2`E^8#4GoMQTh4X(H$nV}++XIA(w^KGCHrkku^3{`FvsgtE>Q0Tv0xY>Mv*A|SGN$0`dlf##fNuR92E+QO=o^T%iv4zA- zXCuU}CbQEo;rJ9Tk{HP6MQAbAcJq94rR@(U=g`&;p^t{}+)5%aoQ%?Zz-f;VXcvc4 zD#e)LAr6mjw$)ETYQKVPJ48CE&K`78vhf(IBiidx7N;7;YHEMqd;aQZ|Isi1*r|?` z_0OgSy%N^o-{1ds*|&(bzgpzt>bBjCBDF|{Ta6-hje=KKLoWv86Pn<~aS8n>*o7iZ zNVpRMErz$_csr0JD1}BSpyQ^IChIuT@GKplY-6zk)xP4d@vU*B`@XSf%W|dP#hM7B%rtkD7ZwJLd3!faX`zFf$KtqwQ07P zCRv6+E$?Ee9}VJB*d@g_#)@m5^eI@Dsr5HYX+NZYh*R=b=$nScvf!w5q-@zh{c@^C z{=WG2W76=_6Eb@Ieecza7e^YOXZ}%~kqHUy0Pg~r$Q1^tHY&%*@Ne=g_v#5$#+3Tu zkvI_fppY!ib)XADCsG{N@Bv$W9sN@}Y_(B2tm%t%IqG36yG?c2!j3~l&}gV8c5zq| z5jK$e4QnRPv>*o|U$)UKRZawo^NvxMhip1nIR17G7Z)8+Y7zMo3v@1(Du+PCC-kmw z=wSH}{&5<-{qV7K_~yxZXL&|=YdZ~W1>AG8tysSV(&tqhVMj{c#n`$M?30}Tlfg}$FcFB`{7bzmfMGu(K#hnG=p}L9IQ=av#-n^l)L4q?n)XFi zQ!nbWFJ)zx-}nZ6@O2kcgOAo->?U1I%!W6mgL50pHA1)O@=HzJ?A=i&Zfp@}D&}o$ zS|(1kK5I+EOEq;=>riFgy2`aJg8g1nM0Y4^nr(N%(UsekdDg5l*O) zcSZPfYddQkLV!d?46p><1`ZMCSu9H=4W9=1YE~2Lm{t>Xo#xT9ZY^7^**;qa#l?vp z--4CauR)ymPqwA`@+X4hWpBLeuYo{GH^p;>8R!~7he2jG(Zk;O^FL~8T<;;7{dC%c zcbdckO{htmrIA-h9&7{=+T_At2^PA8`Ms|PoQ zA4Cv_v~&j|mTpXW4=kP146u6a^Z%UwADoX^_9_l8%iDCm#lf}O9I7r(*hl(ldg===%c9-=r7p^ee`UH*t}`P?<3#K^y%gGV)7>9SZuqG zEil-iO@73}`WTQnY~ZJ8fw%0(Deeg#7rpK&)OYDS|76>Jw17v4@1B$5yW{yN71tZG zBPQ&uVpg9%QOfQNA1XpNu#jie(s|bi*)>vUDcVVO_H=X_rVM(J;v8(W7U&^;Mh?y# zXTmxZB{@JH937pTwNG~#_mH(r9Zd{WF8|KX(?O{NnOoKlAI#XX*in23PYrwFD8A9< z$6|G=HpSW6*bb`|JySTW`5i{LU8ye;lR%9_MN$l^7qT!5)O?T@X-YQvQ{r)*$1s$YJ_8pga77eH1yrqC8TC<>0J!I?m1w5?04$vm?#omd@vF znuB?LToY@7&)I|qIBJ1bAhjbsDkq9SU4}d*HXAnEsoo@jE*ju)i%-M=8#olEY*wGv z6LOPyrtE+z`xKW_+)Uqre-y0%#Tqs)h;$Rn^-;4>xtXTDX~9ql>z!mDgk64-b>h6m z;HMCWxz)v>7nlZ8KFwhLm!)(X73=?Zr{7P5UUn9IbRRu`v7`M8rk~4_wYe;99CF{$ z7TY{!1-RY#0Vs~=!ahy5#q7$(c+b!VuL|&m9b8t-n<@%eB?}K1#myZ&1Au%WwZp2= z+5iXUhve<^UHu1u?`VjfD?b3`fhe!U)8q<8Gwj&BF@7yJRVxl#TsAwP$x`j97fiXa z=)%}RfHuX!i<@T1$1Cm;J2{r2u4sy01?)$u7Rvi2-3kk@9k!$Ai&B#u^Voykc^iZ{ zDq#m_*#^ZuXIog^5_MDWyC1w!9$z4@c55r?W`%O$JhO*EGZ9Et_dqu;Z+{5b}~O;J~dO{ar_<2R$oDm%mIKs z7*?PP3Uj0l^>%gGt(VeR#4*hpzHrpEhA;W$pqGaUVpsmsu$LB$AsoT8K*(*v7sDpX zCgtq4z^En}a2dYTOYt7b@P(lW4vnp#0S(Aj4ls@-H*gIHp?pO^!3ZjrsjV_l{*7>k zgI<0H#d@Mx4zSZCb1P1YZ2CAu4x{CzL2;f^aRGgb1%w2j#D#P4&e|;&3mo*b%SSvZ zJ0SnTD)BQcuwTFuvWeSN+&FexHl;WdxR9?6iscI{E)K<48GIE|Tsl8GUk~v|U~bVc zh}1dbUWi0st4N}PVYdd!DF5vOkSrkIQNyCFg9O~PD6{OjCIz(^0A;o5V2#Ks+x3qh zkc$-+B65wu(-B^0BT+(t)GtrZ-b_;O=#N{lDBt*K%UYCiUA2LjEXo10Ee1#vjQ93_ z!o61MVq}W<4c&p)+QvnbTfAOP@`%^!;!Y@)$v54)grj`C{*I&1N>|oA33=YDm(sv_ z*YG3ZIAOlw^BHpYmD5>)@0tsCVNYPwN-+ts>7y{(>1cO=LeX^04Ou3iN%&OBConz< z7Bd^4Q2C@I+w59kfn?$+rm|wXm9Hpf<`8~WzS~HN=NZZK@yGDRhiB-Lcg0@E zSK6%>IiV2JTKFCCF7mCzSm%Om1jhlA+sbK$VZp1H;wH;@mp?jgY37)Uzd3N&1EQ_y zvv?=a5AX^CLMPpB6EI)pQ^PzXqiqG0H`v=Ry?nxGN81Yo%3r0z%E&K=40UtpG2B?# z{1mvkFf2FUb8h-BMiomMZVC(cO#CVV@d6(66J=Lcj97fcVB1ZDwOuSj3fRjJCp|Cj zu_%TIwlKjRSveqvMo1G8>y21%OnKxksp#kmMPsl+5S4re)m2HPj3Sf-}GopL*mA0%po!K6atp4@qR+$ zStJXrR5(j_i#YtdW44rm&>D8CK3_KZ0PHoe;yr?$r4g!_(b+Ze zDVxKF7VKR>7B)c_I6P?H1P%x+d3tYpwq<}Kj>OyL(?8q0@JSYD$Qztlsd*wC*(;{J zCVb&;^}@c!Z$bfLnrnn&Z*~gB!i&4<-R8jy?8TQi450AN!Pf`{Op{jx1!j>=j@PW0 z3U&s+RU8m;E27+NaUOi_B;UN2TT!ec8Eh)vd0vY@t)h4HM%cOWRhqb%;=XW9#(%I? zPJs~lTT>jCd%=r67_8=1i%P(%t&su5aH5F183C;S7Rz@Ah50c!faf>quP3kILMnn5 z&Yp)H0*B8~&aUx|KsZuzMSymoh7g5p=g{*a-B+b7k(GK0SwgP(1zBQp>_o zGmY2+nJYg%d*;0xnUmK@L6_K{YlCjOx$!w2W!+4Ys%@lE{&EHK6%i;(qnQ z!5M!1@CIFGz#aCkgsHY^&@Cu5vOO1@?x#7|gybf~0?3Lx)BjV~18qi3$aW2BsuS@E zG8c*=k{N*+g-y3kM^?-#4YIQGDz{}yg}+KTpXLdv%?pr6rw0pnrdpa4e6v~_Lg01F zMHGyBv4xgeSxTqD$-pPm7@`O#(4tT3$Gb9IooRWxC1f?_Tf-4Q=X|NWNJZYgo9wVT=WfFgMgFF!wic6e}f8??on z9&KsbqE{ba7;O>SSDtIoi+ew5i%QhZsc`&4JrI&alzNo3cM$qluO>zE)av3+D3u$L zJgU^G;mkMVn7&NF>PpAq0ppEQa7J(sCXq8__C)p@h`!&Qh?4ozs&@)*=mC<@KuaV*M?_921eeHgqgje4P{57Ou6y|5 z^u_**-TgO5*sS7?dTA^As8HbL<>gj{r8O#xkfXxQYDPj}Ay)?!mJO%T15AEP82P};S8{0Nxe#gw#<6LzZ9pTbR5?8Iu^l0zK5L=xE49m$9P20iB8IR<+-8AtuxW;i4Nw7Zz+%Bm zRlCdjf>_p&+w}$ci25C3)gJP|Jr47@3*h{rZu1%(+fp+|4Upsia_e^{H zdP_SxPqoW%tc}wzQ`TCDW~T5CCt7r+ou?y|jLy<HhByT*Ixdz%b+Zhz^LkiS zD2AD%>451|SV&O?tBkp$SNar&Lzg~@PxEefoKA!~486&xU-JP5Xnsr1()=_Xgi(*U z*?jQg)CXQ1=Yz>8i>6Sq8&~y&(QtfP-L`VX7v={kzlUHmLr)+7aAGZWN;TJFB;SEX z{dP^pbs57++3xqU%N#iVeupSH;-Y^Py9{(NpoHO@h}y(^j4VLg=||JQIzGP@F^roY zZArUKH3l(r3^GCBBJZ&|rijjj+65m`8qOb~#0?oY)7o*oh7z|}K^VxF)G5oZa1+uC@#?-Dyx(F zqfz!b8)&=f;A=Y8U|~Af2DuicgA~c>#~Mv4+RJ>RMPah3olnpIr5)seb$LJ4vVK21 zgJXxPRbD@^xdi;^lFzW_IV?+)E_{haO=OhOw%C7%k9V}{!nOaw`BqcrDs*8TjEfDd zVe1X7VHIl(RSC<2HyKc+4mNDpaq3Yr4Qv>e0aGzf zOw79yASpq?jll-;YDH&TXBi>Uc?~a%Hm+%(=QrcShB&+;MvEmA{z)e{18wZHe=YSu z4D4YM+RKrRa%2C1{*!7kSGFhx%*8*cbm{Z}*9U5OE>ay~064ro|3)8FJN?f@gt{;VEQZiTIr+47P^IhGx0v&6ZdNql;Qmc!u zrBt2^UPyqpq0lG5OReD=c&U}*umyN2ED2g+O4!lfr2Q}(jI}P!NE#-Z@i`0y`K5-5 z<8w+@g6C;I(B8lVHq!oU_sxFi+2PB_|J}eMAClku$CqzkzWSnl&O+^BJi@{(AK+ES zOm#WN8QcwkB362XDmN~lEZF0{OY{&ph>=gsRdhznZMdHWLU|bu;w$a(2$`4ZCMyVq zm5Cw=q3q!iKftr5r?aC4dhh^a&;!qQY3!@83R|%fJ7hq|K!=FaD9DL?PYqY(FtgeUcDsXVg$$9?!MO1-W~28AHLA?=|l^_ zhc&mf*Q4}7K1$GMX)>XH9c4tsT?(u4*{KoWZDS=x4Az=m#3TSxW2-RD`le~bG0Cx7 z2P=5UJHc9^^g@#Kfe&60pLsikdX1t=(m@ z%KYZ9PsNIq==Lr#@5M#Wz$ypqUrRj&t2}zNzkeX(z3+onT$rqF!!=xRP!Xm;Q48D8 zj)vgc4qpkVqCS!mjJ7<7Yk<+s54I#yDMwO5Dn787>-eo=`S$~W{?kfmcsB`g~0T0=rQ__=}HvH^J# z;WzAOy=)SWu9#=A3!BpAc{uqO{%Oa#c7|3);gGzLST^*h1hvuxh0?plrmKo24yr#q zy)%5krOKx2>QuY8yJ;CW;L#dS^Yy4nDUxsD7_HN`*MqiiFqJo{Or zrDD~cIa(^-{Dy2KJQz*;wL0d_Z=hwo`%85v+eVd{OJ8M4jdk35{k>j)pH6a0?l~&E zL(9@$UMurr+JkH|)YVFaC{4iSHVjPb_4j)Ht;5Htv(UWt`aAjl5hLsMHw*~r+~=@f ze-H8rlpAAaQv~Ud27bUq(1rwqD9&Ly%m(2kEs9~!A+%hTR}><#ZN2_Z*6VLWU$4J~ zWd%!lWrcCm^|xZUs;}DE@8t={i!3%RbJOOQ6qtx+aFgqHo{ASy!9%vKB#^KrjPQ|4 zxU%${Dhpny;n+(WbLOexFe=L&2DJapYB)Y(njwfYg8^NZ;$jQ8%ttrgOuOIX+L{y+7JBKa%f4te6!`26_giUzqRE zeboc^PwZg@04= zTQ+Jbe#=HFUUFF|oH-90-22j{DD&H{S<7G~M4oRQEKjJs=(>l`lWp(#%+g<`?}g{` z)9_5X?(#4<_gpSob1+qGxuV2~yhio{L4o<=QJ)Qo#Xfx_(WOvJe>cjiMCKuUJw&?!4|i z>AcV$zwKy8ofjSL+5Ya^y~CHCmu%DfXYcP)9{2*;wXRbH`fe@cix>{iXrf8^mp$R0vHS-ePGSYZg{7)UQXA@al2iOSGrNsC^m^5L~Wxn58k) z9_N!ucs?SPHRS|0qUjhyYw+u*gR@hVR?Cx(-R$fEyup$V&e`rs#U$jFOI|6|_l8^HZ$lHa9 zitAkQy;>zRTmV zhh-8)uRa26G#zbfN7G1qLu++>--9UtR8FEAya+D_Ay^HTuA482_nMNOcA_2OmOi8# z`y?Dq!7$y_P`Vp}zq|`JDUW|av`Z3O(Zjem`U1NE22OC=a0^ztx-y}I)L31ay3-(q zGU{j}S{7PBSVq+;p%6$UT9&f9A{Dczh+)wYzXD3AiPd>U2_-7qO${))hRN}P6z=n@NIP{=wNhV{HA)f%0yVNi(8>o zZbY&WW1WU4gx4AVj^iMlzZ{-Sx|3{GfJwA*64k1pS&ID%e8Wt7#fW|~=wQS=>rO{- z=7r~CZY`^%m~Y4eNY(4#2PXq#OVeFagd9yn462NCIxzoN*dPAWLeQgW!Xi7$dPRYa zEv~esVM6}Hdiws4Ie zw2#X~++pGTmkf`Ew5cqY%1{hVWTpuXe^PmEpcttXV`*hx^&!`4m^diYV3opRSomel z$HD5HZ0|>7J9;vHYj--&&B*@-J;Jtj>ZSv(m)Z(zc-4kps#13`!>BS>D?uK0gt(Ai zYMXeV;lXcNS3zA)Se2pWbD(JQuGgDXgjMI%F%!a?M5K@k?DAEZRJxiG>UftFo!y?EecK4tXy->5 zp=6!ra*!YrMtwMG9m+s8Y*cLT7Hv%`gC13&p-ib5kMiNhpR(dE??5)p1*VIL;I=6a z6Q3wuO$SN`Z$EZA)5ou;-`0T|?feKGsI_01urt>?P%69taCMMj%?VHAkSTbTu95k3ZhN=yX2#zsKL!fg0`n z2pvdy+D`Ed>2KMA<~?lyU8&~0nW?pATkka~RZDsBq67UmCOi?1InO`4Wc+frT>SE< zPiGhTL3-9BPm4gfv*VW+N4IFMuo>1V)U`HX$04a++?_`70bk_(HQWd+T;{Kq7JPj3 z-0OHous+Sq-|0FlaIaf{@=VS}9+XP=dQnQJ!Brq19-oK({?!|ZG;0sxg`C9a>0l3T zS};OMoiE8?eH*8rLM_*zYm<_}Hr23x#zw6>3eRYDnNKzR$VYvA$+dnuN%K+8D#c+o zg@|8-*z8cl-e?q@V0z#UbMy2Q7)h>m$a-R{x!J@D^IFC-TWu;#lPV$TstwM(f68`V z?(h^6TY#fr?t$z;ddi?M({j0@BbRkctFTw!0nbU^qqH|;4Q>Yx($Tb*7wwgmYOSc& z-s<_InqpXntSC~!`2jp*pynB#BH=(`*|k#aFE}?EiF?CNtEUz^#DUhs(;sWGmXo2D zXKgddW38Ub{$!N*Ng_|OI6ukHE}^1Ch#K%R9K$_LS4Ec|%-vl7ENwy4;si;gvsT?W|OiT2hXnoE~iEqSaJBy<52+ z?p)$Gk6cXRMoQP8%G+^59nVr z9tly|_70M=epBu=J{H^_rF(5yzg5FJMrhZt0^f7|0C`G1aM2&UJKR0(y>o7ZVtv!2 zEo)eBj?$b50J&(_A(gzE1zkwZ*EZi5!+L3hWcG7f)^*`YXY6I5(C>hCMr;xcG>F?w z=xf&+!KtYw*;f#H21%~vkhB@#JCtt-RDI4y;VA5Dl)WC6$KwxK~>jTK?Gq1iasPz2q>Xv)tgO+KeR)>0S| zyo*ypaB(;Segn>nfmKOIX9(QZ4ZniBmTTwv7=CELMRvY14m8GI)Ro#O0>ErYKMrs$ zTZPZV@i}-7A{=qyz(fVB6j+0A(_Y}l4H9xi6Cb05KJ>E{4#Brf7HVXDHK4`QAcfbE zFq$n6Ckl!KS-^H);M%ZOlm{xeQJs=~ZkU^<--v?5qT;Vv-przYE2vwn|0L&4i-uK$ zxy>T=IQ6?)Ux?no4Gan(Z!}OW0hQZjcns{b^nW!kR-O9&<8_P0ECutHnnU8m&RJ@5 zCU=tduvnvUk}pQ+8xOPyEW--|-?SXxwme_uYa5MH9IRZN7+4n|sR|Y|;M*Wr34X+~ zR4jFxRmcF)e!jZpTfPY+?tlCn(^dNk<25uq5Ws9#cWuY?6sugbP(EGHq9BfdD1flq zQEW&3Y@0Raoh-cbSjW#hSK0*H5x4nR9Zm zV42?hlSB3&%eCmqSs2LZw!)BBj%_Wa^6BBiJRONP@el&^9Ha?toM%gwIVWt{hKa5^ z?`{TA+Q;$~E^A6j1wOlMkZl&qrwb}1{M-)AbuQ9e5&5%zSDQ8Fo$UW(?@g53MzVF? zCphnbwojby9oJ=HAV@$bcC;l+o@Ki{R#oeS0wh6+7DcK!*p_?uyWekRV#owIXsoiz z?A}{VBNIRZ$jmkU>t7+GT|QV3NQ0DQY2Xe(Nb$M|T9j|M-f9ik2UHEOCHkq$;)=v^ znXS5PT}8DX%ZA(FW$EEsfNZf7YG)xBFcjMW~sq^t@1hBqKBE91A>n113VT!hsbf$x|e`zRb2XjDzjr3>4=;KPi@_5XVx^(B+!}0S zH{BB;U}k-g*8S~)+{#vlOi|3oP)6ok(Z;ZWYN>Qj;)We0I`8_uN{Xy04I zLu3kk-47*cexZ-)?xRF_DDS=<_Io`bMXzR4{=NOyns^CQE=?xs-KSfJB+^eu{5z3f zcc1ot9nH46_3>WHuW|Blgj=h7_bpuCTz0rkb*5A?oR4~>%QKQ|cBz0zt-4f1ET03Y z7_gi3R;S67PhM8C2bfiHsk~!Ji%Z3eq+pR0J~Ozz#jUpnE0&q*Lg!xOOn5t6-lKC= zh=1rpr7eUW)%tl4<~@_6t1eJba_TgA6=%UvEf zX+9PNLrbJ9v#~UCA=-f2WCgcv)qE?ZLbj{#N%~wm?owz$u8uulh!^>1ZbMiir?lrW zvTNqNsc3M?>^{oHt#T=X){B0(J4pYom64xk6tXm1vtghr2B_XGvo>aN%z6>=J>Q=* zj*5y?4n^xd&NFzBMfb_>2f)1r{?$|w^F!9%080dM)weJQ-KjM;w7!<#f+&8j##Xjk zxA0s(tuIOc3esnI2$sS--NWG5x9(Hxu~;U2@a0Gi4kThXv&Xs`>p@in+N zU^S6k2vg>3ZaU@h&RZFF=HA$PxdWx&$E~-}>^QqGA3lMp;@vA}Z+G{d^Xsekf76Z& zQ9C+uyg$6V_x5dn@^D1yg2S4l;#vIb)?hd4HV0fk_o&N#3N9X3=pAp1o1)HPQSnRE zI~1#EVUtz|>&=$yK{w0A3aDx(_tJtl4VaA04XYUEJLD$6wyNTU+mDFTPyK+^t_{zPUSB zLs($$T7A!F?q*VLu9=&wKIbxX&u_h&fO^UbE9;d+=ej{`rfzty^ia7QmPtGh&L0bq zRl36LT)pZAE#39x0@JOjxYm{kS69`*%P6~jh(q+kj1L5sn@iyInGTE^o2|bl3geS&$ z!&;*?TxN3%4@Pa_@Dg;x(xFAsiVqDf3p_JeZ`eIxtAuXG7~7myYjAGl0Wk-UB0v)m zva;%IFCI!;wac;rgA8$(ze{nlvcAd8CB6k4TvGsD#nhbBY7UqOb(=naLOuq2g-jfK zy!9T-hRaM}vGSo=VZdQTwe1M#B0r7M)*H>ig3D}pjaJ~-U86sVI-xqjnT0Ts;%bm@cHr%TMcT+2O%C@p=q$+WxdCe;RYcHv*n+@#mjKne8deN zT68?aa4W0o`QF%}KWh9>fE`OsL5%=EA!$WGQeiFC$`tT&oNgD(}q05g7d zXhp-aq2Ym95jX&D#op88E3x~yRK74bpL(M?Gy>5vNM9y!b!=tJ(!nu9mcKV)>uX4n zAwEx&*Cru~Vlmen&A~DALR4!$Y7B}QF;&~g>phkY7Eu>0A1uf_c5m>*F^iIABXYPH zY`xJOYzz_<7ll;owdY^auxx0ACRWTpAOunu2Y4Mq8{!m6r7MP3Z#0Jn$$iY`)F6wa zGPbg1+2HW4uNWNLH6+wvh1e<}PmC?FL%q=)9BQM|D{B+5Xjn2das026CWyU8zyf7k zVgEy_R1~vnt2H>ElZbVJ^o^Mnps8zX7%ijPWyyfq1EUpluqCj3^hYm1*F~ies^#)S z*BZ^?BIzR}HAZLsL+35a21l0jiVw|h5{W+GhAd>fZZf5c!POhh!LbiR438Gh?qsi! zvSrEO_;RGWE!DwgSEg9KD0Vyvo0|Eh>y6gnNRL1Uw8GjF35u2_gA2IXihVF_B5m0P z6W${$aC*kx%|EHzR=03y8($Z>Y0t)tv8xO!zN?X zQ=7Uyw4!0j(9rK2s|*47STVi`h+`Cy7sCG;6OIk1+G-9i@*@{DpLjq2gL)t=MfND` zJ(diYEQWA}k*X(IAodD57RV5DdDI)t!IDEDbw>zdRtH(OEE^ogXS4OowedKV2IX;5q{^+t1O0_fmv zbAjsE%9f>rW4D2H_L~{B!eg< zXtAG&p~Ac~g~gm!b8sHiYsg+KZPqv|Or?j}pLb4|B?AT}9IV)CFhbEa$?w9P5TI~S?jzJPLg=JA=lhhPKbo6SMWdjy{bj2G+b71>Hu|ww~DFZ#g9&okwh6fz4 z%1xq(1jwcUsFK)hOj>iptT&p&ErCq6rBr~Qmc314aVwV+IC8W}8+^+v&X1bI0d7u4 zpI6hk^5aW~0tLYG534jnQKlh4p=dfbit%0Rjpo1#v#t7?iiTxFqX66rdyaLBhm4w; zq;87ZM)gu-Y;#)8!66gbbBIMmY*`pQB;!z(VGo!5TzmhnH=0AkPmD7Hr+f9a6%EUV#!FwZRZArV0VD#AM>61y zHNR=S(Hxq<>jjTk8(Pt@WN3g98><%xq}B}r0SJaax9Q)t2sD9ajF`OB&y*N zKufDh=Q&-L445TG#lsbriC&2zNxAqxRd}&bA@CTgWm-0x!^PK4;G#t~boD!zEz1VS zp0vtGhPKcUbSTWWY$uc-210*6a}i;G3cx$m=whG(d`Kt2JO0E0!jUhUzn+P@>iLYtg+b zU6u@(ydM0Ip!%d+9IhF1)aFDzA+NR4NL;c=6)WyK~`Z!`x-SfRz#2y64PXjnEh)V~!& z1BuTbOn4_?dSf{&_gXQwIj!d4h_9jCP%4p)0!0qBtmd1W(`DI!afhsuHc;9!T)wFo z0QMv#McZyO2g}68z9n>4`^tI4vVjpoUZE!cGcULcV$wn^bgtbEm?W!dn!=ZbeH@DF0>6377&#Jje{Sgp|Jfo&ea>u!AZgqkN~vp>YFQDmb^LcyV8+~kl-U=oQP+% z;aZSDth`F2IXDT8iZvq0SNpS;ElUOmh@i1jVRa>X!{!^{LlJKs6TDaxxX!Ar=I{Vs z;{!%XRK}{n9je2u_gFSu1kH-g1up`VP_z{saQGmscUf;V2a8S81b2q*S{-EBvUG3= z{N+0zTMIE_QX?d507Fz!wd6j9dZRTsip(L)h*Z@ESF|i09P?=T;COP#v)EiA7xJ0$ zcCdI(t2sQ0m!iC)$ybM0wk#jss&^-lZG6D&&Adb$|8OCh^RV7(4v*?YEMk-aCaa4j zqip}mdZ~9=Hef;`D;5s>3>G*DD%4Ej=dpaM^+t2JA`@BEqKGv)9T!yft=4-i8?4Nm z&h;#S9IA>F4s#-d@_(7+0<@VPSckSRa}TqdX(y|%?% zZ?y&oFo^ZRtytyQBD@Xpqk14NaI2adX1&oIuEb`W@X(3u zw5Mm;vTSfNEtY?1_FG|P!ER!=0xjvA?uUA#IXGssuuDk=Dtl~2!?K~_=UpMA(Y?Tn zv5*OwfNVrin=fuot2sD4POj9ft5f)uEz5?7ez0O50(rtHlP`-JCaRceY}Omi!8Moz zOx-!*#KdKJ1Y*{`$Z^he`0YNlLI1G5Yyc=_s)Emvg3Ak0zt#u)tH!K?( zoPnz>6!OAA=TRx15fY!nQZe6Lz118XONxb$yGSS`#mzV;ep2tUY`|d1R`?dguZ~gy zMhlYwNo0kf>W$`b2|nS9RS`gYH!K>K4Ncbl@{O7m!>B;n<36f+G3$-y(9m|EW%lbaACJ7)9dTU4erUg zPy?v@{7v;$|B^FWZNC#RhF&jF^)#Rs>_~wVU+f0dViQs z_cs<+-d1$Yc{g>bOO@Q7L&~6N1gVd*h_xY`5P8McIgFHx&a~(l=I}iT&u&B-yl>aUuE4RxfceurvaMQUC()w~K-&t}T-)n(zEU@EF^ zQd6D@rhe<`8mZrUTIvhxNm$+@>h-unJ|x{!Pu);mIZ}&5Py0{}Lr-{puTV{2-BaGH zh*w)H;-)gah>bM?xle;%N9;0fcUHzie z0_>?#nSV*txo+MK@v~J}H+@M-IS2Iy`%;y1lSnR`xwYvD@y6Ea*m0YV=WPa!&Co4~ zH$Z@~^-ntoqh4>47KM(5WBcG6q`LVMJ4{Ch=`d*z(y}Vq&6P2!QCPN<&hg=RG#hqx zk%=m^T+*Hhq1MTvPBW%d!eKhu-+(psr1)RxCEADgPbu_V(rnDT?tvnUv#Oh2!{w*_ z0WY(c3?^xL`@cTRhjQu7^9*z6Xvt;N5 zp5wWjVRMrodAYPT!`BZd``I+fxghfuL@a9ygk7}1@#*KS*Sqhw-j%}snq>VY?sjR0 zuiAE?7{17l6_*C`tq@}xAiOkSK|!*?c;}J`07)4EMn-g+QM~L%FT92)16aLf967QZsQ_v z&&~OXaA@Li#h9TEX-N4aD_x(MPqme|oaqbCjwt1qw)=`{*wz_7nN8>R4?T6hP>DI4 z2~INjlAKMPqvZ7TOlBDc?U4aKTgCMbpcriMBHIUV~oC=L-A)(BEn6#RVWeOxYac22x?#YwT|{x4n~ZThF5FS0ePvJZket8p z=BmWsCfP@&MWMXFNJgN-SZ zoG(XIoTqO*QSIZj(?3m}QO`MJZcv4HA~kQvoKaG7$I1(Jow5~BRZX}Go%}h3D{0nq zL@RV}70J3c8al5!Q-_!0yPH9<+3<^{y_T#$qInLmo_PmNm=HGrx!vD*eRw$eg|*6_QFnJfD0#-r9Qd zYWnL{_-p4}XFh+eWk=_FhVEJ`BxUtIAG3BT8>{5nmU?xAS@@dQX$=c1gzzGHYA&$5cWTM#pMh&#n^MlWVg-=IqLMzY@q$7)51(el(f% z+k>-%NjjYLRn6EucE7Bi`g9uUGGR=RXwmyo0MF9d>cDm}=F$x+Ci)pa2CL zwYAodbluKn`cW|xjljrvxdd|2bu&^HcOnQy-oeS~!3j>xJ3>4`R*QlFkvz1d27x3k z5_KRw2X+2Q(9g4^U^ypk2WWC4I!Mb6V60yWT?8mh#Zsn$2An1ki*mw>-U#(UiB=ZmZ zAd$ngr69T*DwP3^)sYNBrGQQX+SQ<7KvD{xELdg|Gk`?F?#}Nx>T#T04Fy$B6`ZQ4 zz*&hpSN{$}xsHY&YdEk5US*)>;>wgL1Bw;9J%M_+Tn*z=nEkc_d1;_=SN-nD-AQku zXFFD)V-3{O5N1z}r>n-(QT-FHW{Kk|M?J|aASp~eOT;&7 zwG)0*!&AZ5H0)Zw2o*Ba81#EEM<}qhW+zqG*U;}oeP8`n7358QQ=!R}FJ)oX)ioQ8 zGxqfR)ZjY}HqRjE>M1gqxgLXhhYHHQcK^=?1J>s7V4R*}?@UJrmUn(miiBjBLwv+$ z19O|yB(c_Zy_L0{*%+Op+tjbxm?VtAh+lHS^V~Y0F%~Lt3mO)}vbc*&5VqSEK6gcUHj<2wTw)8|<(@Q1Ig>eQJ20jH;s8{O2)vmiZ2E zJh4I15LYXE7`nJ*RZHCu=7@n!e7^Pa`O`OhPk(#*_UR^wESKj@oatEkf}Hb5o9 zJnl{V<8J?W0HAQ#|JF^1ePbwMFbJ{EbNxTRz1V%Dffd={oZ5f<)_o7ngYlz_bUR|r zNofc3iT?f$9LMgUf6{lhXX7q82#@;|f9s$8-R$L@kiz(hQJri$55A>m!}Rn(u#;)m z8bNWZ6dn>5!*#+uGzmjR*}_fsXfH=*ua^lq0{cWPUuPQ{?q zIyNe+_jal~MMN@-VXS~@y1}Mfnlm;`6B>64Jqfp|aBrJ}L?S14tL%0frg;fLpQ9h- z3J2_&hoH^><=*p}qAIPGfQu>taE%Ew7>q8EX2Fzc^C{O``F5ES@MtzZPKNy3_Oh6o z9F3Nc7lDisJ!}tVB$K>DG(%T1KJK4Msz8Z+Q4oUuke-L*=USh?W2*#M#+%lIcV@4oiS#inwx|{b5ZP&UM?GcBBxb73wn6R}UdPpTNB5mxFoYo)+w<;#?~Oo|*dTm<^XkT*&A?C~$aGT(oG z-@FPebIx^2;WW_zH>A@3hUJ0C6_W+9OsnttV40>Utzd6#h3W=@nX0NYz)bc1#wpgf zWcT&@Qwi#ta7)N+6)qEMWhPX$r+ml)!il1&x(b7Cwd=*8nd=EAL>XE`T2HVGOtAB~ zQBN<4&K%zgbf&qvu=iG#dWEm3%Tm*@tG8NvI0((=xe%I{Uc|)Y!i$6V=j`r}x>bk$N+ji$%XDuZ}%2)UPn~)N@tzUvKGmNG!80+0~Oyyj!L=B@^pW?+8 zE6A*R&x>I+*Nb2+$+S{+epRWMi`xD#fy&?8BDh&;i$`WLN)u}BD|4+#K{(1Dn6gj7 z!3lK=OSx`E_GM1Z2Z!T+=cqC6IuCFaeM-J1PG>Y8i;pX}5iIAW7vEE92Dig|nZbai z{G?9_8nk^08=Al_n^rf6!<#24J)wkx706er6;YxSN^t!8P^FRZZ(rKpbL+h;*R>s! z`B$NnTt|G}qT~Q9i=;M-2`?*-(Kl0!K8v$5O@?q8u05F(oDv3{eU(pa zI6PIdc>Aa;cN`CLY8l&inVt|XByNc(G#K|2r<*z_eK|HcNu0sJ=@Dd1IvqGkh3ofP zU5_D^Dr&Ar&}$fHc92j~#^(eHsmaBwP+z-@9a0)j_}?us@7vkXd6c#l*9uGz{-#$F zwowS0w{t<)Bf0g^o8BBTECks6hk>o|JQh^e=VESk6*KH%zqjJTbCl5QxW0CD>Q|RP zm&FYGNL2|i1~(zN-A={_?a{cKj`0@_Ml72IcyOch_9{u}_1PDn`ELfZ2 zxq!i`lL(0dARAcpuy*`S*niE0kO1LbElVtt0-keWWwXi7Z-&6dm7`eqjdcdhOUvYO%x5}Kq) ztU9KC`;oq)d3QCQK%eO8>t^Fs=Vez^4Gn!Mbp~`R>gsd)3TvM_7wk9#2er+Zva!_e z9{}bmo=bHiVpyIb)h0MQ)aR)S3gn&3>V}5ys~$SP*Ws>62e!m=X#)`kk2) z>lzcHxG%6IYA+}6XIT~3`Slqd2{N;UcQP_lv@A>`{tap}QKMPXoL#$coD?j#T4iI^m`1gW8(Z&M zy%bef3qs?ytkug}Ewnt6U($cB)l0*iEjKC_+giOuS6r)?P^NZsN6#hHOS~m(^|B~k zmX|VHmoS?aZfvDpiea%g6^0sR?Z6Zq|JJ!UVpI zpBj|x^=1esw*$Xcx^(_zRX`!-3L9#XzEb-8!Nqwu;C1J-VS+3MLQq2aYF_t%vIvC9 zN$azoQF!qb5F&3aEuja`M#=B9{&0IV?2d+g=LIhzaZZx%nCi6rDl=ds3AX;#eXs%| z@m=bjpeh zt(>2&DiWuhv2*6sVGp?B&b)3S4W(1npwm%~6^)iZnoQ3I1g$!w`?F*E+YpE+i6oQ1 zRb9A~dJ>FiH|-9ceCiC638iPolM&^J90iw1J${h#ML{Fd&^ek89U_&fo>eN#+fALP zl0~-pK*oBI4fb07pG`6qs-!X`)Kw7_bN!wWrTO%%`aJ_TXyH+{^wM`nGlig_NR1?6 zh8BWiU9IMat5yRsK^#%>o-)kBj72bXe`DuhySa1p$-4Dj%o#e8jc#jV2M1En=hJK;h4JQUMp{X-m7sz^ zrkjE)$w+j%sg7v-zTiL)`{O~51x3ptB;iXixSdMOxZM#b#s+!t9O{G;@9jX?!rxv9 zwj^ul`fYF#ZK=UB>rke|x%PZ^DH2rSosl4}4JgevGCP-@#KG1abtr_(W{x(rbvkz3 zFbe`VN>&OLxwOUhKx}Y0(iq%**wFwyrtjYDZoPf~W_NpQ zd;7(%I@a0W5Y!Ki@^s_b5KyjBS=I6uRl8u>n zz{tJ;KNN%kMC2@60`2euTKZMJl?Tn_?>m8yTbJh*nCceIrTD!w{X%sl3xxT>6-Vj`0jf2HkDxz4iXxtL?3~yUwFmZ=45bhxtHG zl4HnY{*NRA>5-A^(Z-D-;=4uZ*}bioTaR~l9MruBQ>`R+lCk(ORP)>27cZo`^47zL zMLRx^YtIB>0ZGzEMQ(#~!Ea$p?nYy7GDyT%tNJmhpQ7N%mLo&>ao?t=4yudSRBJmZ z*7jO-|A2&XFvgNZ=20cV!_Zg70rz4`EToUa8xY+WzS|{5_nd7_$!=6!$S?llEHt_> zo6x-Lv^6kRH69u-M5(Vvz#@1=p>HnWLn8TG#faX@E0 zIu#NPav%rfO{$*k5US@&A{q)hX4h2Wr?_awS!1PFH>EDT+n-z$}`uy z85{$ZzVaIMH4BXjzh*=7bO_K$3J-*m%uc&galKm43vmpr*MLz4YP|+dvW0c~*8s_L zItVY@ICdYlL`ZNX@YpeIlhAIxj(z@Z?>YPY?soIl=PT{=_3QjV`~120WXzFdER3&O zcr4%P&nZ1b&fJcDKG&C)C1zfdw5mV66{J}@ zKd1b-Jv;r)DwIp%$X6x-0gmj8+gJh@mXVjkk+W8|hb->*;4mGg{o&}~cr@hy2(ke7 z00@y+D-2cD#SqK{&fxZT7t+)n7QPt{EYe>~WRW(udDCreF$lt-)?6|K^ROREY zvtm|DFiGQnI_wr@u*8jQM?ZxwuSSH|niu>Gh5f>cd_EfYRoc0LmME14io2}8p->HL z&{COXXYtZ`J6ql(2`J<#csWZ%g_rJ@u}P>M0LGYXJ#`u(q;InRcE2~^8Vo&RyV zqZ8_ob^!F#=X#UOBvY`WoHXL|X(uqW#k|5QILnU$xu~U~2r-T-<{3#b&k%T4qr_$j z-t@cEP-+st${=fGtE*xT=i<>-G`*)#p_UrQyz9>KyI;fQf7o(l5XLn?VNi?=++a*f z+y2IMG&oG%UmxCpI&|T?T~Zw?b}Xd?`9LZgzNoTmCFt|WEJDU+c6Hzvj<@r8w64Nk zdwt~u03OvfiK3i6flv#;MiYv$UF(iB@6S}l{4He-;*9M9QLHMMI-51BE@ zYQ{)xaJVng_2{Z8YcqmKl)#*WQeRx(E~CZ_@R`x9I%N!(SoU%aeVS^{&QSE6*)`N) zGagHI0-R`ntJQP=rNv8hjpdQ^JC>6wX`xp z%x}#O9Vql5ul7CAv}>(QmjhU9Wm#PW9Mf}(CtM5VDRd!Zhd_5JS+cPo>~FmJx!L^q zZfExsr# ztCc-IQ&CEZ;Q!V2lDQ!{S68cD>nsCt_LY%3OO=?D{I}LwMD@D;ItwBAwa#+5=F$94 zXxcBTMHFoCb6k2AC~Y2sEUQbkE_%1M&N2`&F**2sHlhl>_DmOrsPw@saS$76oEnLQ z4J@v66tuq%U(!6-w`fNmRm7qIgo;&ZNf80rt!PJO+^gZ;Yc{~_6i~h;bq7=*h1UM{ z@WqRdqphv2N7Knwl1t`XXFe9t^1$Spj_z7ZBDDIRFNsk3672lJio#js4~ipVdgc5< zGg*YH20GQx^>L9@vjQz>pGSv9CGRa{;j(M~=RZ{Sx^qq}^?GI*sd?*}b!T(+%sP*! z`;y5(w}b&WcTJwxv8mt-D@~4a(+;FCp8ze-PTHe^lC8g5+FFh0i~XRLCP6nch<*KvV=_y;}*7RWr`;(lDp&d~x#GjcIPT<(DZsTquQn zJ=fQA>iil|F)s^@w=gZ}9n+Y<^=|njt@pi-xqh>f6TNjI4QcbrSr5>!x8Ta6V0l%} z2nl=HD(bT?N2S8Y$*1FFdo&p)Thwsabnd=COQaAeDe^<(r<4cXo;mNnr0KNpJf?%& zYmVX7ev*!l(}}~Seoao3c7MEPbr}IMf=}dd6R2CiTe>Nv6jZYH3`b6J^(}2NA zbQpe%LI2o!nH&%L&UVV*v(Z>i<-*eCaXL`OG6p+U7l*7Tdx)J7S<7OzVLwB zu~qy7b3&^moA6MTy7VCto{E=J@eDHOs3qwIn+>8OtunGIAf4|H@3!@DXLm=|Mec^0 z7XdY03+Do)QO2-+c>m_=(yzGwYQ&qH6JF+zy&wT#SN+F$J$XN?oaCdX{MYCu8|5Eg z@2BBW4T6lQkeEMJCH+Rczp-`v;jI7g;g>SBnwluYEBGldakoqQDU0of&}z_f8(s+S z7H_+Tc)J~S=`R#pZES|EO|K*)wqmPZ47;V*1Z4xSVlB2>Y2}4w^80(1)ml#1fLx<^ zc(n8CcgX!$JPyw%1t%zHI1` zNiKY({xhearCWR_rK+PS3?gb70HkNSx)5aVZ$yt@`Dc%IUzWPXHK`UCzu6@29gu>v=FOEMK5%v|6Yd3(^aqBh`Kq!w+KoWWkpK0koDtMFa?mX5j!u<9Q<2 z9u2cYE5Ze$=#e1XY6c{?Nu{vDP#4!&rPaDU969$e?JpcKc;S6|`^(|h*6HJq?p4Cb zbFMQ#j6CPGjrMU2@4Jl@>#E_TE z30`~t4sU?lbNZ(h^u3611y2>MlCJ+=WysDoekJZ->W;>4SRS^v6GbAF*^sfB!o zrLm#LRY?kT5jsB--Wp4($QZGtM4hzCFv{4_7D0gI;B|tXtr84lPayoU-w}l@R(V>n z%I_mbLXK#-a;&ln<3McvI8dpF^ZL5_B9L0q3o0Nt_9R;?Zip9w3#w*)NMBuLgyr{8 zscHE=B%e@zOWZ0}sf$^QtM9H-4KWJtaZM-zxIs6i0BlJK;Pv*0_xV3c=&4Ic!dTpB z>k+EwQ2AuB>+3^$h}muRAxW5v8_YjlcOAw`36x7a+{_?`GH)8dT9PZyKU zJomcRlNy(bS&R7zkui{qKQ?>8$_q(mN72dn5awe?=6i}s=q^q$h8LSJ*63q?p(xwz z^c*lhFIz``(U=XS_-(xKINMUx$t~GY^Iq1QF=v+b?#=PBvF0Z)yq$aPg(q8r+O4GH z`cW-pJU1IRq>5ZUWE@eaz2ef9I^552W*{Tn=BD2Sr&)xIFGb>Z(=Oe0;mI2HHIDZnkNg&>k@!CXC$D~AtM zz`987(B%yK5bJDeX{Ha5F8KQ_olFm;9$(%x98LQ@$dod_n)F8nKTJn?ht6m;?)Hb& zJH+j#FGrXWrmgHBozZYQ?zbuUH!eC{FqYehHqlW3KbQYF;E_ac&KS_BF%q@VWzh%V zpOyNymcCtS<-APa7UgZNUL#xz4gRm9UcU*x_#dNRU&`88;DvUZ_yI1feP4ILJByIq zf`~Uv8Q6od7c_iIh59a~UajR;YvmJU`ZqQYZg%)$Q@6K$l#F50crK_zz>Q`TC)k>~ zVG`43DCWk*ML81#Rvi)Di5ab}LzF5SL{Tb?U6tS)X9lWj&umP6$j=lv(OIqP-#taT zHl_mFeC;2j5`B%^=zCh$v^Gh#v6(dp5lRfrsGOl}{6O3InH8E@!s;ostsZDg*fv?! zRWiF@=!Xgu+Onthz4YqV_Nz5I_4c&In;Fvjh<=lH8k&zQRw{zoacrPa-CYK{mrU+K_k#IQD*Hn}>N zYj?XiB1A#f*U^=G!95HTMAbo-^EMm}rGc3AH0_ureZgEH7mXUfT82_mws5{SU9{GU zA{+%6tu}U2I@RC}!oF5lguN(^ar0W>1|bXFNjr|;aJ+`l(!fuXi%L(}nJ)@%$abWg zLpQ8d;Tu(j*=b{@p4STD7?;Yw`x{SRe@R~Scb;B5(xIDJp0!Q?~NyGj&)&UJKrs|u}{oz78eb=O)l zxYhT3AviI=J+v;2){PRcVt$;fpJFEW+4a@SR@FI#Y#QxRnaT7tX^qLn7FQb!k}z<;ZSOta;GZd%g~?8|X9~YUi0H%VOG0g{3j=E9X1vDo-x8+pQN*4yoij zmaQ@j9n*er@G(tEEIdO!==dqdXQ!nC8c9e0Pq&BV*p7M_nVqmJ?VcxwaME#A=OmOf z!!{OYI}*FI(`<`f+U`cQ^H&a0@kb7|1NleT!2sv#7~@pj;p$1+==_s*tUP7$xn1Y( z*5o)J*xl?0gP!XwLeFYmB|GAe4Am;oB3@xCFH`fzZN>r!n(J0Fl6v`^<_Ww9^9+TR zk>wY)DC-n8(6Iav63RfKB@KGHtYdcJdhg10?PZux6FSLt%DPUBEkLv5K}ebsi^iUL z)mF~W6w|lw-t2DeIXmz7p6tAT^M23Sdb#7g+j{KmDMDi_%ktptTYJE%rE^KUcoxQb zw@&-RW2y|4Q|DR}BP7yQ@fiqTj_oqQHBfgMGy;}pHQSQBwu-+%Xl$R&h7QcnF8+dW z)33PJ*ZzVs2f@lv_=}c5Vgm@nkbLE^5jVpSPv*uidryD);(gw}34g&w?{;Z_L2dic z{sKP;Joom7&MxIIV85uawPicV`I!F;^k6AE)sCK9>kao_=qUv&6aD4hSeo>3>>x`O z1oPaE2SYSuTYE0K{~>bTMdo**VL;l)N$0<0OpAn7UMgsTY|5X+tuqZ>gK~B}VrYxW zD6U==Ndpd>@gn6+0Ef~l!^o7gX+nm`0qKFFQ8m&Iae`);qBs@h0RiZcs?W=%7!|P_ z{^IoKi25y*^-?yFE6&%P{CmESYvp!?*+h3DgNYJwpv%1T)yAxzAhWNo62Y^=CCQMA zLQ!Ap9dc!?=vBFntt*vZjc%s41X*-U@y#Hl24@^o+|%^7)f+jx2+ifSS6lic^I-#E z6s1eAEiOHK7-@UjvkN!fp}J&8rcW4jxdPZc0$cc>xn`>;_^uG;p1)xCN;qmvp0IG_GZ-=CilEQBn?YEy$q*+Dv$(~u}lS6&I{Z`P1ckSJ_>q*Oby(wx~i@&+ri|>+nefa`<`q_QQYR?qE45` zmGYkdC0Fs4O~Q-dt>{s;Sc)9SGrvFl2U)k%qyMVGM91#8XU`sXCVgsq{^Nrh3l|OC zwpZ`#ZGZFJC%w_vw%||o9Y4Hzxar6nV{ucSS3Pr6y#;&JvfQ((x8~PZ_tI5(^;?qy zCe4n7H>0#yTrEmkYD>^p5Ol5hdgZUw&~2oyKEkYEjy1~BseXJbptv$Wx_V=2_ICE9 zrKir)m3_PF+q5L7>{_+#mcYy^YPKJUg<&h&Bo55cv^{w0eXOghvQ30sl9g-WxkqW+ z@jb_lHqp2}zYNc{c&>}8+iRd2c%yVojAHb@pJZz>A3p?A^&cTbPA9fE&75D_&L$YuJ*HeOD3TNlyU*tO? zPALJR=0vb?pxSs+fE(3@H!qs;*3M6IFb0q@d+rYT^bUusByL-ksM3n&FITCahY>8mu}cQ zY67UqAh82;(p}~^*Iww-eenD2+t<&|gSY*cU%!6ERCXSmB*&v5KA4=siTGf0)E~}{ zM-!+Thj$-7dMKK_$6^T^spN-+Z80abX_KSQq-1mcJD@ASd6aHX)>3vnZIA~i=lOuu zWuD7Eou8Aa{JJpeaV+#C#}8dWtq;le;9g1zm+U=uBDF7wx}KjMtEz=?JY5@@4W*@~ zFf&Q2`f`9)T=z_SzVAio)>VBu2E)z9b8RcUc=cc1Uk1& zVQ#W8VoKRJ8ch*J*wGtXF9#2sliD9jhW+hrBz+BT{aJK!RWy}2xXw*t+!GO@#m zK7m%Hj$$=$OGo=#d~m?g4U#-tr1c0Ew#1HsIjpp@xo2ve?1yWO?Yr05lyX+{dcE?# zJ$=#M8FwWOcdlFcJamPcmFQ#yJ92v8y$Ve~t`=&FK3+$T=Qe6LZ@ohnInNUd%yYA*MQ2)y zR2H)~!sh|IW+?R(aYz+Qr##40WlCFaJ7h)=4c3+KRE#3pEgj8ulem6~(s`?^y?1{a z^gpLR0izegO4AH-rK%$rpSsvbfoMI36@hOR=dQ8{I%qlA1xg?4Hc>IP!Bv_eW!FR* z6^S<7lzF9npQXCQxm>i_Qf5xrR;pHM92oTl+C#sjU@KeBE39d#030BRxQZiQ0yH|n)f zEb%WZU$uTV8J})d4yLRRUVUYDgUirZne}3L3~TBtKz{$O4#n(IG)QLNx!Np8h9R7 zj-`o8im|XpdQGYKi}RLDk|MD}6zq$7Zf{-_tx_ZYRcIr#H^=sYQ#WO%4UY@|@6JP< z8 z$5_!s^w4{w_}lYVG`$H&v8YrRy4xl7-ckZ(XbkXY&AHhd*-NZAj=#4|=0$J}W=%Tm zrrm?N)RD4HuC>(fUQ5N@Q}ZkC4nOyP`7qiVC{gE1Q1kVNdcQAT^VOrbuii~}q(?3u zYZ`C&@!L-?-~GB5y%TL{LB4WR@|9Q7xMc#E%aq=GA~5#|Gnt^9MbvF+@mySCPTt6J zH7*;t*7}mVvS*LVZ2*F9DZEgrCz@QM<;$Bt7~S`5G@H)ux6}QNpStnR&%O$l4HxDCs!@h+s}gRgw%YHZN&ZnR@ZzM~?F*P8 z8E|i?h7S6}9{dXbzH>tM>9qeXX%C=NP#Yq_m-A#RiO`yUIr!Y~rlSLulf!na?Phbd zr1Z@8H~J@s`y11rzdqeLeK>tPne7}j-uM^KY`V64&Yu)>mx}IM%Yw4{&Y8{GWHXam zDrSzA2_>J=bVlrNs29sC<%4F4!s-bB)UQ1MfXU<(d*^_J_dXx=UnJ?peDCf|9T*;* z^am)c;Hw)z2RJtw_mkmNVt@a>)9#}&Pxm}Xm8nKC zeoqdyv1NXoB~yX(_BWah_iNB3ZS=*`NxjY^CR~w^>y|)p1j;1J^d{>8+|$0`u!lBEgod}2amDA5YgTCH-7J&On&L? z{d}_9jXw^z!|D(#Up6^C9b}9v`x{?>Iyp@b-@kcrjbHh6u)pE&^o|;@zkK>BZGQXo zV$isM63n9Fym! z3rklc6(E@g%aGHl;}FCf3OE_Rqf}xy8djHIIZ@FF6E3DBByv1#3*1HnWLhNt%GCrK zM<#&=$n@^tzpoC`xkqY%4!eYDOMrO9e+2)gvtJaZ4v&psmjpQ{jAh)wlaNT*6zF@{ z?n#WQ2THHf3y`~tQ8hJMM-5m)G2nxEfl;rk0z0677^+4O>i!WA%YYRc$h<4jJ02?^ zL4-r;kvdgfUc>a7>BHCRU323c6{RjTyeltIUKdPl^7*1v1$E?wge@IQEAd2(ht^iA zA^Z(qAfL~8ibkX6xq8*x>b`jN?Ze?KX3VlIA8l)o)6ay5?oURYJ|)OD_7}xs->81h zTh(u6G&b`Qn^Dv6G#pEKDa1l1#gYsOV9_eqpoq)PX)X6Gb0x^Tw$hio$#gP18O`AC z{M=8!&~8O%n=Q%EXru|BGGfD*UW?yGs0G1(F_293=pME;*97pYw#wSVn z+25d3HCC)W=xisfQj*d;Xh}@;4T-ikTe1f<7340A1zZ~X0zFlWTWtVLcDPmPL$VaV zo5Dm8!Al^0^wQMe-171B0QCH*GHWk-P<#s+vMfinDK%4_)*=B>lB3iuBuNG|tXh#} z!VLR|T622kWk^@9;S-mY>*zkXS)<{Sl^2gs+TD{ivG<$p`TKXoFpHP3j&({dcRReBwMf#|oN4_=BBbzkQ$FJ1x4 zMe@nNw1nladX+k7EotgZxJw1qvt6?mRj;Q+)$*Y-m2D5s;J z=T&Fwc#$I&82wG(2&b=k3KNQSmN9HA!u?m}9q>F4S2HOH9`KczJ!qr*^uw1I@8TQb z9k}S-E*apSBg$2z+w<96e3|E!h<1)Cs~o;9f$p2M`3l|v*%Gf$m*pFjmNo7ISeecf zBDFN$IlciEA|0dV-XENSQ9heI`YfbOw&h=MFNfkmH$)P;9?Q}|1+b>H0Kb|EC%KYYO0rK6ll|ius$`xpxalp zAg6bVPL-?J7Krg>=qFdm8Jq&>2pUi)x+n<6Aj^aF&b%_+bkOUI#odsAs(pFYcF;-M z<8++3l1geAINP&yIGQ+HeL_L^oUQSpIFE+QO7&=eBV5rvrPqX$1)RizY=K=~6Np&iK4lh5@1h z{p*1rKN}^(VdBQl&bL0ry}P6K_za;p>CebP&GSnk?!kNgzMBlw!;#Y+_7mNRONJ_p zIAHRQQhve*5QDb^7dJPWuQB=e!mAKq{wW$$&}uaBFhvoyA!W9d`p*K5$vi>LukwsS zV}kWkN}fQH2iX?r7nKxkT-sKh>b)!1wUwOmGkq0pY7#!+!d03~uP#Xn1yCrlS0}oS)quZrE%Jk_=t`7Pf=@c-nHDyTg&V zP5ow9@dn-ft^aO2J)ZQ3ljFW~x3RV3+})c^Q|IpER|1{AR)vnQTTgZ-nX0PMsXBMH z${kIr%Hv6^dBmvr%3)ilgNF0jY@jksC~*~TQmv-QFj;d0{cs*JAzi>976)h|aVv@< zfTpke@w?wUUv5Cny71jDsb?~K95Wg3zp(_fFLZPl8FVaLpUE7 z?#O%&QNil+%al7cQ7C7bO4&_t2Av8^^AfMrl@SuaAlUhv5N8|DS}0{@s|-UcWx^Ts zDIE>l@Q)AD$)=N#VSK`yNXJfZJUVec&4!1^qtUUFZG?Zi-BwbIr1yS(*!Z9RG^=`tLXle7bjd{Ls0FQhMb)2ZwR@|LeD~b zI&lW0&#BWMO`X3v5;K^cZaNP{i%Q0wBS3^gM}4jKeiX2_>tZwvB^}8Q;=n&OPm(qQ z&`@ zA(a4YgGUkeVd|xO;{M(E`SI@Y*V8Zl4G8VlMelZ5pN5~mzfjX5vf~p0%uw2W{#!k0u*mTS%`e1 zh83f_+R8^NJqQHP86t;1wZ3iXLaJ>>EZ%@N{-^dfXp&#tEJ)6-^Rwi%M`STxKQor6(zM z99QANunSon{|qIoQI|d@3Kbp5KO{v{*Fnw-B!^Ozr0vHIn!B!44e2x`^}6jQ@Y3CV zH5yp!n~} zi;DjwZ21I+EiZU+kawDCSLOG0>$UWz$?5Dk8J-Dhy>yi7=UnI}RaX=!RmazuKK)yR ztJH6)Pr(XLy;{DgrqldmXxdA8dh3|GrCvUmiigOpnO|tn_O|yy<_m-gb z3%5gNzKDCw+C%J!tN<>`J;axIOc9Zd+`uD4%m-)(GTGKtsMuw-a$csXs4Z7^_U~OW z$vvxTIpwK*?!6!qr-HHo=PNDRrFuhkATsCCTyS}2hVtSixj7Cg7pfFU11(+&{oYY_ zt9^oeVdIR#AO-Ejybq$7T+D_?UW9no>m4&CzuvoYT{~`TsIK&rd*;b}^$Qsjt=`Dl zrScO<8Q!id#RUXzW&D*S1h*x#dD%Ehwdl^vPJenPLmMxErkB$?&2vps z1s$F?2i*^ytzCbs_vR`g>N(fZap!Uc>JYW%;mH*#-L)2?w)&ngL~W=G6{oAMbzXCb z+KN20k!VJ0MK`@H^bEHKvJvuX-xYSEYF9JxO#-3nshFwlju(qk+f!&g!3uZddV(#U zU~GbgamNG_st(tU8zkP(nLtMCb?i{BqsjZ-t*xD3hUt}dsQPt&q#epcysX%EF0h%M zdxxq_Aeg6uAZ*5A!}3&1td!|xv|h&qirUlnovok4x4&IY#MQ5}o-6nsfn4%1re#rN+8l%YDW2^6&Mh2aVls7j1(DD2$&Vjeo#5bEZ% zp^?-Zt;ahz6714CbJ)Dsx;SoljxkkOb*o()s;j~u&$-Te!}+6(vOE-!MN?Ec=G>`n z1xJ>FVQiz7$y)`3)n|CM&tkXxg}`9zB`}?)V_mvz?UzUf1Upputz~|RVk#EFr7{rp zoh$)ct-PdRd&ynCZjfGXm#<&vN8088T*LPEO>x7xHZ-N&yKJ;ts?sH=c->^a+!U{0 zXFbLLDAW7aO>xuwuTOEa^*W|_-Kf9Z6t7?BN1EdD&d%itVm6+fOZ&KpbKn+CaY@{` zVfO{Nk;N42@l}!QimoaMN%FSWB*hkJDqC5TS-kZ@XEZvc`t@ltJxwP4sq^tDg={_H z|7XtO2vSJ;<{?B2i{-WM1&tM8tnZChoiunCV)7;>$f9$Rgjy%4ooo(CXp2l|A(L=} z7KMMM6a;U7AvUYtyK-F{6IJ;>^fmEWs1z_WH<}IL$l$}|m;L3dt(>3vf3}jtx!m7A5YvsOX^ zQWSKn3?rjCpCn@n{U!Zz@~NAqr_QgV84?0MsKe1Pbxx8lJk@6$QyfuK7|zp^6cpzjK;L|Mq}sU(HPEZC5S-%-oMYS7wI&f7u+hP#smK! zk@EqmOVCtcHbe6$e1e&S8g&5}xpjh?a&wfLunD;pS#2^SYt7l^@;;So*t$gJ+tSxm zV4@bebVAtE@>=B8p}HCiq~6Hc*`njJpHIih^e8#mNe4%L`KI=b-~QF1)pAAat4Qkqjq3>r?I*_6sjbT~}L z%-LZ#889u2dok^{pGnH)wW>UPr5bJ=ZSY)b4~#)@!PRvik{3ifw)CGoFU5x`Fs2HP znV0fi5}2)V<2Q;MhXYjhn`Xc^&wpFgVf7!K92`FlZ=#Ifx$m}CeQtzQyuOELyYQP# zV6)dM0nYX&YELd0jJ-TRBt8CQ6(_thX}E9zARgT+csy5e@>bdHGZp7^Ivx!vXLpiP z%nnWE!B)b7?b#U3WMbIHfv``{RZar$ULfG}W!Z1b94>z(?Wrgl6rP8iyU}R5xJ8=h za<^C~sF_kzJ-~6txNuq;xW7;}DLPfIVyg(1k3wHlH)*+W9m5PC2Mv^z7Q)h)&-F&m z&Q`@qzjMSlHhdx7M%~FWIYn(aJMJfcXTdQv>DsPP*T z1TmSN<%h2~@{Y4LQiPW}e)ud&-%vpqq9I+lrLJSaTHZ}&Sg2Cqv(&NfeBd-M?X;%O z%VcTII}Dp-cz7giS~8=ko~-}M)N^|M!y_sWAO7uch63+FCy8vOD&=M4CCdTpEm7rxsib+7uSqZxKA}Huh_=_A9Nj*J(lhrwZn|juiJ^bd~Zk zKd#XB$VHCaW`iZsLTvlQ%B)#XYh@N>Vq3{et*dd+R)$M8j;2ybR~;eBw{sO?&AWwEEQ0GJ0)7osXKI3A2Dx2`4o{x12w)U+Szm$b`<7Uu^@k&N+ z6u0&_+zEu>)e}!Bmx2?IDv8l_YCF0Wx z{1{&x11jwI;+S=N@1EDF`Y7hG^oT-%sKFzKs~`3@T7 z_IoUJy;l3f-bm6TlgVk?nI3=y>hte^;Q__-`#BjCR-20V_uqHgeN?_Fm55NcI-}tf zhw;G_4HA7(U21>hi!$jIa3;60ope4EYDPP#n-AfOvg#v=~cp&9jh7 z$qZKlqOle%xK80aFG6s=(${nSdw3C9g+-25MiQwExC%$4v8LAI*L1J^b;K|Go8oJ( zkTg{Fje0i&v|9HrW!DtEMn?NkTU*5yvrfhJYCSJfv*!vb!g^^ORE4ZnUgQdz=~_p) z)9X(~nYe!axmZU9oLo^%e18l0X7w0chYKU%%|NAYcGE#BnkMG+G?~tn8mRCB*rUYT zTCp2>)Zn%AGyPhlD5Lql=0opK_`iu;(B8(12KngoN5-9qp4|D*fBsjc%D_h=-%_c> z^&FQnCGnP+@7Z85 z8H#l{bnbpR>QB?j=qtazaH;7~X5F<01X>_fYC-wX4{&tYQF#GQ)*=ERWY-%6mIo-Xr0g&)M_y5<<5* z0zrKt60;CWO;GSuJ8|Rvjf26#n^#fr`q~2!7r)sh;}6CDM$`endf1Bn)*5kHk2&DK zx(>u;T>xc5MZ@;^+l;B((o5am<2SEfZSTH#vE}USet7!g#je??x_6y_Xf>m}s#ItZ zQC7-BLza~XrB?;3DXPXFnsB>9b!-r5cU@SUiS&hJdYZrszt$)_f1x@evKvH6kR27( zF>hF=I{M9KQ>%`Pv|3a^>JUMDApq4l3`0eH(JfV*YO6UoDX$swkzJ^SUD6PRW7r;L zy~ncQ5-p#}pCpsj7bE zcAS?xyFY<%F3hZs-+4Nm&iYdliJTAp2}*IoOl@p7fVJkJ)kaSy+TD-oyQB1VIyxPs z&J%L+H`UR%Q?cj9TayV1^~sR@=Z16l4H?kczc}$J8|r7S-5xNX?>;>_6{t18UW_`& zPx>eCPkGpD)w#)huc_Q#SXa6EoO4v}Mig>-Rps_wa!xBc_q(H+^C)dQ0C0SN({ndN z1BoLs_U}#ShQFQvFtA+x{1J3+*6_pKZR4RA=-joJllQaAd)9g>-$V}Ft+gV-=9Kr$ zf@=9!xJ-S|sDWUL#Z}ZKy&vEmVPM>g0gpdT<^!wB!_)CSssp+g3=#4r?d8;-*JB0AO4@@)&(lfnr1HUTIbQyB49Z>QdZdAq0kW*y-#(D%Qr? zFR!BCx00v3Z_QL|9hg|ut27dH2{4785r%QQ<^1HaEk(=>;IA=@kjnfkX5u4b6BO!{3kd+xh>>x9N@ z0VI_IZK=Hr=YSP#>hEuSNkMyQ$&b~qQr2;3TWWqjW%X^BUqGwm*2nWmWafezYw%~z zm(q~G8IT7m-NN8V7+o8$C6l<&$cqnF@LGzhlkL?H`U(Ey~y$CtD07S$OpyJk`7byEm-B--GaoYRueIqUZ&#E&_`LC9T_sxxAB?zj;fc2Cn zEMA)8)f3b#okOnRp=LyELyg!c-UIo8CmU2f9#LGm*1d9NdrvIB3cbYI&m#`pwh7Ra z8l(|UMe6T~N^gw6*2?+WuKF$+o+X09L2wU}!i?!Ag6XpFj=^%49EcevU1=_p4fw>l z{p0qIx!^JM9EYj%DM^O>{HKg}ydY2^#=XL}{66b{gOh4_eE;j&H(+H5&g3w?e|mKK z&wls6lepDRDb!1?RTp_m>;&=XJV5Of1kusKY( zu}zia+lZH=M_u0VuP(|K+j=gAoP_JdWl+Zim*%s?axA3Z}{5@5J37jwS@(& ztTg?(GPM-W^F?SEazPB%e<@w0LIXMvu_;guQtBlh#{|IVZ-`F;8Cb0{vLfyQJC9=W zwsYxv;~8uSg1dGQ)cw(H0-@34r;j-G(!fslFnIRc`(L-V9)`a^yQaK~dDon8B+OO( zbk|y5h1K_b28FF%%rPuVJ@qERo#M5YYR3{#DUjxKv0=-4TDcXAJ6gXwIHqmxZ-dcPL<>~~Lo>9t;Lzj(ss^QB{1_3pheT4B^HE6#HAbP3K4MoD*n z<3~>WC%F?S0rX5{|9i17qYX3T;`?cD>AZTm0FFu@e*}g#wb*>trgn+japVX*cEMG8z z?q(yXF?n1xKA9PdYm&q~-d6)L&W#!F?CkD766Z;7i>hrXzXnCw`H#^$KW3;=6ZsV` z6EYP4C{{=Vj*5sFci=ao{f+qd?dj|O!{$v`z8AgQr7hpucBCv{YTdWuKzzV!%lBGX zphZ}gNVr8dCnhIF;;B8^_TWVDe9dBtin7D2mK?v}yLHPs{!zwgd zwN~b}@=A&`u@#4C;t{n{C}JlOvpR0J)?*oNHYC)x9Og@y5mmyaJPUeD3!tyL&(|Bx z!TKI0UA!n3p8V_yvcO^4vTSfI5~!9B&L^R+8ROf;?8G(!xM^$=^+t1WP+-Qej|$As z9$V3{Y-p|mcT)oj89P2BBEF7$S^%a&a%LW&a459~T(V|{7iP8%ttSwesF>X=O zm2(8waMsGeGv~X&aJxH{3m_m$nJd7+iSr0tnzPjzp9%KrZHhzW>5%9! zm9mHrpKA2@{{{l9(kwfKLqQokhg^|MNJXsC5O4nf_T9fZug4=Q8jWYei8E6sbR99# z9&Jz0PL8*`646-TTOq!%!nvaH?_a)rp>)CUb0P$O`*-l>3_w&2M;ZdSub~#{L+PPW z7kFOO4*^bN6b*p_V$5&e8dN!Hpv+cy+RW8o9^|PqUkPR8##9-k%)#MdpTo8rTaYma66~P>~hFtNSqVF3_z`OPhfsWY44u$#wGaWfF9r~le4Kf!F&UY03dCg4$~Rl zJfLrm%tFrL2?fP?a;7h4P;EQ<_O!?wx#v`0$hG=oyhanAuqqmXB4k2gW~XYl1hr7h z)eI#Qr|OhpG1p3G$v*LxqIQ1ZxD6p#_1prG;EBKMyK~FbV9h@QZrqC>K8j4QP43&N z}p()h$@b@>LUrf)s*X{#~E_|~~x`4{|IgmL zaJOwFecqqo{SJ7Pv-`~KIub#EAgTAvnaGxv`0m7ZE;E|d0ZCAz&5Oc|Z8S67#UcDWD+a#%p_YF~b#V;Sf{e^h* z4>?Kja{A`UVadMpaCko$ylWobm#ooCFi2fReH2Fae4t~noyD`Wc?|hn0>K$vC1{E^ za$QtA4ZCG3s7^UG{1&0PTF`hQAe&0M%WkbUT#a_quBCrap;qe--bHpTb!zARsn;TE z7}D_494434&WR8a5PnYL3YT$bUh4D_Ka)FnRc)Z3d-v+sHwX7m-}WXSr=KgelY<)D zkFs)BEYZ(7buXu%b3j*xb}onQTh`8rs&X+BcI>;IeZ*U>bQ5S%RJz%*15-U-qsa}$ zinexcMT`HDIj&htMm8aq2?4*`b%}IcF@v(wEzO{}vtMfm2Y5BbA2)+a?cC0w{pO!E zgCa5s!zUL3ykZ7rrCXXo?^?f14i4&fpNv0l29?^mok5>!25~y#%4MAK>;qldaMWq; z?oqd=+*B!4_HIu(i5j-<+`K*I;tLatu|$2nok1JUpzIvGJ>@J}?G|Y(gLxTwi5;9{ z?mFFgGih~+%!_~rrqR-IHMnt9|Cf7}OC4)-X$hd0itzfej?KE%fHq4w=+GxDq?jD4 z9I^RT7H@?i{u3?1giu;(B7|1i#ynMT`q;L}>f*{+rBcQqvjk%q45B911q^S^cB>U3 zw5O%1duF-HDnnO|2otNVaHJe0k$aPL*f^Ai7+ym=Mr1{)(_y?-u_ePq!|uhcDyM1D zrdg`8s>4X7YdYMjqsJgINu78;1kgeXu{P&&q1|1wmSgRP+e#i2Y_Vuhi^E&4v8uBK z)LLU{7n=kzvxprknp6ovXLYt*Xm^(2vP=Lnr4;*^6|C$Uv9?yf_$IFnO>&(Sa8ZOh zOybdIqFlMq=^9}TpaIm{CEGBn15>i9bEMc>YiW-lLSSMioLC)`yU>}k&M_$qg?8ua z*t;7cgI654tYBr=AYrT?H&Wt|si_Swfxs=84v6NEcdcA#ca5ol-aZP+tAphmkL=R? zE7w@r*{Fsi{>tqLNCFPGXKmaMY`$BP=Je&!{sy2ms#5$BdMK*$ygdxO_6 zi<9rpdM+0_ovRZx7OPFgg3d}-bqXA@ybr+62d={bI|dcK4u11(+8k4WHY)tYAgw2u9ObGwa&I zD{3=4p$|bkB09I#w{oG~wSX{%u)kPg#lGbwD>}#0YONte5nr634a9DS@FGcCS@c%# zDoX9{5lo`iCh{5U0-dkVzF9g?DyppNFt&^}Ckv!)-muSfuLThkvSGjJa-rQ_;`KoV zZxDpGIGTCMiq5e#TWjWF6S7@^fsuf>Y~;0uqrof}I-O%>)!D#XW$s&6vZ8YW0@iKU zLhxsAVd$FB*uZ-B&?)-ATWWWYIO4<-4FGT;GPV=~ixmg4qRNU66OX>WZs8E4Q=IWb zUq>=^2xje4q21j8q|oBDZL`+gMV*(d=$ypCUo#Jh3;_CK`7#uQJ`uv0vv5kKcK4v8 z335X!Aq*-TLYZn-uCcPaywKHWwVW$;(x!*lzabbFQVNs`?aqP|+fj?)tHq(r3RZWm zz2-Fol_gNU91%b(K!gp)%7u2pZsAs+7 za-rQdm_2??1fOEpvVxUe^P6j<=|;kD1gth{KvLmfT76?nE|xl-qhG`;g%Qk5mu?C< z$sIwzTxDg4!E&o-V8qUgNI+l12tkP{W%ajQXm>YkMo1I<)?z2Kf>m7uo{?*9HIPNl zlsH1XVo((FwY!!}rFQ3pSKEN1z}CvBGg-=^i7TqC>M*DAnt4e-iEq;)=u$)SkRxb0 za&@`X?lA6wGM??QbPVPttGgGiJ86iT!a?3@LQTh^j7ZP!c$P}-?y)H{plx-+w<5r1 z`L$P6S=C|Q>6*DoR3TQl$f>ON8afgiqLmBn?gG$IsA9>RP$b=~@HoIKfBC zyJK3gs%woVnXFf@Xm)Fq6hsP#hl6Z@IBoTf^;j&mJBOmG5pWKP5aDi$dC98o@!U0P zCehwW;P9}48Y1As9=UR%-ML1CMPCyxs@t{n`pS;g8f#7!UJAWOn1=+^LV4pz1LZ=y zW1Nw=^Q(L&ZqKrURb2x#uai{7aSI6rCyx+B!pUW?1m*PQmfD@;mjp<8#M%jw4qMf_ z#8Q=29d03jte%4n@w`E<7Hm9Ey18b9mY99KEs!Ug zV9pUo4^l3)yT+=L$o*yIaaORZYXUS^j~YRXNreRBB6X2CZxp2EVxiMDwguc#kp7MO zq$En=L0_n`skYFJIi*?27n5-xRA4gRb3N-d-XV? za%g~PP)J9>esnkDYm@7`%B4={IJjFVz}p;&C~(n~mN)8hmDL?y>zXA#PYt#m{K654 zTffm9dTWN7Dvw~G!10AeAP0$yHq8cY2Bfv==srgtkx`k4^b4*4c3^t6? zX_A+$>K@9rb=^aUgYGv%n+CE5C=}3OsnqEnhZTfq5IHsvBwBSAxf?iIpkkF(9cE6i zotsQ8T>9`<<9vu?dh}N-HNXS=Bx45$jGG`qTmgB8=jMCyF}< zJd{iA?g_F$q(Z>vLD<{WwqG32Qk7L5M!;J)p1?0HJQHpu4e*);j4Kt|-9`T(x?jfM z?PGd7zt!x0@AJR(2Sax@I)d(qYf9Ns={AB{6+iZyNP) zvDE1>NRB@e!XA%dB-siN<#Lsk9d54kP!K-TQI?*1Ji8!IRfP7FBShCh|B2X6!?arabts~2ncgL(?Ro6(OwZ_^P3HgSD zgGYqm9n3sd*UE)<*Lb}cgvBCU9=Eh)Rp$hSte%Gf0uG`a5EcA@DB@_B%7u34ko3+=9vM;H;Xw3yB172T3Ed<}Hq{1T9j zc*J|+e~yK(TxfSIGd>kZEGt;iHB>U|Wj+*i7`IWAi_!?gxV=``EsCXf=Y$ahIDxQ( zB12R^-0QCBa*Y+;#c&s`K{}8hptTT#0}@&d{fu+cl?(08vPrSX*s_X!%nDX?jd@;M z(>2~JDkYdRWTQkS!O3FHy>g-5HPkiGgitNlB*C7T?u&ga*I3nA?7?efX&7OGPD06- zfsn}`tFz_8>+CGD_?jIOHj$;^VT;T?~s4WY-lb7;CjAQrlBB+IgoZ*wCnD+OUpKv|uG&=@WN?QJ~=zn+&^M7V`~J#T2n2*$e5Y{IWoqQpeCrr zk@_zAx!ou^3W?I5v}Mp@!I?MgQw>T{t%H$?); zP~(yGBqG3Zn6%U^h}7I^wFyKVCVh3Op$k-}_0($k>Ncj3sN>0V-LJLed#LKP`trS2 zCl4QgcZhEs)cQnB4r+;-Zjq`Rt6CkRC%bi|v>7*5H)O&FwH~pNd5*fPOINwlmQT{3 z@wu%1P_^7r&uyyD2JK+x)V*$Hzus=3X|D(M@P>Mnj7_UwSKlMm9etf2xj_9ZrEJ}- zD@@Pw*;Ee?dyFElTa7@hmSaafy|4Sw|Bn1hL((jr@VC??cP*doK;0IqpV#7<9IL7c z{gaxis{9y`l^$wd2YOAuw>myg?)Mwi(VuBrs8E7mZ8ZcDc7RdV#68|Dk*IUOcl&#y~Z=o*osqt874&{5;4b#%aRr{Ry6#2;Ey<3wfM zCVg9VJwln_Df;y#L;`>#S71Ab>VcLKP;tE1>_7b>cy%6bJtUA&{;&A74Pyf3Cx*lX zLOLeofQsjI8x8szqCsEMtB#mhbF!okleY=U^Ei*+u@-ea`JLC2fp1yXNc6OrqE@)h zVIafU13_IBAxhU^p@(uIk23Q54$00jYG4E}Nh4Xo3RsHhzRoK}Y=kv>MbsT}U~rWn zv9DzA?q4ZPQ% zUV7if!*}t;Z2Wq(&*583LV!Rmk%OI2j@MceaNzQtIfD} z>*)G5)q-xb1`bH;>oy5&FIQB{HPu3>>Pt=u+97Wz85-@dL9NNJ&wsgpad`0)zhP6v zJ~%&{jFY7yEMF3-lew>=Z4RERsw#`lrhc_JT_xqA5;^t7Wxf&m7qU)3ryH#d}1p8 zLX^{f(y7%>C~{CdaEw{g>o(=nd>6=_es4-+MRcF1bq?xJrw5B?hv6$7E{?|z&LxAh z&hna0TQ!yHYWkRotgeK53j2D7>lv$Nc&O&Ho~41Fd+Lgakjy~!sq5$~dPRh>_w?RN z{aw#$vo7?#YE}oe?LADgm~IupqWCMM^MsvR#IKd`7287@y`p*m(>goX%d-VR=Eh#V z;nhfJ;_uh$8F;E=2d`Iu+J@k@Vo%jM zO+*RoOH|UEiDIXT%0ZuqE;gPoh&1tsyS`myQfuu8xp3-g^H8secOaf>!rLn%6Sq|M zIgLzY`@K0(Bn^rooj*ylkrRqhGU?sIlp!zO%0%uUh6q=8E%>oebkrVxf}J*n9oj8;-pB-0RJ~*^pudL3FQNrJq>PEo;C1LcWdBLS7H^DXEu!Ilj+I);v6*x5!S~N zK^=|CJn5gMnu|M#FjkdW%;f{w(VOtaM67$30~7Y=$cIJ;r8Nh4dKr-JcUKIkzo zNFW5~q?f$iPbY6ZejZ`6!)Tv0X8&94bsm^eH0xIlldU7f-zG0h6OY}ng< z5ImJpYoeuk@$0i6X5;qZH5~L;e%hu7eRCCanJ}dF7#8cNO;*?B#kON3BZMNFN(OBxF2yj)SIb>?JH12$rj4n$(@O^>T{FGx z;M=l{aM)CQASdvM(#5)^TiKl&!7B;9{q394$L&s~cD}6L2|&yCP?;?8Q|wMC*jDaN z>G_!>`b-MXF#h>uHXC$?7bml1JR@7e+uRbm%49P?lZ|ytXMM#{&!tUH`af2PX=Ra4 z@q@}R%+@HBOP%#aGKfG$p1d}EfgeD0_4ErBxF7>4nK3-Kgb=hv@vHVI~sCT8BOMv`%dtV{M z&Ss)z|E!qS;PQjsJqkq!kMFuD=b^=@OICr9d8s7a0IDvjl-@0*0n?z6ebeu{4aGQ!8gD!T7A^pMC-tcwr`RrOseZI-&aM-hhuK{!Jrfdw>V~c>vh4olr z1k{(dO2uhNB>sYpj(`FsBVCn^4lXd(I1RUw%u-OEFV5!oA8p+tc=?CjOnY6X5(H7MCFa!j z0~|;EY@s{E96S>vB3>v5QhY!QSgTOVddZ17x9AEQ>@B(iX$raaTCl8Or4swQ9S?th zc7Jm4K@of=i)mK~GIXI9bm59mlm6e1cXBmpZ!(xerBQ~grEKrRiT7acJ(g;^p(QkL zecU-TCbj;QqEEwu%0eypl`CW_y`@FV?sTw>zT07W?9+$h~s1Sl7+&AuR^3|zkPS0MI3c^D!#~lTFeN9d7a;_qz-q3p=a#-=ddiTMf%G&vU0~FwY!kv8k&?&(P=eEt;9SO=1ko&eW(r@Wh^2+y z!(|srFB0z|Iny~<>;8UF-w#{a39FbuS0%g|%FXpL-pB+^NH9I9mlL$ErMHA$)$ z$WM^zapQqt_w+%p{^I_1FzK)Sw2cXN>A9r|^b&K5NVOHbIFs(vnZu;*#^O!{CrvJ*LOo3;X@%~D88a9I5==1@ei#e- z&<%7a_~Vm!J<@3w(X(8cg)1kMk$2S0 z8oPbHy7*5#ljTGj%-H@f)Shj7`!xN3ZN`J*Hj9z8Hn}i>)Tn_ns;&yc9d>XKdV!Z8^_~sqyb+w+T@;|FCoCB z8+_Iz`cllk%9QiZG9@Y8BvMqmwMsuGhJMG@i3 zYA8pY@bTqB1~c2G?Zw+onyjjxAN$_E@|~Q`W!%v{tGrwV-DVmuBCwJRvJF)Ntq_@D(0Gl{2alfpEOOX(?^=dj}ir(hX!@VoI)Arqd+Rw`Ilyes;j@r!?cV$q^y{mPH=rqf3 z%W;@<8`;&lZ^eM*AkDpxAfYf_mp?CuaE1FK1fCUxlH)t~K9ZlY!@19^Mt_C-7?(gh zPk}Qo<+;5D@#~EQ16~_y&;buuGVk@EGZ+(Uud(mJ zOH?-3o{E^)lhC!bp0f6)-^C=_QzYYF0KVh9uqJxS3d?sSs9H~%giR%hW}8;h-@!p* zJr~P#KdDhH8I;U-;otUDJicBiX4js%U0LW&yMb!aR8kK&--VEzK3iR{w^VnOF~6tG z`=+<1)sF71(5lVjOwY~T>bj{UdP(z>nj$*ZdNnl^Dt25t2xjcmxKuP{eua9TzAxzS z?(WiB8r`?Xlg;-ov5Lq3ZurJKnhc3pNhFlKVpl{`zqZ?cc->VbV3m79dDWpp`?r_+W2&=xhjbrVer7z{W2sW@(7TQRT2s}vEGq) z8q<5WtjdL~DkBLUc!%DB_mlU+`+?ynVHQQMAdAQQ?#bjV9uMOo)SqE8pJUGdxA)x(gq4f<jeeqE| zou3T~7pL*-!Fcc{9?#D_xpOc&Cvs2xR$BbG_iT(II`N)QCnKWvOqqr9(0_aP-zVLL zIPH2*l2M0(gSq$J!J`*Pza049|CFzf)Yp7Ux5a9|RnZNbaY=ZC9B?V=4P;EkB1eM! zyDi?cvkOa5pcl!xS8I5FV?S)~*DboSV$00CCzF1EmSk2~+2&6^NVFqkRmA{%l8j>_ zI_28C>*Gk=ig4E|!VwlYAn6h65i-9J*9#{0FTee=sD)?2HHdIme%gj2TzN|q5w1-d zC=zW}k{51-2zMotabv#`?FT_dfpSR3bvWPTiXodY(cD~0qH>#aN@c6lAnO)YDe&|K zpehZfN>tt#58ut6JnsJb?0a7NsmoTb-hC5<;dlPU|BF?Fwx z<4Z84s6mn+35ecm<1WOpoZ&2#WW`bj*#NDFOsrbc6)~lj$o#7HeeZ?(m+&%)>OEC| z3FPJ<@^%~rbSC;xl7q0-3h14mq_KYELbo{-VW1`%PUPY9>4o}tUxWkc4ES40&*u_R zFhfE~jUEB?F^}~6-QeHqp04Tg{7l{37^+*9p-MX6RhN^%JlC|JHQAHIB&qDs@BdrM zO^ZB5`l6bXOfk}$cdkdq{98ntv_E3g%D(r!e;B% zZmv3}-4mN5&FPy}$nSJ;^{5jf5BwvMIa9W6N&zlVg-cF} zX+4K3e0X^O{*g_$7F3fFi7!bN-iFzjE6z7X879pvMtd3ef`#OU8%|a;hHs&AzMY%n>82|nt<#85(>lgN&sXP}KGmB1 zLZ$e%&rkE?1rqg?|CRy`T0g9g$_l{H+f|-T+1V$q94S9fZCZIOb7FNuQ(Q|Yq>h5U zOXIoz2eZIRQy_<3f!!#lXYA%e5~rNCeOX3rbcB!1sf`8M)*8B-Ky4fiZv?f`mNzyQ zVRSIIl2o)-X0i;#6_EqCVBg{;th8^D=lo;og-z04)q~cuSURRux}jAs-wg-D$=moM zCQ<}j2^>NBRQpaC2!JN0w|s^wunB+Vmz2{a#p=F5gp^RZf`$sDlQ&vd)y1-%h58QV zVQQLH?w4uf3iPMX78_BhWEInDAhJsO1l5hOdq+2}N|Y5V?Vx^~>XUwiE)7J)q#ybQ z({uGC^DD7pQ>=ftGLA8t>K5G-T`T?JfBmbq2{`+wU%%6oxR9TVs>AqDRjM;|NUIv5 z+sOkTk%WQ#$1uq&>uS}9ie6Z_fswH9$aEn%t`W6_?7os^ttjNS5PFWl}EW?u?|zBG6M;PieIvR30oEq@LS>WH^Xt zvspaoVcwwh(n=D16O-8;FPWZBymQP(Gj86l99ol8v%uSfxByYlW$snxgCNKc4O6zO_78EG4V4_h^~HN-n}er!fYN=xNn&|ppSccP*i$Ajrv68HAK zJ{idia5^~bUcBHT zn}U_EHNhsVu8OYI>3sq3X#8Z9-HNuiwBo)yHUlpflo$d8;R1WUmZQZlg)LV(;@tMcOJ8HD< zA4rhKHM)68IaaGx)N^$6rCLf%OT&1SN|7Kc!gy>&H{T*klVrg5k%0U*A&K_R>Mv_| zUp_k7Iw<16yz0|7)Xitxoz}>cakv=>U#|qVZblG~D-vO_x}i`=O>Pbm#%~9NG{fZs z`4bWx&hGad@qL0eHW5`dx3kx9oZb%2Eo9ig3vP$+1nwX&KODf8=ju;ct@Aa|{!-AA_ zJBJ}Vy?ERurj9old&E`sMw1Cl*V8lcQ`9=|Jo=uMXM^+eLE`;i@881kAk-1YdO(I4_UkGRS^f_IiTVdMXJ>=)eBzO~ zT__*?FrZ1VD|$Y@qps<({p8>KqUNJO#vp0e;u+lJhw; zjNAc^m6ld!6L!FLqn*-W{8gZxqei=JhZ##51_dTUX>ihd+ebU*s7~S6W-go!VoHW+ zlVQg;;Ksss+UtLRc-sGYHo6A5bLFRP2zM&>c>z9>m$DJ6jrE2CpN&GER9J3NvKLLa zFqbZ2y9JUcL%>oyvdQ5|3ei6}ceTwz!6@ttf;yvsO*KD|B@~T7u)LIQJXaZ!F;YeE z$r7>xTF}D$xt(2^qVRThkyf%i zwV}Ec_aF}Op1q~YqKYrZaRBGqu?@gEAaxT4#UBli~IeSnc z2y?jNWRuyWO1Ct7em)?lY`y>BV`opHov&&3TzXPmdYWDSWGl@c%&g@_} zje^C6lFryvdv%>FpoAAQX>_Typ(I%cFK#-+OEPA5NK~ZaTcM9DwevOM#Y@A$ON%9! zmNv5e_;cRlD`)ss<>}35ctH-{c!n3+`I=_WL*RPj=-9aQW&au(?$%sIgJAR`bG+JSs$Z3uC$2;Uf`ils&&ntCwxF)%M zS_-PNINr1QWcVRo^m~KunM!b8tC`dy-kBsgjCsoOU#iwm8(qEZtj`nqXBwWbm)rgc;dzZF?J_lXj}=j8v$L;HNK zw6cJlZa7Ltfvw`tFXl?9&O}0^7U<){MKTzVE;^Im1%(!GYd<4A_4))(qEPLrI&6>2 z3|B8-PZRY@L!kz3mG2}>xdPjXn31dpm+r)cGTn*rar|nfN63T96F|2u32+>8_e-)g z9-KpW!U-+fuSHG-LZPHymE3J;FxSWKCU<4f7WcT6X|oN!Q*^ya8rL+xHb>*p(Y zr0qYC?z9bZH_nnIO2K=!LB5qbVGob@!tdw*`2OPQZsA4|j|w^a61?Y^ZBRPl?uJ;>qz4xXP8{gu%mo%fO~ z+^gNz^>JoyLgpkY>`h|s65oA;ZB5H+ z;A#r+=mB-GtwO;mzV_&RQ5F>+yo2WeV-g|ZL`M1qC-yZ!i?|wTjb6IpozikQ$LlbR}N8hP{Q$UubS2)2# zoS)qYZcO1+|8l^Ki57Nc_~vkFq9T`OI}?Lrcd2qRs(rk2vQ2bS7=m^L1NO)?8EIN7 zr@x67c5so*&SG-E^}QJABRfk~c4XpT`H_%|Hq-}hJJk1TK|=rJI+T*IS#JkTJ^}CJ zlWB53B<=GAnZPhvECW4ROY9pM3Hy#r(4ZF5fZuA6BOq!uL!Y6ks+qN1s$OW>#oGD% zpqed^AesgdxLs1SaLnO9X2~PvLhd#z`rvF7_xPv#oCopfWYC`vD6sJS^Y{&6BgmVg zWCZ1TDRI4#7xZ`3%tWCTXZ|9No+aS4`R>~R30!8(r1w1yr(3PCOQf)tRx{9t!TEeJ z8KaixS!Z(`PXkQbF40UX(K3fT6BM}_`u0)9CsPI_i2RN!%Hdw$LL zCIkym5sfsoDxGN7NmOw>^6GMEbxpCo&LxDw3TU;BK&+w*th-?%a%jb&tV*RD23mDL z#M6`W_;fKK#sd5I!S~;P|HBVIlJw;0tf+Ej$)&@^`9j74`sl%ZfhvtuE-1 zN+mrX8X7^%`n~4%-vCTo1U)YnfFAxYImFu%?ILVsfq8sMYjaie?{!DJIJ|qmegBY- zZPWT2T(9+aP4jE5zkd7PkxJN*{^m4@#m%<WO@V1mxX(>N8 zMN&kwpP?zhPA9ReywPAb<9O?Is8WrC!aMcGN8q4O$zIdHJRli`-yj%wqb3X~8>&>H zAaqdn4QgIgpdf;owp&4+xU*raUdKq%!u)2p?ZTjn>V=kFuwuU!D%v2ZU6bEM@OgsJ zlSa7hfT40Bcbl~_i&gfQdN2N80AW1PpTws}l2F z#Zpz##cFkjZg(5=9i?WHL%Xj*$%N9J{0^e$4gfpi}lTV}u!*7+5mwjl^AZ#_!MRBtz%DAU*$3ZUm!$+R6TR5&WkW}k1ULec5gC09*1 zQu#m0|0w@RnwkTpcKkqgeC;V;b*}YNlS}#3llyGo6kLOKkB|r}RZ!Zx*GouOPl|NF zqOw6{d^2L}i9VNXEj2#Ew(rCvEtm2zb9(RW6Sut87LrQ4f8xrZUZBZz9u=Wu-DkMn5+AJ(`?wde$P9%O@h;i2OrQeq4mNR zhN)Vs8Tq^;0j@PxQrUP36s;OoX@bqK(X-nWt6!N;d-Y@W9V!(dDL?gk(2eL3(M{>T=tdMj&^$@y zv6B0~`~9Rd@Fke~R&{@MfsF-{Ur^3JWXL&1j-Lp8MJU<^T@;Y0Q2ZqOkROQSN7&kL zbjzDr^}=eSN8WLc?n zL&G>pVbp_SGa2{d5owAR)BRFbq9;H9^z-O&_+wiXFE;^=mAy#{@T*)Q1LI4JG$t<> zJ4i+sJ5b7(*Abu|41C1#W~9ud**q+lGR9_Z`;i)O6g|oK#9aF=Y`=|@?;H1ie)aD8 z>mR9{<0bR(C0I$9ZWU03WmnP$LX2xlWZjolTVcD+6fMFD46hYv;ze@q`7N(r+b3{- zt(DsmMc}_h6ZygU1?aWn&TJQ-^wFuNAI?zQzPUKwE1ad;mWAf2czryXj^g3&-{)lne87 z=uqPW??3|t3Y;?!<&BLWD9(>0vUn;-9IGeod#xvq3oT`*IEFvk4tf9C1*Naiyt!*B#`>Ww6QVc#9h$!01*;w+w>&Er$y zy9ncEn#MI&(w4=&5iS4fxW|Bts9h(}43q>ydX&M1UCKsoi}Lk=_TyfUZcWDV@I(hj z8;tuC=_8_VG9P@1J45U`>IMGiazT<2pik!SPTq2iOiolv17!-uTg&Mx*;*FIdq7&g z3|`F_uij06?L2*UKI{K>_2BGVd!{;(tz@<t!%wC$dlA&B5 zIPt%|_o4GFe9@WLc6r#}IT-$ikxR1lWL<>r_KCX~)!wra9Phmj??z|uC;jj5zIf31 z@x{Gj7Yl4fZduM&B;zZK?K$=# zt3<2Fm8r_yxB0nKuiXi6;Z+IJ6?cJKcr}>I+T1?8Vh>b74UGCSOK(xYljUlLW2Fh) zMct%nGqVbnMZv5=Q^zz{u6U*77Pkvg>uGJ2@lmqXOaZtr+n*We(u~;X_6x+W>`ndwkbR0&j>IS;MWv1n*n1b@e6J4 zZ8kd{kl@_FNZ5DeNWpL8n28qLZ$}N{!6+-+a@X}z?Lxb*rOka$RF%i(9CYz`ZAet$ z7Qc-6HsX+?$ZYI*(e!ajC+$2_BqkK<%T_rYhq~&Bphcy z?k0psf1?j_Zr?Tc*G0eJI&0k>CbL;WSOn7YO6gY3Z-fvi(OsMElIe|AQs$>XmF0So zp8`n>&n?F~E@n4HP=W5+7J`aJcikwu&viSx-?CkY3GvdBlcKdTI-!>~{B*+}yp6?SNhyd*XNa}!<2773j^r-enW~g_ z6fylW_-7T!nXmRjkD`wzy7Hsyjk+(UocxH({FSm&Ia81&F&+ugb^7M?%_tt@_!~$X zm!Q_cIU^tpvwU{dXN+f3Q0o&Fo>F`|((eg=A!(f+L94mmJsy8^gkt^sXP$if{+an) zGS9nhkSZCKQX{VWMi+C(uVO90Twnlj}=W{g>5 z@?v`U?Jz;q>*=V^o<#0(9*N_^0At|E#bP97IYxp$hK3hmHT%2{aj>w#Bw3}+)>e(BqPDQxiGm}-< zA~i4)=smz)N`0!;BGJ>MZd287s5+6lE+8)(NL9bD@6vVD%-2s5Co$So*Gc%aWl$JO zD1(9z6n3!^AFB!!;t&^9u(8@is4AdBiTU^>o_h6&00jGu#(q7|TycvEzdmpLunB2} z*q}%i{10!aEpJ;p(~}-b}@_vjVa49jKNdWCDC?N2nfc!*OP27pce%Q$Z8{SSdMY7w!(lK zQPfvp+W-pnJyyLm=t^Db>MIHw8;GHr3=n;8)CIza{f2_)HjvTnw=_uEzRs%bQAI$c zB08y5S)rXQYpjs|Dxisq%-Hh$I&3ly{hy<2z$O5pt37Q)Y=VDwafehIEBq;fdLX3If@tVnw{mz& zx!VoENUG#Xui=BTl)e2&icPe_gW=0{NRnKl4H+%3^ZhfYLfw~mer|nhL7D?Ne=YiX#?Wn~$is_p@uiT_VR3e{~u8OI! zSoJFW?&1`x5D2X4cRRsK8LGeD36^fwS7ps!f|2k#{X>{hJWZ}&xT_}qaua^woPZl0m`tgTk>Yby} zpLmHkoxGoTZ{&}Z%?EE2?{`%B-HE6>r`~+gn^0#q7@x)7`R^WK%V}j46D?vk>CgEF z9a8<_>v)2~u!eoF7{27g^GWdF)%)m~vJ-D;tKT*qQj^Lm?eS?8^~%C!CiiPI*0cFH zqb62*h?8h#n@ag<5j;wz%zzAIrFqds5uAO;9-V#15}kcVO!F{W{ z{fJJa1qtf=^t&j}E5A1`bAtrsH+T1H_T%hU6HnwVk3}Gh>0IVLyw7;%ohDQH2h=Xs z$~5k(CFZ^D&b+_$yML86IE}}%VLYEqQ@Z{br+NywWuA2BNiMZ8N`wBux}an3ke@77 zLlIh`WT9_y;1(jLNL>9=h9Z(tp&lsGHYGfHK?|Om@FYJ`C{aquNd8e;@LSsEWJ)+4 zUgB^9BYEdu?eM;nPbrOa0&Q%V)jFiokwnr+iIlRNXQd>=<&RZ*t+dS+`7VN}=@Zk| z7oX+E;@LMCV`BE8T?cuyDW9bqg=P zzhJigX%Io7k(l#?&?6ItHHccpo|+867w1guR0XT1J)7t$Uiy+5C=8LXZQ*6JKq~)1 z7Wy=aD*q_?y=#TDgN81rL>?UPy-PZdKv4nv(vj23YbVBAsR51D7;ejE0UN^)uv#$A z3d}6E6EK9ALDH+(u%`|Kwy$f=4SAp)Pe(%-l6X{?WrfI`1fte#+0?Bo+&w<(x(wk187|@ z=XZK>2HWE7pn-u6ridIrm*tsg0KHY1Sq7PY9qj0a<*OuGsGLqkj42dS$1zAUGCiHV z1zvi11=Zn+1sTHhKg)Q0x1K@Q>KTMb9>MR0%SE=LMgW&-@2A6dEjfC99URb0KkX}` zL9m(&A?(chwnNx)X%pa8Gy~bYyxFJKCGX(PG#+Vj934Zi*N10@Xs*9Y$`8>_=SPWk z2ILo|mhHk*E5>-)0%(z7I;)g?QSi}9K2S?Yh*Wh@SE3eOD&5e;*zfK&@4b5TW3B)F z)3f0}ethbE=huST$!P3-H#&LWok_+OSdYpb{#cOquo)QK)Heo0_PibM86Zn zt-lBzQXN>0jvR@#P6!iFN~Bh-To*dJJ?E1?r3lo)?nHKNNI0BL{h&Xkla2yFXiS=r zlN)51B=w@qs~rDSsIIBy+@NM+eun`>#392ngS$7Yrq*y2qZNhARk=hwyDz7_7YKVxlivePGiDD%MqT# zmmQ{A5Hi79(Azxe(R3+eHi}xXP{Tz0iOkCH2l_ixu-9#EomGnaW=W++wz4AH0sd5| z)Bw^IIyK_Van(gUP~s&AC2V*Cg`-w$gT8PFnBtRN%3u8ciRo zZJlIgq*5WG-*9mL_C*{$y#@u|m7lhu0x#dTl-`c060KV6DRz6wGSuu~_ z#ND%ja-LAoprc$XVg)RE>JSkRM_en!V^JVn`An1RMpaNQl)G$5r>Rfw$z#9{RZXuj zL&lv&T^(CZByjvKl;1;B;TGeT>0`kh%Q2>aWk`fjIq~JFPkTc$0CUml1^FlqZP)FF zhFVrH5cdh+p7Q~iLVpY zU|eUzt8tcu-tp5?>4t{1;+B4L&>^qro3pr+kbMdGi6KHhO#(%eM-V_l0my+*(Z85t zokq~46b$v<0&b&F-Jlb3hNeI_+t#%N?~;jA{lla>WFm_evDAurpuW+rTQ?aE?S}P( z8=`HB>gplphv?gtg2`k6HMi?aIy(*e}C>k%*=Rd=`}SMflrC;Ih->g9!^j7#6GYAMj& zDUU*$LbBYrNu#PB5t#R-{-M-%<1C9A$GHC1Q>5VESdvVT7K@Sz7@#bf;KozojbEujQmZKh+Vf}a;u5GR!wZP<@;gK_FJ%t8X?Bkj z>J1EBO4*enU?5ttl<_hUO*ZRWKz0kr)S0b8=A{8yB}KG;;g5d#_1?R>XxKjwG*^5F8UOj=+2z@t8UuT86 z?2;ROvY3(nVRw`q@5NetLwfuB{BIHfrQHZ2lF^(?`8>LJ|3!|k+5+dcYSLzJE2F8l z$fB3AY9j(+6EE7eNm*N`(Gs3gDc&PV$|@Uflx?`aNEn}+qW2b0T`!&zh*QVnj2%gG zz@d8$UtPQ(b-dI9a1*n0ar_rK>Rb4~tfYLUwo{pk4gq%(Ohlqusj z{(LgLI8{CBPGBIzSD#C6uz&vZpMk%NZ5dYFouIZ$-tBg?{jWRrRezVzI(5lj^3R~Y zt0r&j4tG^u&06@E_?ID#PY0cadeR?4vXtaN>dIuw!;J!BBa-SZ*c7yV3pO_an+%-N zEE#36(mSQvKZRS38doc8g|}yPoJ1$I=3UFKtC6l&TRGZct$Nf=X)UJn(zKFO->oYd zHLR*DmU14hn>7VmIjSG76~;yffdZRzJhsZAL&1vfZHmA*iG5;a5YW z##%R<>e%%Fsn+#VjZRSwwHY50bv+SfR1Tl?a(rMyCcbMOmDAHJ*4BeF}=Ns6M?3MDZTi0(%a+dpA|oXNx=Xu3HMWEojJEKH(EMoUkjrnygGI|cysI&5UJEBVY2xyA+ICfSLSjZP0s|oR;4=p zbyr!ib^R{&39&6A@}>%sZTrxgNGg>!sSNWiXU{FQN>$x!Y*`x`timSz7hWm*UG_$^ zYmBIw`p1iQBGiRe)9-#K!>!|$cN9S~e|gP%mr7wj;f!dJei^nwvm=LRnA9)}NjhxJ zW-C+W-YljQ2%3aI<*AlxMTinUjYge1_dgpLxK21MuXr4}vwS4b>K zpFU}BU6j<+ENZHGs3KANa%2*qmghA(jT%3t9SL2u-O`~lrBb6N7FxLi6BI&+e>|q$ zM#6d6NW}9&hlEcfP@2%#YS47_OC5(&hrQOptTi{Iq3Ym%OM77=N577bse4Pd2t5j_ zw>mhyZd%Kx^@{Y%X>o6(Q&T@pFKjn{_b#BI-UD!; z38e`LwgNtCohD;nOKJi#LIqUR_kKU}B@$<$E|q9wR|YPfdToZlyk6!diG@L96*rsw4dTcvYpM*{|`#KTWq-TTIVsrs>8$JIMzwJDXFMq(u!)U7RIoYE4us z6?F!q_>?p=LIPI07L`2aGW5~*@m}Y2_jDjcs#WctsJIiRbsgew3(e05LsT$kK+?xM zK~UQ*|K9z(R_9V$Xn{Uj8l1GALmw^GQe^t21I)2lk+!GM-O8a~8wHg#)lE{S5&5x6 z?twNN)84CZUncX{=hxw#bLFRPsKd-oeNkPud_44Yo~@E4*o|b@*i-wMmK%oE2ZMDTK=gYJp9w(>Y?N9u<>Vj7F;rLys8Q?!4Ydu zA(TMkmtYnhW975Ll`Mh(dISj{(QdTrZBG5VaKx5N1KRu8cW&T$_8mqBAU|e}dcf!i z*Jf0s#Q84x={VBDPDS-X%dQ2q@Or5$?WJDHzTd8e(1i(>iorDqTW%VIaw+#`@&z*l zcmp`qm|lICnIl;dspt0N=;7JB_;)Vnf`Z}VWv8e<=OoEsJc@A}A!aV=)jkj~U@*hl zFe4exP_nQ+{Ov{b;>F=lPk%W2$KR((|KxA)h9|SLcv{YkGxK+6%vj)+!gPu1X;(^dYS(W^zd@Sr` z?8_dD2Y(6rqV_JB(=$(TViDE$`Jexh^h9%*UmxRELw4rC{#*1fRIouZ2SfEnK}R#F5iOEHK4Sp_%0vt*#X_G zHPyb{Rp*y>kk%WlIafphtLT%J7$l=lmP_TnfL z{MZyMchBPCAn!aF!0csEdClQ)u>D5r2& zJ7)=DE6KM_h!l=LqtHoAtUlgOukc-#OI6Sw!e%!=1~h+Kt+|2C$PV~!lF*1OSuO}J zczvX>*}g%|v?>>LzYbz0`)Lhl@Q_5`xCK<;>(nE!RJ+iw3s&9NOjQRnqg({CO9&sH zOYFjy2m{$-DVK7;S(3t191W)4i+I|f#vi=DJbv)x!AomG_+>5oqsibd+1j5bXS9=p z=bgtF@o;#)nBMW8$8P|~Dq(CvB_&vTFsAI3s1!P5HxVv(XSy4-=Q0=%hJ?86Er!JK zi_h|T+HEJ^zyIxlTlhR8R%ASkJ8|#7bqyeOG0cV3doe=U3an-1%MYA~|_<7WW3P;|tGSW8QbASCeb?QohnhM5jbU1ga~7LShCM z6{3~;(5NPNmocHFV{s^!l2%+OmQVxg_5Gl^j|ZQ_Y5(lBp|_auS}~zSN+ARpK4?fw zz(agi-b7C?er>#Xbq$#C%1_%A6PEWvMY5C#oIyi$>jm(4iwQ5mRuNi6A@@-c6P7kf zbN(EhjpCknKrr7CgReLdrGwEr%dk)t&RDt2#(g^PlY4ktn274;$+O*s*a~5!?Z&%< z-RUkmhtw4@6fV8?;%PL)s2EgVbE)9xnMTnYfr)DZkd>P;o0C9$XNTnS=TTVFErp5YO@ySBmZ>WeavW zpBZ*_rE5H&Q7Q=wQ+@It3RFFtSf=kbzR-q#h(1P$Wjv1?(FD;HI0-B8RbZ%p<_^`M*xPH7`2p&C${e-sFvYqb>bIIohSR5dYkq)OSLw5Ph8`GQuE#<< z%d7B*hB;?iH$PVO!O$8t<50J#Yo&hPsP$w-)%ciB@WdT`RX+8|>!FMFGj$7m?&x8) zexRoF>3hrzL}ic@NV4lKr4` zQVC^<>)i3NpWnzsO*91E8?|2ba?#t4yYfcVP!;j;)pg!(}8l=t!bi1gB-M(Xydo|SyvTpwD01rwmYRw+_cDD z=zZGFT0q8SpFnkxo%Vhm)qi_G3F2#zY@(|^ZBxmnyq_z{hOqmMP$f0EnUYP$UPf;g z&lhm7|8SRgnpuO`(s<(&_c+X#x5_WAaeT8y97z&$aG*d-!D#B>_)@9tb}9S#7R0JE z&xXz#a^8YigVF`Wf-EYsGbOKHzg|qAJZ*@0Td6>o>0kvo=lv}wn&{)q#+?~vO%YvX;WD!L^#NKR zlOFEnbyx=QkjdQSxDb>}8S>2XWG~_rCyc2S0PXkZ8kh zHGdj?aA9;oN~S!A(mRiEIPFbF-Y5JdQ- z*L_rZq?r2c=;+7c{Kerl;M^-eZBv|E-sTmnqsDI}&jHK#X5!q0nsb@D55B z=%uRW^k&nd>WkLvX$iw%5v@el8g;FnHc!w1F5b74Y>AM6(FEUSzuSsF85jATdZ;v1(k22&C2wmm^)Q9v^_40$V-u#+4{ujBDI z1Mt_=$vk{<|MBB7fB$s<>B~|&+vjhe-kbb*k-UB?TX_}Z2(SK#$STX6=!w*#1^24Q z(PnNfBelVD$SQK%qgWugBZl$_`=HJOmQ!y;vEY=tJwOeCosCCaDZBl} zS=f5`>u(1KwdU(*aMDj{G;5&3QLEsLq^ch8-S1Sjlk;`7(5chvRlXHn^?cBEMAAK{ z?&V?>6l5Yd7hgedbG&zcHklJGNNltwD37|rKWBpv5{&1Monbtc)P{fTyxZBwHTllY z+0H((>?KBn)WMCdKbT7tf%(~D)G-xt)>ReN-&w`(baE~=Wz-g<%d|NgA#$v*_@67J zjxWxllJe?yjwLPjUtDIUv+if#`$D6 z8+3*jC$nTc8%WOg%)fP2*o}UCBVw8q77^qBIVjkahQBNq+M8s#l)mRsfM6;*H|y13 z_?oxM5kt-)G^(TXlMyRq$0zQBWTXs98dKi_QKuO;YA}=RA8RLK)usGci>O1GjQL=O zbNX2FKU_>aabYGjJ1R5ukKSO+wS{PsM5*n!XC5R9h~jUhP+jWwVDF;mqqOI-$)0&L z)N(jKdc(<(XRXp@E6;Rio#q~tUo%;wfd-{UGS;>PbZr)>S452}l;%tC;Y6@bQO*4B zY%)#Gl8cj*6UuTG^BT6sga00Re;H4BkK~4){?&^;)lkZc-th0oi|(M0IRX_ec#FvR zvx$dur1wtyyb}19gaec57UnTpo6go0m^9(s5J`YPWh7t0$2UylJE_Zr~_hcZgMoFHANnCc#xHg8OFh5{- zUQVV&=bhZ~o{h<&Gni0XRI1F*Cg;0$3nHRY&t*=Esk4W{Qas=7%ALW7?@saX8_Hc$ zD4}^2tg7&?vTr=z;~5)SH#J82Dn){? zh}WbiZ`z;AkuPXSow^d>qs~rG$jUZ<6OyFuM~u_thhL(HkEXA;f5u<^X&avL>87VF zsHBP~iL%ZjQ3y2u1x0IuVO-FE5vQsYyO-B(_!zbdjF|4CEL$7BSJYtzf2fUgNvl)~ z$@oSnjO^owg}UTu6$MT@7pu=?M^$qjac>#Vm8zQUWS_5*6u?PL>Zoc|sdPgdu*9co z6vw3Ml(czdlJf@MTPB}Sp+*<8WZ37zsW=pvf5pAG1Bg!puQL@@jk&rwhyD}`y-DeE zI7J!E^SFIE27WY#JiI6roLYzs&5|K%vIiyqlo zDQSz9?$h57avh5bs(;l(SRW+U=PP+FI+k^3WTrx>;96ZqhqJEzwP)~Aqjmp4Oid~B zSaJrJ!d<8J93@MsmU9M|N~@JD=KM%Ilns%I7_QWlYqcyWS1>KK!x}-4e2k%3Qo6y} zyTifVhu3ikU-fAl9>V#aqz_?qoAq{OLcg1F2rr|mJ1ipv1cznBR%H(FRC%m&hhm1c zH7p`5+^Sa$m#RXIR1B+AIJQlI|PqgPn^KaXD%|@)!Ouzde3I!)vuE z{o(N4bf%bz%X~O>6?oapT{q2sQHjT@0GF1-Pg1nB6p>ZkYjE6#V67ga_#&J%FeZKa zPIip}Gc)(cg?Q{elo52`*>C7K4#syA-o}9Zo|Ib%?ZsmB?ks^$LQ+u4*eQPtv1BP! zd*H<*m@VD2!La8|`rh}G-W_4OOr~$-cFwR<=)W_r!Fz(<;5r4>|`yXqZ+X`+ZszVDN1|h78AJYI_5tskj50mJ}d(CUWjaPo! zhPW}?AAya8b;Z=#Zhq;oQNhG>=+h=&W*IheMMPX1HF9Y!Xk122caH4}V!9Gs^3$3% z;WEyhq8vP-lxT)3J>ym^fg7^Q*BP~RGDi$d$y|BMXnY{9D*_fFD3jIA4_i@-@WOE1 zY%N*2P_}uodMaaHz2!GiS_Hfr2RQ81vl8WgQ^}34!7w8j=YRh5{}p1r*wm~Diy;cW z?3{75dxSfy`7IxkkAyC@6+|`=G^KRh0KxC>{qW$oXD7YnJ*T@x^ z&j4u^$lT=0)8BDCBx?%JaFPvAJ|+ExB$%*fjgqZV4vBlxKXrh}OTfh?eCbbVeA5F* zt{KI0BrR#pCZ!-#?)| z2dYMeIb9kkkEAdvF22sL5;J)u51{0R^l@}0S0aw4OfW0&iJRz=r)?EF`Uvp_B+dLkjG;d2Ej8uuYwrhG27m_IwvcpGy1dP7dP@VxXf$lx8* zCekk^xF@fl{(5ittbYwC4m;oFo>nrIX`OcA@@xL5S_= zkGyBz>)1OVh(cl)6;{d?NkSk>7G25tPP&GJL+{<}L()APO}gU?T1_#ogI3yiLUC1J ztK0fazzSY*@HGR{(-HWu2q+4gFa=f$H^>aM;7tZrK(#H>Y|wCJ1%pxYT7`C8ykK4% zb!TJX#Eb75TlVH;|f%G-#tPGWTppn)^V3jpUf z6UHU9OqVd(V6~qMF?l8-2Zw?`_d*6V`*HIYYYHHAXa*3gYQVX)|3!)9pg-Lid|U2&7Ac zna43NdB}U44o3R@Zru+(5WrbLDoo}QQV|VH$ras))CS@!YEYU!>D3ERMWTKRR6%z_ z^y^xCABuPeRpcQ4?4ZisgTteHhx9D9Ep3H0y~b}Xqa$s~*jm%Rw!9T=Sq?HfIuat4 z^|O~CqoewLUomvk;YsT`rZ=4Dn#=v-YWITsjZfhhhtH1A6An?U7c{yJKj}8w{YD2S zbr8j^POFF6Kj^iZO)Iv(TCC~MTlHqv@Wnpj=_PDqUR~k(=cd$IOMkY4;rr2kqfI>g zY&&oaB*<{}2y-dy*`U@IO@0b7M9*I+W!9SW_fI}>^*LLj-aMU6+wXpVb~bxWPgfnm zw{g^!POV)dJB-VDHvv0-OeKI>i{VTjr+IRWYy!$A|4AM_({fDY^>JWc6 z98Ij~pWTSqmi#)&VA~4hv)k#sNcauP7vH-#87;;yx7zaNZg#QW~~v%~vOp1lxM zm)+Kj51@Wc)k9Rjz5fO`Jf92|V(3ZdVgF1$)p@XlbpF3h1K4bO{a)vAaK4zwJpr9V zo;(&-(L^xmF!p|Z;9cxi*}- zghD9e5KMAhFTxOrxlLe;qcIWQ_(LKU_zkGzl0MYRvGZB_=9gH_TR|bpb?baAU^G47 zs@mNYgsg6UJqGdbwj1{^fk8@xQ;9)JwUn|r_02DpqTj6W%`ZTv3=Ld~HcpHtXybmX z&cd&e@C^FT)86^_Xa0-F*I+oh^3yg{(D-Ig7PSmXD&RzfUm*Jjxn4YxdnYOYr5NSm zcMeZJ54@2H__DdnNkPDs!to(m1kp4M5JHLfGB#?3nJ4?>h`S{ApLZU!ud5Ts$CF5i z#zRpY3}#Y8n$S5cNMen`g79byVU|ugJ4B-~?h?&bKmTV*ggs4OPeip|X{3?p$dXQG zEg`BxRVBuqi&1ByB6+ScRcav9@#G{gpI8!-yhjC)zV?6*vl|ic!3Ky$D7hYd=3`4l z&fpqMx0r>mS*MAWz;6ihD#%pYbeok*H*%fkC?>JZNry!D``$NXvnSN%z>7}-jJ*NT z9z;PqUiMo9@80{T-6=NVLz@Dqi2V(KnLfb&xV3;>@j>&xE3}gPx2ON;}iJ1rq4grfAFJQ~qHT$I${VQVg?wsjIE$sSAL-Nh z&pLhep<7aI-aCBxbM%jT?dL~w0^FvEbPhp7sJ6)5Q2ryBbB!hmUM1CQ&DLy}3mH>$ z{*pPbE5VqY_IGkvMp1ss?z}{!1|N!;5cnJN*FABY8+7DYTk3T9l!@CVf8iPU48GKC zGF*HZ2y&Yx9%twIAePWwL!ni5Bwod?n3$hjV09iKAR|vo9z(oR?*bBe&Q_0473|K{ z5=JE@Ar{H9iZFml#;B>(yt&^X_>tPq(WxvGq9G)_lfNGlSt4Jzx16ei8qBdAevzCi zA8_0X8<4#t{B(q-865B3ef8nNbB-zCTFgOaFhsI!N$S@e%%s<Of7*x6Ih`&rnZNg7T8ix)W#`=UOM=nMd+)xV#FH1w zXue7?y3IF~Msu+}Oz^8}H=+Qp4=yG_BMMn-)}c}^%}=MvO!BbLQ241-PoVrfjNZX8 zc^_beZ3hvFr@1E=n$~!~KAVv-fu?8b<9Pf$!9sRNJej6rVde6LeCC!J zg8d4`qKGpH!++EY+VJAx`ycPUd-L|q;{JK>8pNV2KW#&?sA9X(-X;Xwtv4hh&CQgH zGMI(E2nrWKun@7rCS=yc$$d#Op(b%Qh z#{-VMyKn^6sk>Cm=&~;dV;*=)u11{#cNC|c08|OeQYyymqW5+Xzm7RkQ9VwRK1Vru zR!%YEolko4@Qyc4hVgqTzvDe7`dK`}S7w@w4>R${7bIf2`k}(iaO+&JK^{zlwUUt3`0ce^xEEYo;_uNkH_8QV>NAqA)?( zM{i03J|_I}-eK6EK3M$Ry$)n>)u(NU4AQMq!2$fc8|{X;EtTMafr5qWe~yEO4v)3d z?A7}SP;sr>?E76E9(zh~ABzGXpAU8&TA9Dg;yFzq!4G=Jd&L%ZUqh*osW!wIlx9Qi z4(1Y)!5k!t0qFfaNK_CC#g#0VSSN+r_ofn0ttV;k@BTzn=9>I7Jy1H6g^Hbd(wV#$ z5Hr6xm+H{nFHTjLx|7KiV^>UcNCIp8^Pk^_wOwXn3u525_1Z3ZPzbW{uRF!-0W^xh zH`j@2fTCa3Uv}NJ^DopaJ8)2@gU&)d@eg4ml8R4XnM~<~7AP;Jv=Xt)(OO9oo}#7` zGjXn7$yR)qO*sbi)F~?l=9kg zEi?EqZ1T*w2R#f0WIQ}k&7Ta${fWH35r)P2-~+C)yhXKi7(QJ^>pruTfdwBcL4@;n zCvORJIysq>thd9$Ecf+z?|i`&9^upR%TN8`;j?GsyTAN&aC&gRxBL2B`0v@nDrgSZ z_00UI8Ad3u=XlT26s6EtS3Mte-R0@jy`1LRz_#N(4Ik3sa?t$oo*Gcf7#Xv)&d(-u zWpN&SkR`>ElhKwh?+AM%bMegljn@cuSdCpGZU_lf#&$6>RT1HIRmkr$sZHI!*6G3% zlt^=`(WZoxkw)vSikdU1Dr{b=CK&@%&8S@QKmU=Xnc02pm7!kw?F^T&(YG_4>8j>D z6JBp7>DCm_&XcZ)oorl87Wg@L;Aya#_3(g_&Fo|Z$$h|*z|_(fUf)}?s3K6vKKsW` zXMo@!(JBe?O(YH`vYZM`!}Pc`D{XWxLPl{%P=&JON2gjOviG}m={oza-s!lH)4E=J zPdvcZ@!rFWgCBl;*Z8%6e)9fZ`_=g_Py0KP*57B-rn-BGY;*`3Np4z=&69XOu%9u+fveE#}f zbpG?B-y2U_!Tp1Naon;lwmx|mquP7KMI47Fk7h6GFZ`!xlj!#-{N3rIY<;r7-nji* zL|!q-tuhc+#h5jYg@fUoz^=pOZ6fe?IC;n1)tuEG#)3h)6be&^&UM80W%DSaucD1E zSFWs5$S4EhN_9?-!7v8;Mf==#Q64HK<;ru5@*HGY+|%c`C~x~Hk9|&&O^se#1$EYP zfCFsH)lB%iin{C>rcJe2-L>YHm_}0Fz|N<^oVV+(Qj)t->rDYXW}5mt6Nhgx=Pl;E z#hfDfe;OS@w$p2!$mei+Y>I%5-gy;!4hW1^XE~+n<5P!|xw};(6l_69Xu<(!rP2)q zS|GZl4j)Vh_}=|L_TGf6Z6#|P{RQ8DP$%p5IlY=evn+b`>KHJABqSUlW78j3wq;x3 zDIOToeeQ3+&#ux?Rg#TuOah5dzb~;>rIK7K)t;XHY$)#SaXs8Z4e)5F;r~!g{32*+ z*I|H|&R~f0-=MPtMD`jsv{voGN<|5zYt7THTQC?<7pvC?6cxz}7h@zVn9iuh$|7}P z3=9ww1aC(7x&I4{hmb*VxO`=`4MsjVQjSbiFat*u>LCh|&q4M+;J)L_osq9*}MxIC&}uElk#BOHg%BT z!aJJc$ISE~W;>ShofbaJQeM*V&?-p!*Fkp(9c%~Kfi>;$9U3z;o+09Et!{H88Ue}k zVbn0c9gdD@d|3~uwmaHiwBK3LQ;l*G|EB$a$VUyK6$YSNhNifD-dqp!poo6N0#wlt z5uX?RC`;g@rz-mmheG%gjOzpm}r^(q&U{d~EtME@Kpu^H& zpp^n23hC;8fV^flHnYr2qt*tj{7^OB!=TpETEP`aeL&XJXrNLuEd3$4O(7bn!ZDpg z)IQ)Jpsx(Z0fwLuWTe zQWR8RA_%-R9nM5QFa;@CIMv96txB>i zGZ_dli>HEFuxyj1mw7kSgtwV^1@5XpZQnoHoRTFS253#KXKcqwp~R#W(<4MifNTJZ zFC{UD2BiO%hKjmD-9c5vnpv;G%UxwvpR2kLGWj61bR4Qdt-xQWPG72a$i7tRuWRCO zHSlchIWpDZ9CsX?Rx?A4SFM+*5#4DB9|h%DJ6!10qMr%Dr)aF-f@;%|0 z>D!v6S#*=}O~f||=Lv2Ct-sUY$PYp1l7nGtZjFYnl??jg*d=WZUxPaPA@vY9$#1n< zN`FCesOHz`?)_RR9iU1#`hWkg_BT2LxH=oEfkCQ%fbo&Apli?RqBIz?A!rV2bZ=F! zhGN)34HJYfwFdjqG&w;HmDH#!oq#0toKE@9clK7{qe(6)$rqFl9|5UK#6$2m`otPz zGcSnnj^tXiEd-AP4;T*%xyU#wUFg~N>B%`lk&ZqW^Qi8{%Lb!X8?>sbejN;_v>K^} zs{M|uRtbNJkCT^qptp9JaV134a?T|?U(p)a_q=n-^^pshb1gA_R(36+AQMr9q7IW@ z+Hz0~A#*LQ68e9^NRNxDR#iyV~ASu_SWE+h|D5In!)M%J$gDJJpTxm?C4Rnl;jCG7lSzhTU}6*X>KaI@J5dpf&#XwbpG6jev55&{>X(J*Ux4sI+yxMw+Wp zqU#wJrzXHGazw;OsW0+9Bx_zC!x^@h>N&c5f5s){3J_t9uj1B@2j zNKki~al`DB(8aGHn*q($kS!_1PtX-yOLT}RM6ncDd(HFcM28t@paYz{^Y77hM_{jLoJ{Qy1AFA>%4x3?0((;5*_`)^k|lZZ zRZC~4(4H7wLj_r^M#SMvoq`f-Aa3^9o)j;1iFt`p6%-{bh+#mB2pK61ct5HFbs?>m znXn{R%WUh;Wj0HwWWjjUW$#DDqtabpkqXF^q_TGlNd+nzh)sh z_Au=8(~L|Ce-$VQ(o3B)YbDu%fY?K@FXK_dsF<88WZXL-X6g<&HQkabDC6>TNh{q3 z#~7#8Y`|OMq?Tba^!o&ig%|%Ta#8Aix6UUBwrX%OND%?zFpMckaq3!Hfc3x{o2XEh2cD2)wmCC7h^yY+Fz_5+$wHg_?)9y!b z#aw`6s^5oojLI&i)Ak8Q82Ayg!eEF_?x~0ADRYR>BHhKe;9|FP} zsrRP#cDtj0u+q$pka4ylec zv1vizi)~xJjp`HR?MijN>>y?|H61`m8feFOPylWn){cWvJ2*HXjRePO=HXb+YFl(_ zgCGb&kV@tE1l{PM+iZu|yPaR@d=y3Lv^Cat!pEvPKF4AzAW&UGV)rkr zm6_Ox9p7**9SKS$B;%fKp4*@IKMpz<51^IJeYFL(vTT3j$`~->I-}Wtn#x!TGmedS z;>3`moMK%Kcgwc6^pYIqIfCEM2WVav!2P8j1qG|B1xr|_`??g|Ml(d&gMj11p=u|Ud24)8WNYVYc4uVDh zs@MVvF~E8N2T<5k1T0MHI8QpTx_k;0%hlpa-RUPmMHcmY*=w)yKVE`-ZPs{{#o zzT?|IvhNBPPujYio$2kza=C0Byd2$gXHv)cg?6UdH=LRGu(@`oM0ie1$1QTG|2#R* z*`|S)qR6z1ksqodD7|1gBT{<7c#-diR4!b7tu!~?NRd*IY9|FQLIoDgx)NnjXv8OQ?EbxR#61X3#SP^sNga7nH&7H-@*jiwYBB5J2}y!?wW2hsjL z6rj}$@JlK{={kJOFThfiw1pSAs$4C)!>i-`LOcBI8~4nz;HTT+A6AislX@~Tpk+ZZry_tsK>eZ*3p9_bg{PNM-8$6t=%` z4fGNa$H*@O<)d!vJQXAjw3Tv=nBx?K;WerNeY%XXrUb5%E#%KydKI~xmfm20>jpdG zZ=D_=y`;w@!7^IGx8M?`1J6;A9OU_82nQ6zmL4?dZ-{(djLQDHNOUbV%z%#2XNm^n zJavz3_pZYAcE~*z?J?M|i1JRp+}S2hlz|*)W6!aWql>ZGU&lY|PX0-U`yUUsC==C2 z;wo;XLHs{X)yGYPy$3%WY8Db<`Hgx($bO8?{!-d8j^ix+WfSG_RShCHS#d_i!oKs$rP8IsLWu0W12pRIrIbAay1B zWSB+{m0QU3k9=!Nr&R+`$Op60Uy2p;5L8<;Jk8Lz@PDS0#)=Jc&96dGzg7qeMpwx& ze9#(zAjv@C>a)#Hr(ec<&O!75A*i{pwqOVM1hu-U$T}DPO?k z?QS7M;qNGLssnNB*YvZI1`6H z6Obc9xTn$PmbMS)Dc2aW{Vw=Qd%rMMu3S)Eyyv&(TBlpKop}? zjs?Og@IU3a*_2jn(;04qC8x{@+t90nSD!y1IC@$?Ji144v}hdFrktWd@UA)7hNNxG z2HTLum<-)ew45bFH$+Dna={>rCK5x4&NtiSY!3w-4*XfBJn~+Q2PrCKITw0ESwxbe z!E`}!0j`!9Qg$U|&@a2hgz!MTgj}lxf?W@rp=wUh5=vT^vV_|2OW38pdOP>rrPOhL zv0ZBBt!IwyOo}>8-=)+gRH8Kkcp~LV<(E*_x{M|C?!$Y%T>kR$L*sr+C^ybev4mz` z7_);Jzt0l#eAJPIwn{2lehFo*%UD8hzP_)N4=&yQ&Ha{8Zk(TD3C+GTW?pPFZ!Pdw z1cN4u)Bg9_H9`?cgX_5k>p+z z;5ZH~VtlB)Lb4)*_v5}Er5J#w8kEv|nKL(~_YwfK5@ScYgBWjd6&CdrS^imyE|>+~ zUS!^+1Iz(HD;nn)d-Z32P2r>KAP+o#qnBhRVbDSxnqnH}yQR3XB4>F44lVx<$eq&h z^6$euIJDJ!AfW13?gFvw+*Cz8zrZ;j^HIGAsc*N zV*CBfoA?}pVP>rmTG$iV@6~mlnY~5pJZqISr$yI!&YE3to#)2+Dc1SSt9N$WXSL1& z3gLRoB9{B2>pW+2FSyQgu#^BXbqCOwbo{v`HukOQYC`vat_-AMpv-kSgp zHS-M6y;3H->{2rbW}LP{00}VTu+s_7>*Ly>H{8-b^u|N2i`v@3;418f-|%4w_AXsV zrNl-Mffo+6eRMb+4>Z*0u4z}jAv$$Oy$%7WQM9`njmBZSjnDPp-o1UPb)t5gjo^2K zK{OnN!+&W9U5q{ozinuQ^8Eb>7omFK{Xy7?Sl`kBfEQq)2;|CDffck_qz4w^#4+m& zs0PUBCT9*x4=}0<(L0|W!;Hobqhr_|!`(0-VnP{JbW&^ey5UZ*+YJX1p|dhn(X+!I z_+s+VIY8=5Qvdp{+wc$9z65}f(Sz7e)K{hntM>`RyXRzdatsZLa zKtulZAY9YM^9NM{N; z=3Lh^;Q3RLOD>R52q8=0C4vPW1_KSOi-xmhp$<+43p5yc@rq?`#WDvz$DnrE9Q25( znAOaAw#E5Jr9wJ)P=H8%hX4ZEs9%2oq}1G3TM#Lg z?t7fu#sa&F_0!fXP8PV~LW2CHTs%w54*A zxgV?`aHSxZ3U&JFvvuX5%5QQJc;cnwiPn%ot;(4X>}S zH`={E(XN{rA+mrP`qD*sebw*;ft&yC(Ny$sZHP05wB_lA)l-(9%y}R?Ar6n4M<6AG&RRz6?;BDtYHt2 zG)MxzyrI-K%Io9)%7~ z^Ck$@=)*deRQ!n62gx&957rw&bwDC{9j*a<8N<+DkGe=7qiOGT*XjB4i{PJNy@np) zxav{7~&`6U?d`SAX zvOI+KEpq2UN4pM@kj_;UW^ZDjlxz_EeEGRa`0d;P1&_{v76o4?gCb4m-h5bJFLYL1 zVA3usJw7}JiH4MY|4v9USzn)`=F`eeMVzp5(__-g1saUk$`jCFGtM`Z7I^#4+n}N* z{!(O+V&pdVL409`z!aJe_ul?<6i~4p-J|ND1^)B26}J>iyyH{tMRnZ04%%V3Mb6BS zN~-Ij!T#EyZom;RBEWsl{_-QlXn;N_gig(+^V z%`MyBajs_!Vj{OR@s^RbKl_*q~n0tv0`owKj4_+D6&~y;%flC~(F?5)Z z5ZPol<#f(-IA~pQt)ZwRLwBG3(7g;x~E!z`4UlQwv0e7fpgO|s! z9FxjJNZNqXie(>b15AWJP5Gbb-8sASKht#s=B^cW0*MfWg2*#cg0QZznSKIA0qhm1 z+T&Vw$uZ&UQq#fntyn>^s&EZ2%>W8z_ZG)wZ6HxI(U55a&pN^2Qd`q5#@!~$x#80Q z;e2==U>l(XF#2ZZJ@>4q7ucYl!8O^0L~f3*vX7k>vo7kUU?YKU3a%39sa&z;OC5wE ziIQu8iwmDi^j8PB2`Rbs*J3@nUTX<@tC-|x9k!(bA?WR4GuJ3;t<&;D9$cL&G>}6Z-#WjEf*)gR{~#wL37q1;|G>;mbDgZz zqdYjVU^5U`=Sn@I7mK}2EA?ol9u+EDEA?ol9*x5shViMO2&DMe-Qi2ihX+RGQb4A| z#DF+_7BG7$yiK2`mf%}LSZdUwSf8Y}o%ch;si`t1DR7wyE=Jg;@JkXxjgd5QKQVlo znEz4$wLp)H9XS|C^2#Wu$Dvc)cP>WBQHxU_5n%3WggLf@r3f>O-3f>VdMwlN2ZggE zz`X81L-AlU%n~N4TPG|778DQ^j`738%SB6$lX&DT3Uy_Qp3H*fBQ4{V^+7hm%KA83 zIcEXGdbV=TGONT|IcMPvc$C5Cm2%DfJk_+%RXdvRRjauO? zLifYk7#ZNLsH4@#1Ehc81j>Ey5p9ANj7CWNMV`BM(nFpb65>a}WypGudRi+s=@*Yj z{JNFk3={cxCd%^}K%|#|+%Vx=5*9(OHG8kl#)ee|Cc>}+V0ZBsfhNPMA%d2C=EtiZ z;!uVaLW9N$h&`d1eUAU4E*hPnSsR4Yl#w6%<>Q0n^5OZ>yVKLkF`5PZ$J_8)dpBq! zH*vV8y`i`~^EMShL+J{2ev|BP6>%_JGx&^?mWrKl0q#_h6FQthGIF927;5>*f>ePM zrmMn22{uvgbDjiXLY*yLNpgO4u+Wue{=k~K3*JPR>bqE;j=DxjD%3B&t{C0zvMv+AmH zEA4a0HiIYU2T5!$qH~ED&FfFW=fL=8-QW%<05K`4X}W0V9`WI5T#tHlWoaCO1t`!? z*op9WPDWrpAwC1gr_!#X6t|rC80I+EiJ`pVi!5vTF+so{!1W)XTNcDg847}K)D5qP z8xheV$>7QI5NV(qg!2$oS5V=QeT$^UpuM3Tj@zSXosrmqXAA8?6GQ~JYosG0w|2xU z~?miV?!_{Th45*<~MZyQK{ z&q;l7FEHW)SJRy>6A121>MulGz<&_WF5?=Lm~Gv8q)g%GoM%pf;!b#g)l)XVeBHnxv^_tBe&;&>yBT(j=q(5$a^wD zvj7JNG>Zv=`jlFLxi~OL<^s~!TpkCX5or-uxMO?;ajY7Mc<8Y(??9G;l*(M5jod0- z-87J+ZxJa%P_Kg&frRThr-mh#0E%3LD}j8~&)B+#l#lws-9z*=DD2i-3G4oFK~JM)3AUf0^nP|ouH2e#`#^&HgDED{jmA|_|39zt>4C^ z-*%1Qp#Q-=vaXxM^?C&2Qy*_#6kFRb% zs8g2SAWATQ$NRHf{l>MBqw%)f`ts)G?$O|nLFwzuSC`WL5wGwt`o?)%W@l^`3CA%J ztS%)cgRoaL8eWY?G$QDkNbPKq){Fv{cbBD{GUT80lrmAaImvOF+XS&{GYPWV!|y3H z)V+xtdeXj}2>sO_{wyOtC9njb4yM4*k%vJ zTJshJ9!!H5V0vM2OokUMr8EN&7tZ9;47`qg;73q$Y>T?Yeri#q{$<>@=v0dNAH z7qqn&I`s$970e;+3ONfsIHL(+@HRv@I7Z=H?sY+gJ3}P~Zbts{EK#~kZgg`$Rrp0CVs8+7ojF_aLacnMlXxQaq>Tc!^V zJ($i{gakP0)uJG16In#9HM+ves7K9zXxGtj9n8lXf)(TO;1cZez?gKfLa^jefT7Ed z{6jl9UB~LXfccHS_`mhnpg#kPL^S$`c7!zpy2pQL$EUxIu#j+KI2^+A25WN2?~zU+ z^3aQLxS^H7XxRvsUaIAYtN2J{*Az{Cq&7#H*@Ex!5Pj!=_wp!`6(_ zmf@m&Z5pN2$pQZ8PzZJzBtJX2v&1z&fxLlKWfta|DPREm_dx>^cZ`2`GGwJ>?AT%q zVM0V=kNqc1Pts3{NbK(J?d^z@_=6%6I+zMkgWa=qx8!(WqyszE>(e*U`^&>u%MVM; zf3*c;6UlB#p^1_Qoet4`Jd!6GnwSgffE@%S8!`qMkTSMi`*1SM>;rT;=r$@HZ37vkGl6Q1}Mgp{z*+BZg-* zOPZl?*>Vd*?nVX!XpQ1)oDtjVl<_}UMh$qfO`4GV84BJG2!8qHjtREUy z^vYSj8ZrOX7E~kBy^-_?AI7y=@@TemdIWz}l*3PjNntv|;Rne6*kb_tZb$%Q1HnS7 z|9AllQ$xkb-UfLwM!mkKuVp@r-@OfhcSRq@-{6=C8zZd_)08|FsWStDhFhhHHv>^M zcr)O!py~=?cjFA76yRIkd~?*b27_-kdi)dWE4ZcqOawuoC3bL4@?J`pnCQJEfqKS(xL^{qg60#+(9zBwMOnc6jThyf$KX5#jXsukeQMGAI!5G79`?A$v% zg|T209l@}yCxAuno+n8KoH$R@owVfT%_+OdW92h9eSp-PmCfN}DI3sj|0pkS_U!1B zLYbmEab-QSSo0*WZn_6g&{?ww@IVyg%CAOscpRE*@0kO&4SC93Q8`bZZe58MFyW4Dfd?W}uB7%=?b+s`}j z1UFl7^7gK=qkN!|pUXGEiVY8uJcbRq)4O8TS7?+A6l#GM zP9U@Njabwwi~7GN({D-l0RQkx4sbmPshY$Yn_6f2My!JPWjP=(=}UZ<4YUqQ>$d61 zR&T1543L><&yFjH+7MMH`Zzq%v0MIpa!^iv$qirEwjpi-5PZXy3wzEczQC}8$b5s< zfJO4=5JN82a55dloD?)TkkG2xM4fWnx!_tiU>>PflVej=p1jqxiwe}JUuSrP2iN`7 z|H@$3kIiaYC4oI`5}2b~jsXM|fT4ITOQ>I4^7j7NuMc+~z>tB;I^We6G-UGI2(xHV zq}TNs3yEU7B!E`s@ME)R?sSX_g=|s!!!$%{Y|$}_S|T9XH9*#n4H#t_gdG@ySZ$@P zjtk-+D0aEL%^51ESM}S^xZxAxmU$ea_{q3O4IMVV;o$X$)6Vt$5=8)0QC!f9DUM|^ z*T#^lPwE)WyyP<#bu}~!68VoYT$o6>dP@n^-0%=gPndW^O517$gAVy93w)@jgK?)% zsu)S=YSe6^v?D%)mmmN1U@3QcwLwt3#W6a9PVYBP0P;t20PO35HpKNew2vXuzrY%R zGoZ~eI)L7Iv>}9-QCuje3gVWRD)Rjj-ol*FcZsA?j=9>U7+eH`15~6;lE@6*@e94B zC{3+;OIaBYcm*t7Gj$jlTTaG-Z9K}jD|AErhdEC?hf9gOPWNY~31&acU1}SiAyx`V z+@OlPG&ea(KgnI1>m|8MbFIaBU~yhh@)Xi3isK&4Q>~1ZQ0e>r1WxvD z(A@(X>{iO-Vslo~UF;W#d3x5ud0d9O=Hp@K(zvkW8gLg#X1BkoH$B+ApFW-*zS(P- z2d#roFULnOvs@(g7MGIDHP1bw0JVPOnYhX8cGdBW^Dm$KC$HXrDPK*z$)zN8&2x|N zuhs<=zKA+uJ+b%wynMJrI2%md0a53ixNu5oj4)DK&l!~*76t%e{xnA%jQB> zmsMia+j2)SPmCs*4T(vn;Evey&%~%y>eZ+(Z@ybruSV^Q&6*pk#JV{+XHag_wS60I zJhdqv#Y)=wB;wKV9BolfvAkO`Q&`=US0aSvFbV0#>I~g2wpJo!d6~8|ZZInma<&=k z24V{JjI+nY+)9M3M2I9pAl4*MgKeovZWgJIN|GC(#XkC}S#AcROq9DJ3M2U2TFB1l z+ySxkcs`?P5^O|r_|B$+=W^BxUEKGEL%3^(<<9~s$)*tT{@A^ey7to}MVzm3XE=Ef znLcmBqe<}7cJCTG1*c@fQ}|JEh}o=TtRNs!_%uy4k5%x5;!vL8gk}sW3;47%(;bF0 zCuk9jbG$3bil6ql?sz6}9Z$}bv!DI>f0UYqV%cR@vv4AtAEVke;Uy@!C76PO1D~OO zIHV$O#WE2f6~pmm5Wzg%0;aeV0`rT6Hvq!|(5iL%ApaPmkbJMfoam!4fUY`3y3?qI zT&E5d&kePH5M6Q*#!zFrl?LaF2IvLw87GwrvYs@;RfNMn8>HTAB6|wu1?%AN32A5o zXTT(_h5gYAPr-G4jw_G4=Nx4=>YYoml^Ax8hd~ArTk+U_kv`}7KH_=!1f-X=2y(4h z1n~>>I_O*=Rk6fkXHo);kNO~IfR3`bFI1GLOYmZLWVEPd$2&>Bd~TY7dI1yE$FJra3xm<19qkF6;A zMRkc?PsfN79(`{RF+95AX@jbmdG`P@kD0HggdTnt=z}OQ@%njNMwSjDLLW*7UR~D! zF`=sif`&~Xi+Dp|Dy^Bo{Ba_30`g(6Vjo#A!-ji>n@p5f7T`k|y$PzLe*-UmCj%}E z-$~ftdhfd1Mv@Tx_e(`6J{ed9KoOMDK`bBX(nybCu4CcZzU_9w8xRl+f+V7!C3+Pa z1krf{u@+Db0vuM|NHqlZd*dMp%QiY8Sk!{cZb1Fy{IRgxTmwYG4$V4JzMh9SJ^WMe zWbpe)nnAmzXoh`UL8_XbI#*dT0<;YvjtLcVk7iou+uWO7+71EHnL zW#S~x3uZ6|SdhUg=+P`8%#0Hx*?a;l$h+D|Ga;MxEz8}qtPJ(|7<0vPvH70`{J?jC z1Rx-B;BH%P*`?5I(wblzejN5ezT1VM4~Q2Twhs4r#vhmub@=Qh1aIqf3WVwK}R2jJny=>)0SOokXU!E30rSZl$=X5Z@Cbt|g1 zw1|E4fmx4DuMrq;C<_9H_n>_XiSFh<=7egojEJZl_E^i>e9!AN$FR)?Pj1n9$;PZ^#6w=+uoq>s?E3p)=iH;=P zh~Ua8z*#Vb&jLEhab0*pMVU`2DWnbxt-Std>NvmV z;<34m1AOX^HNkoiMRie6ahXl&_~%><)w1GKyfY;~X|LYxmQS>I z2QSNSwKwlhv|ao^bvP++SKjhZ6>Vq#;6vptMp!iGf7zs{ahyW}ZjC?+K4b|V-z;xLJ%O!_3>KCqkC-NMO)c9o$p0wd#^Mg~fLSkP965H45}DwO zYu6c^OF)@rrwh+CJBTskF#OuCBK>PR!`Iz}V$WA7;pr)%@iYNIIsQ~a z=8_`K6z8dmGzaTb#u_dvdScF&C$0AZ2w~W^aLsTBYXlKB9#Ag0|M&B+nv*4 zX9Q^9IaCqq^`^Y@`sm>8YdYrxuv=T&b4SmH*8Z-Q2P24QTzgR!Yq|Kr=?WGXLZbS^ z6_kB-yv45c|g9+MI+Yv#g`9{oetUE!eaGcWF zxLU6hd5CnGXS+~jI_IEA8vC?s2wGI-{n7rr;{&3NVMZRmBUu~T>w(GgR%;V|;RTX5 zP)4VYzcovT&F7)QDnk!y4JvU|tM$g+V!+=$lox2gIpxKmd@@;iK^!ZO^)qcS`R7#^ zFjoaCRo6g(wo8@a0^IvU2Z6d#okXC_Q1D#S1u*?N2%*Oc#(Y_)g^@o>gO8q zJY$i{G#Y%BWof48ped82u_zxTHCYyme?s4yuMa zJT=&dC)nyKhFdf%F_#(M8<{>O^9}&F$^;Nww-_h2nW{A`!*N}&!HB6DB@_XzLO$TH z8IrzFF&7K0)3)JK%5P#1K1+{D_k;|36P1gB@s%Y;YDE^<7{Fe3nI6Xp$AUHTj|C##kY`6r&0v zRW+-~^2$Y6p-8F0RTNi%0Vdg0l#5*MqtF}bR(f0TV1SbS?-aj8sFiszRyNg7Wm75T zz=`(e5=_8~e~w4@z?H5Z@1h<;(B6(Kt^*|TnsSK&Q-Mnn)+czPX%BcL}B<-yn5*(NI@Y!ROXU>IGHOb7*h zrCmR9?|nxIhIKl7Sn?9S8vqZla034P$;oY6PNn7@9I!{tsXa2Xj(f!JP(zy~*D8XM%skoM2-#mK8f`}> zJ{46w+PY)mu=H}KL_VITqkxgUW!kBHG^ifPqmz$@g+PN?m3S#+9YDQ2B>Kp>V|)%j zk_j**52s!B4QmI_BLVyT40Ij7;e)NiwLk%c(vn~Sa*??Hsrjw?F=#E_!8!ZIa&kBh zN-M{oBt!@y3(5V+zDv$WBMD+iC3VS}H^KEt;Gl_a6&WU!qlBOcs)_QXB``p__)C?P zGFMi<6)!;M(bU&W3*&#Mx*PeM5eUq0%GTGtbN%J{r`NTvcXK{}zOKkHosy<6a!sCj zFjXm%lN55S6q2<*TXIAhbw6*S;hSDR{B|x-Q?MTZ*SIuF4@ z>;m(bE6D%k#N`ny$SFwb8LG`-`AgG-1+JDWFf1E2T%j35C6n8@a`-IkFnhs*{K{Ir zcz{r|{5!18{Pf)YvOmEWkG+ama_rypsN3j~^%e~KVQq9iAf`GH8&0fvV*wU}H7S+6 z$*RjdMCq1Ej|KOShnALZ*^7=|y;=9S*X^$(>$T;jrCX3}11EHLj}K4*XX-6lrhWr_ z6z=t~8Qhe{CvQ(0we9Blv2us8^^y?(1^qr)fEjV%+2+l9ryn-oA5%C;U2^O>PN`ck z>E@{F!{FWN`TqIW_HncG(!J^LXN7~5S$l|jH9NdA%F^aOjSj$1fXSK5VZa z`!#Xa5MH=9wb4p`Fw%-7@H`F4UBs8q&6jD(a3~l5qxkZtYN=}QFZEZXVQ(OKr+7Ok z_BfpG%4P+e*?mfxQJ?Cf>L=6kR=RPg$+N6QSI`t&l+D^oH;!E*N&?zL z`BolQH%82GV%;LAbbqD@-~~x)i}*;&&xkQSNh{v$7}N9a4P(34w?I=r)B@!6k8d;- zEqXIXMH!?+j!|0%b!Ns|tv#;hP(Mbw{Kl?F7lUuw^EbCWM%|CtTn$Ocr(2h`PO4mc5PG|=jdrqO>1mm?vm+c18Fx0$@ zvrSJk?bg{QqK`ypu~du^P(ePyspZNCcoEcas%-;Plm{9iGKHCD(fV+hun+l^RIbEQ zDH-zBxUHF%rh8kKDJDbe1Z;nFvD6**AMPxPDlR3K%0j6!gr;cRyU{cZcta?RR*0Ha z@xNg7N2GS*@OMU1%R9!du`B%M36a!Y_Fq9iybx_HgLV%o$j|oJe?l+XIn!3B;_NyV zp%sbC=jvRf6a>jbNbyD(KYhy-J~XK>rp9v|)UBuDM}Q9CU&feYTI_HwK;^`!+5IPJ1pkj!{(CIS zXo7iw^BkVIjVuBMv4|R|J+6HT7pH0@2A2$h37g`Q$4cTYb%8%Pf1;VxXSq{4u zoegBN6Zj?hK=E;Qqm)OKUYHNT`t#c|0ydDr@04}r1My#Ju?xh*vCr{kgyXz*@?S7R zbemeE9Z_)*x|ev;s_ee_pzN=~etvu_?1sci{2cBpO8R@#U)k{W4=f$ha)qq)l)Wu{4T`UzUgGZz zdlTcgg)d#K0KC7#Td(ssn!m;7umegQx-L5^E?f}j(3x>R)`#6&ym!Pq1?^37S+%8L zFR3PtVkubp8fh@l@^*sl`$ahk*kFr6mt9RaHQn8^y)9i}O?i}{%N>2UQW2GARf#3) zZ&6)UaPf<1@~Im9kcL7mhZA*L!d(8GS>?LzTRI@LrsD&Tg+&WyJo|e8{3Hynmu@K- z^IvU2Qz6|g$yPwB02P5HtxmBCewL()6t`_?0BxfHtCMXZGBZob1?(FLTTo+Jx&*JJ zid%9k?WSDQqy2WJ)j>&IX?4hY;F&T!TuxeDsZ<_^#DKP_R`;ut)d2sTMOH| zqO^Lj8kA(Agi{37k4Kp!8^HU#cN@&vK9HSKrVx3ziJb@uw1&B*TU&;ZDa01ZoGir; zPWAU7G{twgpMSX0y62QCWnyAURN4vd?cC%f{UrBxu9xKA&b2<6dmGxSL!>fD548=@ zeWJkc+2+UU{?6gIQM7!mb^fa@sI_KWKdG%y#w?^t{OmNv7V~Kf97y>jB&$?HcrFRH zCqXxd3Or75bd$4o(JUlVgk4bj!-fFIQpu5|GX=<%R=i!Av)BO5JC-M||D6(%^7kX; zCbUb(<>t52l~eA#TYuZVSW+U=BG=430u$9o*-M{GM3Pj^iIE4%aX35&Bq5^z7qXD} z-Mg@LOFk&h?GmC9F_7s23gkWt6nXw9C(3$03n||h3!Lk5=PUq1@aB^1A%DxFS3_=! zlaSK(*>jNci%Rr#=37^&Xp@Txzx*N(l2c(m-_xpRs$rz_gdH0QRSlF{gfuEAf;Z;KzfKQsj2svWl2?EmrmM8 zx?QgB^#3^gvaa5wg8W=s64IpGqa>u@ycJXiuaDcWOW*pR-`iOdk~(t_k%ctr&e+3E zk(RZ$)$R8AO@DX&-Kl@DFF)L$CJl+tg(x%AjXwvq%=f*&07-Pe2MDZ>T2CSgDd+4_ zTtg(4Gy6dho!LQ2Vj}H`lOQga_z|-?4W{7UzitPkqC<5Wkxx{Btr8U-wbE^5Fol%j zZ%aEFfih+^?uK=3n_Npo9AYrmOb>af5m1H@xXOOOAiRgN|c?X*%p zsT^xZ?@mrD$Lvf-NJ@Ss&nKrDj8y#9-J15?w9hfl`NxCP{quAj?Rn+MfX9lB!-pZa z)jeb6zp6LYFkD2jbQc~lkf7RNjzgq1iuh-0V(j*7*yma6C$MB%B`86@z@j#?8tz!=5Tnu)AA=2Z?fAjaqEf8jXCnMWj6sA z`?n54YMrpHcc$lbdex{MN_2^nnGt!?$j7h>3f4Zw@+R^la_WFW2E<23kGKH&daQD! zN}#|nEo&2L4?L$rMMB7v0?XlOyiozLx9*KYT&!$F!20}^_l@@bSHB&4t$gT$`Nz524mE|AtV znk1|DP$c!$xT_f`j)F2{ChDWe@A9Kdu35?Ihn3Z?ZK52n$KTMz3fq6r>dVxMY^kzN%W z@{dixs+RlbckfT%Z&$SEpI_#r?%~d`-0i)seP+9IoQ|SOUB=QfA)=Y4?ucR`i#wY7 zn#HR*#FI726@k4zI}2Fkl5|#~n?gFJWPq>6V&xm+E5U~$^#IBfUEo{-xmMgo^l&JB zeS}LpU+=%3KR!EkoZ{e5;Zo#Xb1_$`?`$#G zbbTrzS6OXFoTSwo^{|^^5=XavN>gA*@O2n{3mI`Z zpw}n&Er*oFU0F&yb%jfV^5^2djc7z&MdC-BDx0DwW4Tc~wu1qw!~E*ts^ph<@un$b z{>gZ<(!EQ3f!X7MO8sBh5OhP#d{`|oN+0LxcdG?9;{wCVm1>8f6dYTK0&oVI?Qn`K z)x*HwZpMx=F-Oz@uRhArOv2BkJ9qjvOMBg8ls4FSv58GfU6m}~*)}uaMy|u^)SV0y zlUUJw6O|AsxdFIUF3l#&<3e^prqWONfhnfpk1yI)yxhQ3Z}a+W^C&mIzUE73-L2d} z;2RVc%e0Z4a)Xk6;KUZYOjd48C_)x5_!KYAf+wu94T@7R#a9r^lY7*DBbAD+x#dOx zv5Fwk?;@_LhwX60q$~H#lbmHgNk^>@xI&R`Y#|@1F^a@RXp$(z8GR{69(w6EWMyrOK>&hJBVCiJVCXRgk@sji{93 zNK{&B*0O-*t6sPr)x++VcF@q^r@)xP^HGr%GM%>eHBeP*+V}pmaPdS~>1}(rvynqRLf$9cy z^DaO#4HYL~p$pKU^t61?4sSKimmGlYS5R!+? zah;Nf{{bxWhpoT+cG#*eKMP^@t0~UEpC!pvs2pbMTMk0>S$tj+yE;nM1AK`x3E)1W zy6RH=PHClGkBi+ODrhGK>-PNNhvtWt`w91Xx5)i2)yw#s^91BC=<{?6F{O{6AHYn; zk{3#r#C+apw#G$uKL|)wBOIf)tdSR86wnFl}6NnCo zkTvZCDOto=NL@#3tn{Q-gOV22lPY6rD>(L;0Ds>lRw2T!Az5!tYmJA9DAY++d$BuU zKVeP7{xq&@jQ|3Fp#4>ALYCM0w+-zLN})m7!-YwDcGfhM0;RhdqT`El2O{heKPQu9 zNv(ky>Xz~4z{}tXK|t3rpg!kQ5s4>H4g*3vM$$|Hg@%Dwmsn zM6@d^K#oE$=_jcGmF>#*-kva;tOA%wt2Tw!2U7t|p!O(5%+sBc1;$9wC2h9$-!#^b zO6}#V0P|mM!MG#eC>boscab+yA{v^US`oh|%JH8g*7!q}ft=N?wE~o~Bufr51XvF4 z4;W(q?`vfK*MhbNhh{h!ftMF@VF+4Q82Lkd#dco@-nj?9-YcS4CDR_Vr{C++m#$U^ z8oJjF3CpP?hnvnm_zivq&e#g40^6)v&N>lql9iaaJJ~x2DHDT+Ovi?MON@TBf?;bE zG|4ARj;|RLGXI1}Msjt7A_4M2;z2|Rl8RRWQ?|7P26;S3%I%u7a+%8x4nHdqdj~HTVZZ zt%GAD6u~7Y(&do*mZGk96%LR>5e^tx2vx?Vnz|zl)#SZtvuZ}^u7!HL#|Iw>{BXAw z^FXM=r{g$;_lxkrhG@H_$1sKQ!rl*qHqugB!M9*gA10>4b%fH?M2FtN{tpKr?bxz$ zGlbX}cSb`r2*%CZm9#k*n>nH-GWj41of90+7S#YTN&+m{uaq? zO}9X|)MajS-1I^g&LjsXpy3F?Dkw(_l}qy$PLX+2LaEcdfn$R60dd)!g|i3TEaWp7 z0v&71HZmvYA`F3CPk%Tkr|*~jZIS7E83qoR>8j0*X4JrQOpJ)4aTg>Aykqz~nZ%7f zzqCWlv3r3^hWPSSZSHf_bKmx}99B7P`ejQ3gwr z`(i4jF)2JNZAqdt*4nZ_I+(>>BxfvuCh|?B-nin+#%jq4w25l3e!Ir>iff8!t&u*0 zkQS*Jt#K2Y3G5a!UT|=DIo!}bwtCbh8vP5mzfv=L=aSfuGY}EwO=9gOv*ZAgW{dI3 z^cbcvRx{p20R$)+h*ra*nZ-iLJf&^cIzR$IA5c6=qj(c(S{1e@G{Ip=+{2k#!L2J| zi64Z}B&*!4<&dy!M4TdP87Fx5DDSd)0-m~SGVun?U_Bb%)>Yjw3ZVHaLcbo|`A>}l`1y`(Su zPD}8Xl;B>}4L7vY z-{wR7;FGY=QXdgQg@Tc#Mj7Ntdvq>TUHAR%WxG%jzk5gx(ogcC`c9=(5qn80hP4A~B+@ODF&;|AfEHW=ftzDNv*6C}Zg0>D z+A$V7xqRvuFHrC92-MDag#}fKbjO4$VYwCn9tZ`qovRUy+oSXN2c+15C+aShJaQ`X zlzqReED1K9B{&E9UPpU>2EvHlfWncx9a)lka}L)jmK^*bBpX#R#hGp`>$6ezuC_1s z@o*C-UBRs`#&Q*$j9=G zV60CB5xVZZ9c9E3VVM9&3-MvnNV%gdZNxzS zYerhsr~*qzzKv1)HYHZ51z=}LxEKPKTV0g;11?tJTc_)+kEbdI%@SZ0zGHZxjK_pA zfLr&RJLpGHJ6UzKb4k%xPQbK^00PLC0s6yiF+kLAx=3&{d?c#-Fb4~JCYDRSyE3w@ ztSXNXed)@nKI}UtxHW)01*FFTdK7O%`Xlgrl}4N$YslNQ5GV$Xm4Ku2HHB4zHo@Wc zI_QVpCURecHSKlSy_Gd&_PjqrLFNmr)`N1qaT#I=StZ`7ywjlY@{4QbA$iIOM$iNr zpigeNogT^}bt8feu&ZqR@2vYjrUc`r@%cHs`x>dp!?SKydNE5WzB=>{KYutJjos#1 zw;f%Ai6mLmHn6CS!|A)9W%vVE2bG9>K0sw5MKu?#Vw zKEV}2Un19WQ6WD(c11NNux}ZxAUXhsr7Z`<0aoUFTLfM3(7v}-qVYo}8XF+I@eL30 zL>n-34}|LQhkgCyCVXuSmhTUn|7wdWvANxi1^-R(SRr=GHXTKq<;(b&Rc04dl+RFZd(Dp3{)QaTSHvl``A632(jl31n* ztK?o}@kSTVOT)WFWp-(3JEs2wf~(g3bw&@&axSt`gdZ#Dj7cjdI@TGFPqi1XfWO#9 zMTAiphFe-?tbHBBd!!*k7od)E6Z!4|9fId_*wZfM|1i=tKj!1n3AuAoROu8YFyA57 z`6f2;rJQe;mMm`;)i@A-Ie172p^A?wJ%~At*Y&|LsD^D6BF>bTQkAA$a&RWSK19fx z2+q+i=y1BpatsDZwX4jp*+7a#wb#4kK23I=jbYU3g~JWLpOS`@5VN4p+NOa36d*X3 z@UD9(H;DHhRVnI?2L1y71sbFDi}RDgUO0etB%D+duM5H%^;-3CJVbX^4Z7MxW)regkTqWo zfM?Qzi*Y?bdP;ih^#H#+{uxju@eGz4$VGfBr$m~>IVWq8P!021q@jB@9I$-3XE;^E z2VP%MC5g0#R7u07ni&rA#)L2~dDiFkvOcsbnXBkf6mAtWCVm2(st;np9nmYA_)e8D zZfK=^=iLUxr%EV2>#wFXC+Tc-?76rFLWaYU})0TTpGy zwtA+ny1I|FPl^gCdg)VfzNU`M*oQJT>}qZS?rvsVh!l3!;W`2A5x`_PQC#HGrH%~d zzL8sLKjk?v-%UOFRY_`*&AW{OL!HId%JKgv#}cUrHF%3NAK)W)4at3}PA%jzJ5C6`i$d)>yk zOZA%duzm%wAY^rg^}Gtd0e3bC>b*7XBIpDNm&QUq974&hMfKj&1En?)_|V9eNU3QP z?S&%1DIF!}2Xg6&zY=bpER?C{^>@5J^`kMjjmVg`v$S;S7=^e$0hJ zkqm?}K)&J;tt%v^f_qYe7;<0isM46zq_yD_<1Fa7|4vFDLyX|`i8qmo1SYL>myMI9ZSwWy$*3#veR409c803T~w zy{EmV+M0yMf;r680h``PS?rtFwR-PzFhWBn@}KcD=|MYGI1^q3w5zdhf@HBcCelYc z+;@=;CMyZFSajS1jf5aj?;1Jr{a{cF;Vv9*p`5v)f&4Df$iypzf$db&%{QdXa&SU!L$Na zec3d&P!uzH%^xKPMb7U01?>9r_I7DU%n42G^@Z$uBxvA&ShI9+V`< z5<4Kls*vHxH!Cr}5ft9olJ1Y&iJ40!x=N@=E(%opr4k@|M5n20CSohzhb+wFl-5cH zv;js>H7gtH?c1Z{#;4EaKU!b=pYFkiS~QL->Nu>cP^;lxb8M(m-vk0C{ImiXSR#sL z1el_9E}0BfWH^PdZ(fE|KhJQg5Ues)WdgzKpYxERu2vT-ioo@()#~C@sCTa}($?-m zfcAUCA!zMx&xc`m7>$sLntCXvfkXR@BUm!)W&wwee0|XRGVopz{wvuTCWAv~+7&R; z1YD?cI#aEIh{WzYJm)fntE~z1lO$(M$OA{xqH$IR{cI&mRtEj3&s{T0qUdUMsReOXQ-zXTC0H0q0vYoBeRy zXhdC_b@398NNGg`ga`FU3A^0!>CV~a#ia(q z7XZAP+AxUfxP|Lrs5Jxn^-B%n7vS%z+Cw5VEF;{ z`LDL1{avu*F-)K3m?#=Yu!tOD{t09HkQM+$l>wYJii|q|{77DX8Pliq#Y{+@b0Gk+ z_!A(&_7*a>60nbiz*h!VM7`2V8z)ELUx&X*vL2grv&Cx!G{?6{;=c^BT}2}e+5K6v zK96OWU-!O!c(q-vum@ViG@-BA7AZ^+1vLc3o)JvYss&nW5H|jKCKMF^GLM(0gIN33nIMn(}BcWi*25OdMX)0{AdX zmA2poGl=s-LDJjAM=Fb4fO5E=Eel<6QWEH1#mt$OpR3!%a^d;|@R=eVj^wW8rLWOK zjAN@@xMm1l4aPwy{TKP1)#gSYrK~cbngI2i0k*#lk}#13Eb*N$1e$v+b(@fzsk^2D z`d+-@B@fgzXPf&2^X2iYSMQfEyDw!&%_)Nd4(zDz4({qzY-H>+>tr9O|93y4qMJNPkf7 z2G-!1X!x2CJ>}OQYwMNcYWd^m?$!Emel*_FuBn>IQ?#84$UR9jO7J<;k$aiwm9cw@ z=61Hp<`lDuC%NQ8=DdJ8)hDOi0M*$q!u%Ll6*@Bit=ds4WDdp}1_%UqmpbHJ&^=}0 z2wGCXvRW~#YZJkZ zZI~rmhgYCX17jW$(){;XHG(!vHGxf393rSO72d=Sozxe}2n{P=RO-hHdLKlK_{>Mbgh7WDgVlm}u(gEnv0JN>Zv{`k!jC$Qf> z+x&WQ`p018kG>plm(0&6!A8d#mkQ^N$O6Cb#9^iBH`wgXHiOchJ-GaE^Z9+{xD&4L zN;5}+gJJZIxRL(*jLAp(aE+Df@cokgo9?W&0m!an(L3%WU9l!O*!nwe*NI{?E!weQ z=!{qVk=!A(U}3B8KYsae{q<(&GnQE*YbRr2XOy!TYXTZp8DKs%EFPGoKeF$V`iLXR z9N#l|hK+E!k*YgNt-H>>BUK>71P7gEgZic*QkAyi;l%E%lQF1t!OV~P?%D_K#rEk5 z)XtIi`~$j7^EyqVvieo?LIJ}zY!tk7T)1FN@Jv$;2jMZ~S~175TWULO_dbMmHtSNL zDrq2XHSvoLb&|kHaaWg=P)nFLb<9uVHt*AGUV;nP4)q(k3a?}XLA9GGD(B9;nScNVuv&e&m zh#(BjT;Qk-2NsyO3>{2PP`y(J7V`tI0!NwC>6hSwuwYJuYsW@OaKrK)=WO%v!`@Z( z!}{g&ok}JyGT+q}bSf3>LF8WYL4-(gbG*7a_tH}i9Qi(6*e+Z*sINq>e_8nv_Fi~S z(uN@hj^OS#z;!Ke7)b()GO)De7hN`%y#L+~QGKrtOu*U{Q<3q;<%G{A7%(tj6=bd`q?v-E^Q-D)$MT%#2F7@g{Pc( z9}E`*tsM7dku7NbEW|~;*;todSVBUMfdm;zNXrZJJ(9*rl#~xgW@+fQ4mNVjvYE~4 z$gGz6R{YRJ2f!oSE5F?-?^SS6j!j*loJO}%Gc4Ujo1h3ld-iN9kJId-R5;4Z@&o8J zQ3;@k!nnUh3r*V#FvX2fd*5fZ%>VuR_-yml(SNl6X&=L=+QYx-F>FPBZD;@Q_;wmy zp7s_lJsvx*6@dH}Lka+Dp8KwE z)9&#!6@Xl-Y*7ja$^2(2r$9L#L*tp#V#f{iw$7| zaxF&L9iNN+R_XwK#Vs*BEVgQLslwmDz>Ns3h^OIid-vZ{k9m90C-;q~&eBH2lIq2Hl}+yv1-2*(pmwCmO`WPJHN$%HSeV>(zJS$gg$ZO6s}(lm z3PaRKG|Ui#0kf#*>p7PwnQcPra`vc?t@ovJ#rW7-Ap9=wQMqw`r9Eoqjc1O{Y>quj zSwXgeTFNDq3HPzF<)oa@x||g>?0hJf4|*R@>-Up#xp97_6*R}fn0c|?V+HvL2pAU1 z4*|lIUqM;xQdUs!)gSK<%N2MM?0c=C{5ZeT3Yz(8!}Wkll;GuN(dVgp(#)%p>q$uz z^YKKAcY51j^jGgcT4Y8hYVPDn@vJH?2AdEnRz~=tq2kmAq##{p<{@@vI*-hXgB5I= zsuc(H!z?Rm8rfEjN*+$mv~*1tzeXrIBv=q~PnjE*J~5RHz+Kn!fv_~-jc^=P-xasR zd@C-N`e{AXd2~^}(sOL876gaR^lc(5cS8 z*yI<77-4_G`;5;nZ$M}kU)UiNL4fpgL%aq*kBwo*4eShy$`{FZ0zViH!YctbrfmzdSy}<7Z5|D?V2in=5_>KY~0!HZH%bxS1bf zzd4@c_}(YpB0rtqF7L#ihsp0Mo)SKY_|)Rh{CU_KQt8ZX=>&m~@*mG>7u`?M+zJtUDV-~1>TcEhlP z0!4%g5n@K=>R8)tp(yg2_P$GnxwSG%Bx@Ba1TvZqAUU~ZwFUpj)+@sR9d4Y|P+~-@ z>{M_grq!@s1l4h!h97q$3K^9Nq=IJL#8T6rs7j6!G4gVt5sE71Du}@B&qs`qa;ZQ@ zsCvjD68KF=Gm&fTZ#jxESV2a9t%wo5*cONgEYAWc*6>jQuvWV`ygb>~A0T2h_th4S z80EJ!1{u+j4FlK;UfZ85VgyYLMLa{4!MP5@5pC)X5K^44G_;k52IOp}O-c$T)mniP z%k{)bS~zcx^`XG4a<0>0O9FO}v;%#mVC7jpx0_ zW=WjVSPkd*5?eg{GW_h=uwh0}Y;N%BZzR{3Ze27M#D;ieHfC&U@R8%070ICY&*vovGKcdb|(g7#AjQL0(fAbnJX1oh1~@=5ciqy$@di z!}|$20S}W%@ZVgn6OSiL7G2^zn;s*MkUkVv~xf_m(-9ciGO>a@el5Q{+@u`UkV#q>sPI6`0iiwmSJ zm%Ry&2)lptX|QwfrAgJNlBGt!204S>Cft0KGk8Bw#In(%clsi9pjIeL!!@lE%c~VV z3*2cblg=n7u+1j(UZlPV0Cs&qKRut;gW%qXJH$SStd}y5Y`)7Y1zklKgyp+>$#*Tn zETx`@>>zr_JL*e%)OGjue)+AIe5OM+uksqbYitVy9f+e zB~Y7GFAnO~+uaAK2R8fFekqGz-f16`OumI^1g&Ga-40sXAozw&7(yS#o`*;j83wma zegJ`G4M&0j|JtFF%@akn<1R^im!}PvOnuzC;e<@j;UWi`2DBZ1Z$!n(gMosmtQSkh zdQ_wQ?tklR29K~lDt%s3jP9$?i>hMs2S|O`RRbD(vfix(#o>*yTR!Yx>j|CDjZ2j$hrp?p)pcG1~e6*?SZ1wvDu1_X^HE zpp|`gI_pShn10 zi9!_$#X=45H@zPnTx%#KXS`C0D%o*a=vG#V+_rOvJg_Wl7zuBE+?JfagvSR^jd=~9 zd|IYSzgk%Ab`MYE>2Q$zO}68i^WfQYhdtZrFS=)p&be6LY>Yj+oS%vo@ML<~@5^pz zfNZ(eSqw1S&1Uycj7+mizS$7#6Sm zv@H#b)ooKpqDiQgUmJ>kJD3iwxN#IkLXXvi!Z?JZ66Ftk)V|B{QLdJO+-hA`u~k`Y zlGg5&Wh}=i1obMht3eUFYPzLWJCQ4gYP!f{GqvOyTmN9}HKBLC|K|BWoF4{k`xmD; zhs>P$^ir7P=g`!jpC?d*IOoaae28lD(&?fvI+4;LT4yeCwvA6__=C*mNeoGS9}Qy{ zb)?*XD$2|ghuIGsqU4XyfUd1gL>mp<-vF*yn(iKa_`^vIIoO}cJ|lyQ3s5VHQ1*fy z+61>*n!g%lV)+IJqDEK9oI>e{rWcmb!1aBgqYzuw)Xb(;wR)*#8;2BcoNA_OvASwS zVoRS=E#&T0?z=-mw-O2Aif8RLcSuMmnb%M;=f`|aO0;Xxw*J{n&v!^@L)>+Tgz#gT zqLkByp3Q>tOCkB11KQ&36KPxAG|hl~bQ;&=Wni6j=~*S$#`rJFWOHYOW+vhImsyH(BAL?0+ARsJWj`;im=_RXG2Jc3aXl=|6u+Ql{ zhpWEdJ!6oXh=xGH23!u!W)Ld2_|!?KMq6AvP=X<|DM|oN)K+t}tYFeED}}1@8aj{G zC~6>}_)$O{5N7w{4X5L=Y83QaHap*hs(4lBYp3ScBVU%RKCttiI86j}zsBiCWS=W1 z$lIPW79ntH?!0voU6AHSs7<|A*p}pMIK7zrSGAN;OMM}fd_zV(c%LMS0!L#KyXB>Y zOjEA1Dy9o5DUIn`8iFPnhKSK5_McdkW`x4U>~OWz>M(gx$Wzqvf>tBqKVOUj>DS3R zT&=RI!(rgB?J%J&Zsav3H$jBoLmQ8V;)L)krFMs-2z#OHw@GV&#}ff`)ecvxtm-f? z31V%Dnp-x)GO9%R@mwNfBtpg<;A)}SU16?;j8Vi~dsOp+Rb9g+pYX#~UE`fg29X9@ zWe;&$5tZaaSuHfXhO2)Ir~j4Ot{!TJLNScjlRf*g>nl1IM6EWw(yLxCa|c5SVQg5wWELtW&Aewd zrDoSWyjKEdEMLa%TVAlLa}39tQNvL)Y?p1bLEf=yq17>F zBe7Sl${S^h=z>*Ub6G~KyM_oB;ElzsX>s0?kI7o3HKkVPWCoCwjTqM!1`e=5`PHg9 zV%1pHU11Gb-Q6bhn6rWaSind#Xv?_dBUvppJB!wun{hZR_c1S6)ivhmnsv)sAiW$G z1y%+j|13kJYt=%tYkU!oLXWVtMhgT1wQsqP)f%fhOT6A1)Dp1ig8-OAGKg$4=8UWs znw@P&l0}b`t=z}FU{%+cCu=rmOx5j(cZ-BfM0CLHGRCo5Xm$7ZwmUa^Z%{XJ56@>>6JxL=>zp6ziY-RkaRaSLaJhfM^W9)`u)B>pU!zDTy&u(KVYf8-y zvt6|bh=I-2ZScOy*i!CqwZ^LM0*>3OyGyQJ7{#HiW^~D#W_w3f3(d~5C6FM2t-svI zykKS5+-ME@lQ-|lyhQkDM@=MCb08~)R@ay+h#)do?BznJh%Q*sH6ICf-Pr=1@OeiV z3lTzzJh1y#Ej2sWf;FKb!hy$~RCO!8tkzi3-N5sMHK#PD$998dgs|L@Ko(}PIh55> zv%3K=3cw+!r#+T=!OG4>;kp^f;mo9OL)(i)Lf6a%4kJsc(C!-2NZZ8+sM@!zWL4*2 zoL{pqq8Wx*I0Oi>JCe;1su^?Ss->|fRd*wHv(>8^_Y*L` z+Xjmg)6SX>tTj_DHM`4X7Kx?`vGanJonwiv>0H3>C;A$weVK=lh?qlJDKxtV%McXo z4KX*^b1*Mh)io%F)_{MO5bKe1ti>j%V2Y*vs+O9aYjQWbGr%wFtRY+i}s zdTY9iXA15`P+h~mf<(XuZj5BL(CjRG4=;cqg>oPBf>m8ZKwJZ3$?Ph~3jl#~xP|V+ zTFy14X6HPRqLK~4isfo8XQ7;=1%c9|XvIrSW z#8-}}@=~wXSk+l1;Wa=_WIT>H#8-jZND$_sQ!O+*3l!nRXU{D6F)vuzHPS4vUfsx? zY(zj@#sX!(iXzNcceT*!8sM4T9D%LUx4dLk=g^C+IS>PoCPAjmT=oK#C8QuWww_9% z*||)OS?*ePePzcOh1H{mx~mPPA9e;#>Ht#^Ly(VJwb1NX$nwH_ju}|)T2Zp1b0K%G zd8OiF%X$E$yDs>OtSn~WY)z@zJy`BTp)*IAK%>UV*{ZzVnkp+g%v)Y3#|R9=vMrW_K06BF@A$;9H0nRmrN(u~V%(uc1Van0suC5C!_8m@wWo!(1*kyC>$t zCYya_JIo7Kb&j+JYeWOwB}o_0OG13-kd7?HQ?=0Q8podKOWT#5MeIeoWM$_-AgdP% zN2|R1h!srEcOMbP7`SSo**R<)C_Vr~kI2D?(oQ1iWfg@$Zt zpju;PXItx(BTRoWBC%ve`vj`9PPUp-v%6xu;uLR*`OofVQL?go=q^`}r4r=Pyagmp zQ)%x_cvvkoJ4eVHdkb^6x^(lBRhH}S}VYX zZABNX>>5h+)#E182_ijTfx`wfraUM=-l~OW*CJ72Gjiounq6PfG1k?(Erop^0U5(F z$5N94C~wAA9N)qFgz)Yk*pS)-4$I58nh~u&I?wI zB^lYsp0?6tCdMd^KbS!{dp(p(=1Z;?nqA{$#ij;pujE=3J9%ZFuGU!5SvKS5nu89c ziMbZX1Sn|GXK@x=3c6~k)m=1^*ayW54R+y6SG+FJCzfig=q`>S;w!Yu90`IVRu(fZ zViXdeiy_6Kz0?J(I>)ZR1|Jg63mXitTy!!4)@*ZEtQMMG1O6z) zQF-5a!K$vI1zPu-*#kmJCWvqhGDACJDjSgV%B5!KBndJ65*$c4aJ29rs+=p;8mqc1 zGkf)GMr922M>-=$8sXnoKU53N&XPPCk(u+n{F3v6Rb7*y&edH*DT0>;a0yomkTtT3 z`Ic)+tadSe5bd)lBk=?a(D0WBmZEZ%RUJmjvS!Cb zvy4+)BO;wU_DDJ4jRCF}THQs>$3MVirI&fhip~Kja4uf6oul+;$3#8C$%6x@Wte8N zmP^g<5oIst4R%|U1vm**O_tRftGX)-b9HyInP3UQat5|$mD*d!hAM?-$54&SLa7h58~16Hxtgj`u-LrsHzwb1OGM1bR}TUBr71uMG7@U9ySCnoSz}4R8yj0tTlxzm3IEEi^laa~hyo__=YF z!IfBUwPomPja8lH_*%26pin>##Ip{ugyp1+^7;8(Ei^luOQq#L<^`*})>?CRusYcr zIHg#@9C)_-RkhIUnk-z-va0u5Ua+cb?3C-e#(1GyC!m>Q19=bgr}2(CPRgZb=Xixg zStD(f$1N{d(K$}gby_f11Btr8hmp%7#2MBUPOF7x*O2g$!toibAcLZ0Mdv_+YY~i# zv$fzRA4^n(*kS4Du%VVq&F-O`2azFrgI=-7235DPYK@iMMXbFNV9$`ETdW znf#;PTACv(%&6cExNa8Gl*_`5*w5Jj^7u)EBs4G7u_S4LrBNZ>%j)j6wz=w^Rs^Ch zmOJu%CMfJVMMcMop#vainX^TmUn*sk(>TgUlW{!k$*404lJRkR`PIS0Zy!H=iL!>G zg5W59^ptFA`lHQIq(U&^iR*P)2)u9zW@p2gc+=zg`DKQJy-Q_KrIEinnH=pUO%h@D8{P!|%2I?Je{5x@9?DwkDpu`QU@n)wnq|6=k^ z5s60oVYu%V=|Z+jFn`k&LWj{|>+nDiC~3c{>^5n%+?pDB2z-6L4Px0J9e9VrH=7aT z%m%%r6HkvjlW8xR@*0K{ChJax_cW_V=ZAT%N06f)3L#t8n#Jk^B;}t`%0j4N_0dD0 zwU9^qr;)9SRCO(>kDN#t#|Ztw5CvKdA@!NQ`}OHh!*6eZ)aS}i+cK2BW;@W?LI^QK zdSPTgE$+}ihcy1W!a!GL?>m>Y@ylX8wf=TRU>{Kp641iwAlf)Uwkf4n3wcz2UZIF0 z4ghg0zm|qe<^?Np2A;gDWi50~NHSn>F}VSY5V=f1SuHg?hekmB{XK56PSKsbU{&YP zBcuY2brf6`1knTJQ9)$6_+WsbtW;iKN+UmyPktW`RLY3NmlPpM;LA@}f0h&#i6@CT z5>FC9B$9K?O;l!!#1l67BvhsPuCyju#q64)D_4AVukTCRh(urV)C1B+JF1d;yvjV0 zJyNoyBr0QJ;;TCPk#-mPPPH^p?cLD7JtR;Fo+0@sSs`LjQQVORa4R9Z9nsmBZeSQp zpqMWm^lP|dVIv?e0?zzIgk$5t7eDbRCAcr?`!emebH^=ofT9w<4u==b1!KA4QDrJle7==EI5m|}E- zw!2Nq;KImA1^q4hkdJYtCF{-Z4PWoR@MuDvHXV$9mpcLlb6ebv?T9tYS7J#O&sg$^ z7KvYaEm^=|g2c60hmNcVblFL3*kD75IG zeEGUDSk*ykrbE7&z)ctbN=J2EcfTt$;0k>f<__x)^(5V*F%;G+&nrj&4V?o zeWgH_O@yTPKRtaidKLONAS7M+Y1;}()iY92pYjdya~1LPPDrxZzSw%MBcoUWXi`7| ziT3<7&XS7N!>QZ^yaYXEbV=@RHLX~|TUKmF(?pS*8Qaq;v21=S8~G9no(QIyO*rg} zUDTw5&2_BU;qYs*r6SE?FFMw=RJx(Xi=#!S9Sa!q<-G3riKy}J^r~bDwZXRqb8Gaf z>}!~s2|8^N*-PA&T)TgoKD-LMcnJl*mvY--1P*?cB6!s(!mPB=VmNOr;DeJ zyrH&S!Zk(SlckA0JoLij;~%(QtAY5OZQRDm9FPW=ejLP&)98-{L#e*#UGunkF42k$Xts1Vc_iVOPRuJ%{X%Qr4B%f zpKgcm(P?)*>Tng`7ln#7T5d}#csubE6C|@U5>cVkiujo!YN;c4IbM6uZSA?CZm5fm zNk8-Na<}xP8#MuTGJaq2tJ~&MC+MnPH`QJD4-Zv}GvD!D8rpLu7u2-^E(s$jtE5Fu zZR0JJ0#%i?_*c{*E^OKnhGq+LNfv9#Z$%PPuL-UFzU$vXF8JF&TfH>i)#6_bVVeZ~ z;?;U|uhsZw{`338#SK8ND?e>($W_@+1#*dZE775{G2TJ0Yk^!_(G#u$KxY%#SeF8z z2A(!6^wPXvygg70;B1Q1hXgoTTGUD@4=dDb;&}x>4R(HU{K2)h;^f6KU9Pf9AuJZu zYp{yr5PBJL8kt)5UZ#TF3#K9g7Lt*=3)F@d&+&7xia@ z_hciLETd25Lg3A`5)Q19rW@7uM*4EE(Ss;^h)&CSu#ik-l2X)pAW9k>U^FFD@?j)UA6blXrV6L6;34vqsV$FiRyHX=R2n3ssGDZaR8W zr`LwgbGfbG1;WX%smwNh%N?F%ROhqvLoyo;NW-P_Av@nDvskPUeCs{uWgqX6K|IV$ zUy(IavQ?Bj_bkQ+tui`29-R*-mm`E!=Ly*@2J?mJiJe!|2~5(etw)0?UI~+FJWnJ6 z-?c(TJTR(Q(FH%+SdrbUiWPB5;w_im7bMKy!iky9$^$35`w_l7UV#%OTg#nluJ9@N z3{sPd8zw<`f%vq;g@+T+(Y@Z!C;xc$)0^H6;KVCGZA+Y3-7EzrVq3$ZuB|w!lYYDy z&W}^)kDYz(3sORlrSwnQ--t9fnUY#xAV>6X3z7jIOLFh5XgrzYtkjKV|45ni|JdC@ ztDRNoPVjpfkc>~VUuw0&`7rM03h?e^Jf99a788H2bHWlB&llWSN+Bd-;CEyVMGPc= zkQ?krg&1h+j37lOs6(7rQOgn|VVpB!w=9*)K9}i?mH`a&dVU!FWniP$+U)K|AChwD z^K~7BT}cuYSq9?#8k3_Qg-~FljuLdG+&~>~^m%Tj9xrUTYw?D8do8563Qjoa;&ciG z04k_H&rTf!7#y4=l6fK}CfQ6PbR{EGug_57EJfGPlxp!l)} zTx_5p$nbRjxxu$;m>I#3NU=m3C@16PSV$_ij7rY>GJ2_2%PZOzw_f>Z~ zITtrQuag3niVw20=Vz-SqUP7ET|C(JbgCsIuhR9N6GRBHkMJGyectpl zy~+lkH*j}Q+@^zqlZvRZZj!2BvaHZ$eMz-i=DA%xN#(%QZRYl>e0rCQ(pK{Foj>Q` zx|{ah$w=Em2A4@@dHam;Q9cjz7>5FJTS~<8>bS{if3qhksCKEX`Z}ET;HD9JA`ar* zzz59ET=}w60oPjb`s>Nq8KF#ehVkfp;hduq%8k8=FV5>a+8_!0`HrH(8^kagJEzm6 z|KCT2KFzeT-n)yAW;$b;Ab716BB(%A4bOvw;1oWil~P7I$1RM#c&GK7IKtgKx`+NT zyAmj$O3PPo9QQ8c^O;iG6#CE9uW*`0u4uV$mT>EQvNt18nVk3Jy%%mw_~^w_XB%8+ zr|}fagmZBk&mE@8WO62d_9xR_2jR*YT{@?eP6xS0cTc8Fmz-f+>M+x|_lgn|mM0|d zlE0f!5|p)=LTXUeVhSy5jT)02Qj#awvSeyg#}GS*9$LzE4%9}#-+DGSBu(Y^1(Z{Iwg``zo;Eke3~#iwnlTddibM9YXnADqYV?9^x( zZ-puIqmxq+>kPpWNqQ52vC89ZAd!$ajUw6vYFa_s7K+fN(hZFw9=^YHx~JX5!DVJP zP<}LRV8wJixlj&WZqw<-HB0nNSt|cBD^d^PLvDh zY@SxU9694b=h8W!oKNEmf|h@n%x3ZbI1f)RJ26GdTV|CHn6lh_6u2ZGLdljRs(eWS zYzFS;Fu1_C8o3oWFy`hRxojuNOQ020LEuL4a9WIesSL zoKe?z|4V4~}`AgJdh`_BC*_--7(&_#zoK5+UF-IOYJTit!-;;6oL7xtLRVupi<`s(>; zrZ6KiPcZyojebbU25Ym*G5_Igh9Dp7R_>l)>zDaL)KSv&H!o zhs!E9-<72NZxtek4 zLdajjXyR?A$a0#F@9n$ceqfO0$k}yOjCRweX854-h|%N97FUgjEL(@-&P)t1qZ1uY zqc~({NB8EZyu<9bcXY3FvUiT#P+`ukGhx^00&Jsvf7+IKFyF})3M9n@$x~`WP17;_6l~3Zrn%zUVc@yi!f#ZW1Z6tujAg$r zG!}(2DJx~wr7-Xth1nW&r%9L_Za9Nisw!x8%K!k4_5xTcVZuct zr_;z9*sLDySvN4WwP;UNN*D*onoINv8P^q)qtUbt`>M4|?b`h5}1}v;m;ZxL#x2P0j|+X?z)r*dSA^hi($Z z)mbc)Z*bWgjJ9^GTc(l{@79t+3b(j2+F_5Wf*p8EYvM0)k}lz#*SOq1O1ch_sGheU z;8kC^-FkAgUKQuu>7q*Dqv8XYfs~KZ&rOJa8j?MRI3Adx;a54jcN+XS@68&Q*N;7} z{+x6)7RE`6kWK(#BbD9BR&Jzvq_X@)(qy=9DiRT@0q zvVNkjY1Vh4Bgop+gn4!k)z@3r#q?&0oJ;xo(ue7Xj4buaW2^h-#;zV5YVJXGIC*GZ>I zN|dMyiuog%00cBCkB&S0s{wT_9GQYeLupA80Fs|bf^E_*6ImRIy|0jFs-=u_rpSEw z%Xet*Q1FR5uj^}NpZXkV^`O2kwoa=TJ7fKCKvHes;6w*&Wx-fV%ee%~W(1jf{pcM- zO>e#D=&j1!d8+)J)oGX3Q41>F4b(zQc{_Fc`A;&xbR0eV)wMw1r*{;!DWv<2-Z0fP z)(auMuKTW5jyyryGgi8>cUJpVTH@m`2lzD?H0Kpde68ez4@6bR8xpcnR`H7HBN2ztXS$IReVTV>_|KYn2C<%WALPzy zkbnjc&9&Y}guexASY zxAy&FeOnR5!p%*t=%7i)DVV~|(;KhXQmO27+9X}zw<{y>2$0xOPl!SNM3n>hKs`HlC>+m5I@wuSiDtJ>CPJyZ9bYpXLH zTU)3d4K%tnE2IkPx_d2Mb@fgQS=Zouw;{YXxc9-3s0dmud|xePh*NxjFojfmFc5Z6 z>mfV2`hb#+(fdOzv(PdOon(A67$@)=;U)PW=XrcO=uL*skK+MpX7IPXWWpe-~W zf347r#J-XPwFJ*dU0T*m*E&kJVeF^^v}UVac9Rr*rcCgj;|2Tt7X<}AyW?VxhqI)x z%3C4W-Y1%{2EJ}Ib*x1XqhYHp-U-9=htDow+yoahK8l+??bA{ngKcy$sp1G0&#XSY zvd4FrC)LO)N%~IN2@1Kz%1$7Z)^Jo6@CtI8I@nZ+q;w$~%vxG$L)WUc%bsLGKwV?% zvy^_yoIqO7rWHwA)t#zwHSSdpBl|Y8*z>=UZWV%f?W9e_D>Zl?Fu1r4-=>#{zzXyJUO(*B>&j zt4FgKlT}gDt6Z;PTj!O0bauURD6|P+t6*30T2z6tDOguane4@w6A>M#0l!|y_BCAg z4RU3Dur*OUsoiBBO?aTHQogKVP3+E3XXFqZC&NocEADzBNdGiWhF{KbGQqb=1-9T- zJ9FMoU=Qiy1Aj7}{h$9+k@S9j-+%k|;?c#sa~`p+r1$yqg^^kVmyMO>-axqij$LkK<2Erb- zZ2a@ZvzI?UdA;upz8pChG2G|lFNHKfyapi^NXE`!F4rLwxQH)j4iJncuJ@dUpSg0* zk^`Z9%toZRsySwffv7oV!J%BXj#)8jK94UQKjfs{$Es;E2i$pUe}=~_$-9W64&5eN z3OvwoS$H>l_}pv$bOVuMvV&jkXR^o|zNIZNfPB z{@ZVQKmXJ{ICwcszW)Gf(z7?rw&ynPJ&BxJuc+ zB_%1z;BSK--q>E)d@b}XO8DwQYAmm6p|yXbf{)57`0G6(a3X1WcPB)l{Qrzjh)fvA zolDZeWPGIMW3gJzAg-yZ6SToY(t%G^i`6NfH)uJ9?NlpXg`xBV&ak^RAOxkxHg>lL zKDIS*v!^VYm>Xr7K(|5|v*d2bHCH)HmLFuLq7cedkv(o8*>cwtBUGm=*(XU3EtmYB zn2^Xfjsb@(%@~2Y&@N!)aeLUJx=k^^1zw1^$A&+zT3@1U?u@!u&T|fp1;KgbPz{^t zFvFHT8>*!Rl)@bZuV3DIvni(4Nb!5ApZYrHJuxKqTCuztbailXr6^2zxaRTj3?=lr zcmNw3PiZy4GO6>4cU{S4-@&U9pWXZYKAdFSGmD3(Bz@FXxm}5*^+A1pqk)OEpSXRm zwHNGp!T)t|4OVQf${rcO_sq&k5stD_OAoa=3&MI(7S-%0YXr>#dl=|vn#k9?K*9A6tgyA736 zK9R)etVoZZ#68tqBot`+i*>?;;<(!iWL?G*hf(C3Pal9)oED;(A?o^`tlXvq%f;O; zY1Ezj2R%uqr(P04iTA#symokw=OcNG*Q++hv#9Jw?fI23H}`0wwSjUmt+Yd3|7{=K9>(|3tuTqkus;q~Bx>PoJ z4FiSb<8{2~4PGP{v&a>99_HUe?{Z77P1Mkt}#jX=&Qen}Wx!{?ykCx^U! zo4z?z>5R8jAm;p@QqO@1Z*sHk^hcD`}{B_p}qrSvmWT`n#kp=n=9Wpq{Ma%88zmCkxh) z4yX}K$B`46D-s6Z$at)?`X__9O^V{yu9>=v^P?kcE9bI zb&WAg?`~M-y3@vEL-kqL^(91UTd@AQdK?5THz2*DZ@OLP*BH+fTLb8H;(}gN(zw~c zJ05moC~B2oiUqnio&)9SqL+|4v#jZ7&a8C{lin^v@*3PiIZ{BC;_%ESdMX2!PC0&(B?xqQIE4*xV z#I~l*@){Uzb2`|YB+O0yi!ut@&ml4pqRJ-B>!qtq3;DG;EmAbK-u8p4pD-1Rw+AAuGzB> zfI9?thv4oIT$K;OvW!)WEwH7F<&G?eLr8iV6s%U@On!~Um?JnAqY?lVvsOM6HQG1C z7gO3=F7ME=)vp~xcB8C-foX@QcDAa)7>si=jYpabe2m5Oou z&g{PN=Jp2T{zUb{tgLUhU){drLWTs}tqb3NLuDqqRus;0<%SGy1&IT?Z5ZBA;kQ*F zUgsX%XhYFk0^x5?7tVL_Sb^|h-)-)PtpbFL6y2BWtU{QiUBRfTE)yL5jOWuqhqOJ@ zyuwyK$fAxZQ2=BqcBbTa%p8-2Tz-i@`QNKS_{xL5a4Gh~-GH$0-KDK9*Xyy2igw5X z)F$C}Z9B$cn(B2VuA;b?=toj%Hod!Y*;jf&Gditb@rmPq)zK_SOM&qgJ zC6-QuG692)Z*L4#ixX+S0~&&Me$5)vI+xLFFb}D8VK?q2BuPC`mX>r0A#o4lC8SFE zvW9sGf`;jQ)?vpzJ0FbCQs#!%x04I!m&w9;HaJV1H{`U%rNSi~TKx*d0#7Qb=g!mD zZ@2^Q;4eJq^@MFYCZWjR|I7P|i(Go13}23&zyJ4F>aPcjlUe@9;j@?WS=akUB{tN5 z9*yrmcy{pc>8mF%p3>EAZ4aL>`vbXpy#VqDYR z2AHxMWbSB^H5qB+_ltv{lIO?g*Dq~c{b^fD8#OzS$Q>a$09!$Ddwp20r2HqWIogkk zoamPF-wT`U%w$JwlekbM4(q@!m9pFB%6}ncy7g!C=js>G#C;?5+w#`ERv>bq^1xc zIC4PF_MA71>DZALEX{y38UGE&-xm`n?xPtRD1CuLs@}ZmU1xqem^u9S>taS)U#E|X z5dT+F`OdFBXB}`E@wkErNU~do1SA^M(k%IiQDJ0Sgb=7uMg(Y38)Ys8aX2_woB#*# zFxUrHdj>u5N6CdC-VpAmn4-6&KD0az)OUWsO2`z3OSGM;o;f&KJxkkJsU_PxGiNx# z0JT#o#V~b4IeN)l*Crl5a3lVY+d&iNg~y{`zCI6se0BZ!;Qh^_Bz9y*= zYJCqrCAFvTv&mTO3-B0KRG;`Bq|%J3{3LJUCgi98;`}Iah?Z45`M%AfNu|sD(#ylc zCl8+-JfldFY~sA#!ZD;7;w#ePU~J-+WdfpdsoXd2!QLs_Mp(4bfHliOzSp3!xh>Jt zwg#tWyvwLT1yx@eixfkShD%Wy4#pGkRX0&Zz4+AWl6GMplYUz2oy7}<)1&~%GL<_4Z&Exa7qu#%g99D% zQQzg)Tb8-QS_)bZW;&Fhqi++x2a}~Ia-T~w4s!xjON%w6+=Q_MKyiw%Lq^4kcVPb6 zzAif2rVw`%{Z=I~d-pCLk7NGvfI^3dr^z(Fh|hK*=7Ol3i9>NoL@2qJynDAtLsYYc zgQ8awYM1ra`fQMvwuQ9RZ#Kk`e7)WZirurK(5t}rT8$pou>f^o8NWxwUzS`I4yOa< zFYGoPx3M3z_XERKVVj_|oAwtzj1F6eBL8JTc8S06ni{#&L4CdX0P4KVTAV$}0@R;s zat~H^JAiEvJQG1U1!3sv_x<^Ai}&wu0#9G{Xa*7o`q4GSU@&!otk84OsPm{}C zX8>FOyM*Zb^j6ex;)2LouAW8MU)F&x@US;B%HqCr@cQ7DaV2Rxe|M(IyFqe6qNBxZ zhD+)U9p0qleC51+_4tMJx4+T%Yd!pNR20wM(&>-N71`D{=y|p0KRQ}rJuRRAWK|TC zpcODA06_G2p6j?o3q|{Wq4D>&vFP0a0NlYKKFYLVl>nz&joPC}j~^e3x-&d~JxAV@^8vp=Te%iJGpt^}E0N|?F z``ZfumiZB)j$M&H?9V^7F@1>q4aKw~ewIVrYa{Z{PZP3qbk8mk?-QpporobMqyJP( zU>c84vSdrbQn$pjQ51Shy|ySc6WE|xvy&~Aw2c_GN^X|QX}m=Csu)7n_GVd@B@3@? z#xPk)k_7W>6_{m%r_a(YvMe>wOrac;0WE1rOT(zGMxuOOMf|vmaA0sd*d0r7rUZSI z0=T-ptVK|kHWL5Gn;2bS$x>C|^<-HhwFC*Q%`dBFW$Le|O9dSjbPgjt?1q!~?xV}> z;<%SA9F&F~_KDs^-8G`Qw3b+hO_t%sPnRe0FP-nla?)-qC49b)d2I|EZE?s$;YHMo zFJLm46RL%bak^G`r-@477j!)0kJz;&=!rk3`j@ECG*N>sN6gU^N0B71F=U7b;D0qq zdRW0`+L8|NB54425qI^^TEj?85WJXJ(;x3so5~bZ~u6H zdCnoUGiCx(01BHYU31^VLqNG5<~k!CP(UK!%%0v5U%C>YCoM}E&)35F2E_6;XucT< zgIWd6OJ)KOYrLc-*mu1m@@N}0pV}2kij~JonDGq9$v8%5UmO4cl498M(a2z{p{vW_ zam83{)Tu?VpF;fq;US4?4)sjpfJ}}vnV=MVR2iJC-p6W<_uE*h@!bvE?Xc;==5j9{ zAN+PaJpcIyga??jHhWq{xpzCP z=pF5#&lysGJki@UKtdWy-dqgkbD&0lGIb8py6Oqss%xlIVKL@BG55SNfn(=GvcpbH z9%XGqtJVK4v%x9EEr7CmZ(6kMPD=q-YrVrCtchyB)qtd<*|aemIfY84e9`pmJ{ZSC zJQb3vCC)%{R5woO*Xnop9iM(;(frZ9|2ZB0_xb$mj_4Zg zlUphI_Obt}UKaN_S#Cw}MvvS^>TJ(H`;88&Kw;OU!Q#NWhI1Z5jLJEWd&oxTJRW;Z zgwL|tj#OSOS?ioviT*@u*K$0Z@S}ag(zBKCZDY2(@pJypZpSfyb85Q=24A-wHX20B zxJ^a9)BSDUJbBfAegh}(m7lh)+OE1WX}cY6$jy33IIROMo%pvh{Fm@p+Y3%ya!ZOS z4E<>?Gt&tUwe8JiA`-Abj9x^B!ImUE$_2n2N))ARu7#STg=WcZb9`Sd6;a~OpS(Rd zc>Htc+t!DqBC2Vpd=|~&9XYyZDG4$dqN`R?5m|LNQ{xp!RJTO3Z^N*SQW0&QHONX@ z#M)6gUDRP^Ck^7Dil?G+{rAoVnM+nxtjlc#7nDrW+2Fm%%l*N8=Z`=BAWV2C*GQWx zQ#BKCf&4eKXop04q_Vo3UJ8S12AsR;mCnOPn&xWh^xA~`cFP<@=geXDvDvXU8fbKu zt2Qi4g4&=(5>P=MJNwlQn=4sYjY{U4IE`ngSiCbHf~7`{vcIyiudwabfOoaTst2mM zgI7zf9lS2Eyfzon){7XXOXB$(cZWowB>zkfiA<_DCFuM>FjW^?_%EtyiFl``E^{}n zoXgetF_k6Va;2=jYR#+VBDz}xTAOlrlI?5?l=7*l&zK?`myHao)v`I80jOfV+{_xt zj<7=SpX(~~gK_)Q7LCkOtL62O&^OuvEVbszXId?FDI;fCEm6`>J7ZqEEjf5stB}KlIDC+k_(fMVS zbQAhvstI2xb<)QuqgwV^nEjGK$afM&dXu=STb?lCC%%59@pu=;_^}j;t35aBN9fL| zM~JEJY7I(#Md6;L4;3?CqG+YkZ*Go@9-Z_ns3+=?@{G@VWcAp&dVKS-)16lx)~`V~ zn2x7zM3090*6DDYLzce1v_lqX( zyT93?oePO3=_I3Ma&b)Kg%&&Q>J)>r{%Gr+hXXuJ#Zb_nOnQVUk+6v93KAcYDoI5n z9WA0TVG_zuN?B%l`l_z!Y34~@igd+FG9(s{T7-I&P+1JJg(T#h**2rsu-PsVu6}yt{9TOp4gGCc8}!atw&@^1hPY z^*YV#zHH|u^Li+8q$PJ*Eq>Xd?tU261j~426%pEfOYbhso(Jbs2ihCY-}k+JzwoE` z?xNjKrjQx@Nz|UH#UgVpu!3_A9z1A06ne<4G_#xEa{GD+*-nJc*9}^j^1R4Dy7%JX z@{#9$-?{-U|COJ%rIx?C%Zrkq(;y@wA>u8TwD~#2EM5QDwEQ_HPK8y`W!bw4K5pMF z#!leke!uUv*idpSo@u{tLWULuo<~xJ2n}gT>0U0Ctu?jZ=O;=Ebnaj(Ba%V^b#i{D zFg3Gkl$|TYIj8hjs&wo;GKR4*9wcd{J)J7X==sh(rq!`(e;%w?V1N;2%*tIr-@Z zP%6xmn?0?B7H>(2_eOR{S&|lE5{~&*PZ8mM%QUx{@iA;Qxw?)0QJ})*s58kJ(<`=o zg-iVggqehhG_6cD7;DOo@FSx1|@Z#CU?`J<=931@P`(HoARiv(+dRGxk z_m_dHC9Wb?-OXG@%tPBU1C}BGHT#N~_KHa6$LY?#vcTJp9oqD}7^OD)wnWKM6S`(K z{Y)Pa&Ktw4WL3?pc#D+hkgd=!m)^}VMU-jzMt{CDEZYcgz=SnFy%i24=DTNY{24Nd z_VTJoMAo{klv?W+_mE1gWQ_|-dF@+ISbg?*u5hzSsPUEj6zdL+yn&o;s4yOt^7yE# za94Kf+POP?#Qp2fID9hoN2;;QEb!&?wpvXq55xIO!Y5j9;9 zAZ}mG*vyOR*+Mm8-k?B65yqEmi(5pz6SWW`(U>i!`FDl#;fw zFdhP*y2+l<5_d1(6>Tt`WNJh|J*4Hcoo^45$_| z0n)|s?E3}bGf+GXQt=FWUQ{@x)8q#E?c-x z7)uY6j^l?84wzA}9~e0Z#q;5+xbBYopyU>LyC_Qu)QVay$ze%SGGQfyv-}^!)90NZ z2hHoZH(&i}TOzIM4yU{#8xc%mUTqGbTS0xW)+y422Z2zSabjoGCZCJ+^&9usZ(c!-D#BT2KeqHR0^8g*dYFPN`*bpy z&la3aD(G$*&C$16Ee@LQo@fXHs0hUA2@WI~sAlQFUb_xr4LyaK)=*b8e@J%vBt>e4^szBqNf?k~BfT3e_Hu4|^5rlpL<<3}N z>Bb|_tfa>l0`${lcwU{FjVHrjzl+0nZ}?_gX@B(GsSqIw+q|M7FGXvh{bCK5G(g&Y zt%4N-6n!M282$ z!~O9@Z0Px$^{1ALvSh@y>drW>{Rpa4qS_$LX*VJd<;l^#lUK+7!^vUy`k~3ypSC45 z$v0NMV$epUZu?;^hx+YS0?D3_x52357X{W#!CZ~1ntI$;|bX8W}&3q8^_oLyE<%U=W+1KobXu1q4H2}_q zy4hQ+%STa_y0gPU??{$6PNjidS#BHz9?5Z$fUhkd z#g?=rww@Wddgtag`(Osw5zcECS|o!k6}kTacg$ zCqyqo?VRfd3=2|LD&5c)r2FIKYy#K+WHjKnbDtEoCzG?qNjyN~e6$GMm@uvYP8Ipt z4`4!|_2?@Ih5IU5K{n?tO%4KUAjD>?;w$yJKALp!&9yw&mZ@?w#_G4|!w=5SSL$+7 zFup{YqX&f4Z!n6sLs!(O0>}!Nm|XE=I?;I64GKUNcjU?*QsqRXFd(nDkA^k~LQk}H zM%$L|2Peu(^-{~W#>RSE)RfjT#~Sz~3C@hfEK$7s>>72`GXf%9D24O$);qlAU*(ow1C53t@wP8({P=?5C=|b=kxIKa*nLnu9TD==c9ShDa zDOV&>gsocPT&mh&lnSQoTN4Owwsb0)XN}vIxb=Xzbt7r4sHQ(U3rxkLQRe|;;`@zU zUsfwUa*e^Zo;9nR-$1A0J_y_mvP@J4Cu7U7vP^t58R{ZD@`485F0jwA?YihxTEYC; zPrcc1H=$Fx>eIH=spK1#Sb17x;%u=EvXt%|Yf3%Pyl)N8}sKZ$SBNkYr-}lwI z5vx9Rpp6RL4rdeS1CF-9ZQy4@U)A-PnDp#=ACe0`tMAj%pViNfY|`?5NcI@CqDYkK z`SF-vlRCNo5DvlcvR;3dvR~y0T@L@Q!-1$3ge&;|vQp_r;Xo8$#-rn7j=p2ONKO^w zUqi+GHxy4EpM@%*pCy82o?^ay=^+qgo>rLI)gK)E$ZpFTEqw#p(`N5LCIM z&ui$Px^-VT0~(4mfM@8(3JDi<6Y8gKB;@w7BmdMz{YXJEJej#eZQ&}QVO_4Dscucb zgSsOOb^AgS##hmQn7!1Wb#bhVRreJ8LG}#YR8JQRiC{Ve>Gut*`o!1%#nY1?dv6C9 zyoOZ>xf;TFm)=E~(+Ro_+vF2#`y3tkDA?X|l~P7I-)MZ-W#jNWk)9^PO2NTu|MZ)# zevig=VY|y#fZyk3r%?sHK22nw6Q&BERo-RUS8MaU#huA9wAZt-7fSAuO#!ur+gsTt zQ(35LoNR_w9yu>0b-^a^xe2dE89fsa<|9c}@H5VV4s4Ph9!#WNzRTVsym9Lxt@r4_ zJKTWS?Y2Q$5~hh!Jiv#67yx$sd*h1-!S}& zT3w#I+Z1+^JD`F+T!e%|=Vz(OQhk`Ij)NhM>K1mYP5%aNYQq z`y$Z@32hd`h`VQjFv9l$R3E1VxS_KOabJRqtd8y2@SfrVbL^U> zas^+Cr8J`Y%zlCzHFnfXF%@LAK#__n>8L8fKnnwktUDEMd<`*cBrD;N%fy|uL7{JK zq^n-bT&ZN0K3&To>eU5WT=BI$qW>_kt75UEDQPg3xIDZfQ`FiED3C;f#+8l)cPCpf3qKvMnhpLTKxfmY>k0s%BO1Qh`kmM=JiQ0Zs)?Jvnt%vy!jH!c>ye zwe*+M{nUt@e)jr(?oPia`IFUWeVx(GmJfCKwo0u>um_9p~Jbo0SDI<;wgFU zGE;X(p64OhZ=;JA=(yT%O`yL}eYcy$wkSFSle?f)%EnsgX`A70yU7Etzk3!(u1fzP302z{D~GVPG%Kan(!@inRW{MpkmD)ElCiw4C#+ul z?GR$tyW_>40iVh7lIgqf|4QkUbjse+d0MSzIE>J87{p$IF%S_9`{6l%a9^ zBk*n&bSEPsMlBop-72tE`RT;Y7fU{C>lFJpunKPWoL=d(;eEKvE+jT&7Zch~Lpz*P z!epiFADPSUG#-pEN8{ma=DavLfxQE-8mAYZI#7W)@!9@YUw!3VoTdNczD(w@d~`j3 zH(ckVyUGLM&&+|M9d?5Tezl<}Bs9aisfy3GX=CQ08aHoH=%G;F8xFgKf{O$i_)Ul+ z15Yg5rkcO(uv@8KYT4Gl7w?K{mbWgh{$2~ttX8uDi+IrTk#>p+P%Y*D6ns4z%;p2M zr#_5S@(Zgwh4yoQCHco7ff{CbrBr`kyf}FN-IEv6w%48!g$+Xx(Q#kAeA9Hkc<|=6 zbDtjs3C^<`CxhwG*>et=fd^fk3jCUNpPJ*ar2A|&@LjOId__T8iJlh!YLuj9-KURP zhT*PwDoBPZbf0cZbf3WodMaoGFLaeB83zEd>htNKvw#L?cASulahMdM1pvov=P!8=$*nDmrKa2Ep{-og0baUF{&ktf&@{f5hD<0lr)$ZKF(Z!{X*7j^rp zZU=i|=1<(?KJHd+>qr*uZQN-S1J!ItyfuP{FgZp&p&5o7visH1m}9fG=N;XPwAoZY zo~}@j&!uiP8wstfsnbq7qPkPxPSmA0hcQr}%?9xv1k-E}ig3}?K?UVo$*esc1QxjZzVw0WONE%qX-O@QP|f0|Y9|sOR%Oc2<7(3Z%00yw*F$6X zL%sz=jO{#dWi9bF)iM@;fPP=n&q@DPsan|L{XnpQ`gXE>`R?joeMSG21+VU9k<+Rw zk5r#!9Jr#MAZ<4y74+5=XN)j$@%P=NzgHQhR_B3!UAnUk)tQ$1nGK*klthQBv-->H z5`BksGgK)P_3(vBN4KF~zoevwmR9XuEj>Wi%h7G~iV9$PJik0oi2fqvQ~0ldmvcBW z3>f+@nGX6$4#kNu84f3Kg3hu7F>FI1FEt3&Qo`@YN0oZFs8MoTdTzjfCT{>KTWBwU8aNa6cf(zA0IR zui$?0=tsZv)5SmD9vmFJI{EhL*6s)O?bN#;SU50K1uk(vuLM^UG&U2eC zd|+OPg7cbP4@{2&0)dz=l8q1g`87TTm|S$-hN$_i$r!leSi^xUp%dI>GN|eqZhn zGopSf-|NIfd~g$;Ydy`)j(o)x@gYxZ>g?;*U8+=hVJ9Af3C~jPSar;n5oh3%5r_5k zS;`AN(Jkv9lDb9}>xOhE)MuTIT;J>=^YAeB)770o@}X( z&qqzk_OnVe@)vT{)Tl9R8&|c$jcG{W_QX2oqaJCYFGEdi{#>?^W68hc; z)4|E<{4KPLg|5?;0D#mZzbnCA=;!!p#UFIr)eKNwY+7hR#x@_#&I9Z$XOpwtH>Wy9 zSn;G@pLw3T_(O3?=ayDA%?d<@s>Z2yBkKL z>zFL2qyXCAVJyjY5Id*Sr2pSX1)Hzs+FcNe{F&}NRp|29S$~8}1mCN++aSaV+VXOC zjY?^OZPYo=+1*PTI)+evjwGs(geK}bkU0As$yv`%==|N06s4EWSF~i?e{2L#v1I}R z+$w~vQL+l6Of>Y>aM!%9@wbPigS#{^rb8KySCRB7=GMTgA-LS_0JHmjfg@heqr39? z|5FG>7|#psl5z{{F6M)zb0OI`C@dR(GKACa@qO=K4925m9`nn&-{~kU_^#`G<@_J8 z5FM2G)+A7(L5lGe>KFb=x~#q^OAt=kUVw1y>}>HNj!>m7{tm)@PPP|7Rg?J72J{6?zmN@#2+*hBy?*%S z_mh*GfN)oR+SU-RvaxG7s1Qf{*6oFG8<}5Jgr1S)Qz^JDHz>t}nyJt-yRX-z;x?{O zt4%O6#S|jMRsdB-%a=W^aD=UvvauFMn6R-+?u08DVIFrb&R)Lw$B)ks9)I)f^!Kfe zF!k*e$E2CLz*0G7_-%<1#;UuS8rntSl*_rd6BrJFnU&r6e^?+m5P$9=fLDRboKzCKw4~#gkhFv)oOv z=?Nl|nzSn?81$vu88hFZ8d2D4+E(Ixjp2xsmR`>e^}|n?i61O_-+bT>Rc_~#*r7Jx zcsAb4Hr#hM-^%{@qcVX|phGbc1+^*>NM6#?>zP33KXwlezMVaM^??(p+|DO4fi^#n zHlJ!6&!7$GWX%L>V~L6IGO)SCvi&J9y`Bja{TLn`JZ{au{=f-TZs(JjK$}mC4d3?W z8_N|jQU>I&Y75s6XR9It=BKxU#lVKgtc~1*JJZf>t-;gypdw6K5=^DkTAdp*8GHbQ zZ8UaUcQTSa2|g1UDKiM97M)`Y&K5#pwOZ<2mV?`(E|a(Ht5&N-w=3oCRTsk_|3c`? z6HuB82IX0)GI?43-bzt$w+MvC;BFBtW*_GwsE7)=G-;hB8Pdy3t+Qn7NnVx_bD!p# zgg(pifr?hG+|C`++ppD|Z7eBlDyFc7rW~bd6$qy5@#1lh@|OK*kky-JXKcI z`d}Y>l7CA6AYa&i3SNbbb}28FZWKQDZ%!wp&gAmZzVm1@{&Mb|4BjPVl0RQ`&s1Ld zX*~aO=8)dUxj2pI&Lz%R&g6VB9uUzw={uvu8O(Ma2w)uC^vO|Ao!M!^Et7tK&>i5n zPda;Y-9Ji3lP--7x=uGHFIg|1C(d*MrO)`piBH6n$QOG4J8!Qm=42qsFKV>0V;Ao{ zy~JpyTA|LQ(;)!*>!#dD|MW_R+6vWA>v*n^D;REC58`+6bO0sSH_3Dqk1r`z z8)sd3{K0e%z0$7pTHJ(AlBt7xRGj_gJmbCZIxmt7=Xvt`pgVD1Fdg0{yUwfA$$WA$ zjn7Y$DSu4|j=k_Yq#*Y0uN62|C~$9YqTJ?uE~Ji+&3I+z#EIMDQz}m9 zFS~PbN_E|Rmn2g~w74t8SXcE}x@q2~_QB)uv2?74+bK_^dgTPI!pwfMcQO!KiB%2P z2y`1IfL+u6`g~Td+X^3qM(?uJtDVWT$2pMET0MC5=;6VmntG)at%rvX9~!lkGi=sI zp#B-9#Ks&O6k2MSGiIY~v}(t`BA8ei%lC8=vWuB!N z;m=0}8VXcH+@8IA-h2G=GI&epR-uF&4RO;usY#}2C?`HDFL19JMs6rUQUx%`t}%is zG7$K;n}ail6WTc#&j(*0#@o;je$@r2VJ{NQv?W1L#{oCk?>hIgDZpKxg~>`fCu1%Jn5 zwx{82kFH#=S2D~=k}hI8NXET9HY{jL4hj3D$6{R*Tjsl=TSL%Dc1&ETYfkz@qUrny zl14~j_M>9;s;nJXJ?T?H&iTi|WB<`(qc!WCC#(!MSfljD&UxH|v4sV= zLI)oINLfza%3mJ53OY~UU%wyp)t|QQ`JZia0stD3@ar^c^WUUPr9=+-v8AhA=Kh>p z7nPT@xc>D?=Xd;#VfCz+$pU%Mu8Lj&b z`he3t=MnKnC=5;}^Lflq+0Zk8Vzo4sEU6Q)E2F<@PbKm2={!D>{OteEAdB%}nkpQt zLAJYI7$_P^HYvI1U&vCWRP$Ui()-}AwDa2nMzPFgob%Ko-1ImVQjnHaX|u~`ilppX zDD1vVIxd~i-TAi>l4SW>o7iVF|LMWey^Ew{`2^H}ylkBHo^1uvRT{`uG{%S$2uZp| zJ)*0}u^5t7K0B*CbCU6`@XRp~(#9A|`2uA7))K&0j5_W${ta-UGQO)6l#S?v6_g|l zMXU@OrD%v&%7vPhN;e7x<+pJs8GkcC1l@O@UpmQUGIK@~6h4FTyUB1do}--_UZNt3 zyQdBmfX)8ta7vDfDQUOet5z{=ABy3P1f?Mv~@ z8BRnYIdh(fp3<2nqh!<(!f^RJUjnP-kB^07{$r9xV4(*T6OU?4?))0>J6$QG`1gUl z!PEF+=0FQNSCN>u1oG|R{CtocKO68?;vx5LEQhE`;GDzzI2rGWmip)(lE|JDx&N;- zg{XAAt3M_uZ0IrxuamAKz>%RWQ-qQZ%oZtX=_;T*Z8CCY)uE3VUsfHyCYJrX#n5pf zn)FNR08hBEuPRX7wCb>u>-{?936L!E#5pwe1!k{nw?=ohk>+r9^XnCdpJ-}L;tWHA zNqo=u8g2gHJi7Pd@*m%QPgaNP7bswvx#H8d6e#k2N^2B@wj{Y=VwD7kO!r#4{ISUs zS5zp5-hNORA52CJ&}7sWcIj5zixdZ8p=VMGMXOY{Q+=lYRcH7Twc?`(FMc@pCr=j7 z<&_R*=fn6?Nm0j!*b%xS6|CT)xYIc*2-_60*QiM%9z*GvUu1yOb6#OOM}aynmd^Ja z?@?5o?K`hQV~4yG)RNuFuJiDK;*zSVfQq#o4vz-2xHA|I=9kV7Lf$pQB6)EsX2#d& ziMBWX@6_TbhDO!ezFzzQ5O z%W6tkauj`aQw)+qUu~!|lD#-hh?v+|kN#^=6`1F$1tX5P0RahXC=Sfi@$^cxJ67&gk z2eQ#**w^k%;$DeOY(}aG;1lNSHk{N^vX@#8! z!?=6qT&R;385id;hMegBQ{IPj%HQ~($-6;Iqr8h`h@}c%YJ>5a(-}_qF3Kc08*#jk zPxeGnG)($)(H7yK=FEn2-m9_ZB4>M-5yCx5%!stB=$o>-Lk?Gt0 zmKp<*eI|U>%709(y zM^`;ZF=i=#t-6~jn+xDoRybQiz(zzJn>)11c;>0qa~gF0VHE=h23p)7py1FS%2hY? zlf4RAM|p9W@L0=A=EAyLI!u(#OlIZiM5E`srE~9GoFu^t%#JkSjk?00v%&jBF{C{JO?RoeJT>X1 zVxmlmE;Z?;L71s&FilQSNl%xtSbCaPU-4S%&naUl_4FrEjH~|1L9jwYxymXD44zH~y`?D<-)O2W?YWWG{WLNqk|zJ7o zWhG^ZiqY0l)>=ME5y;II!)Fr;#gdBha5`|lLpAF+gs9&mY@(6-L3`W8Ox(C9xq;gcIs8$!BwwWn>VHmvSR$|~u@WQ)C(t)*ms z{1mN{-oE%sH}mCfS|yupKWL)@XyMw*pjd_|;a#;$8Lt%o$i{mIcv432Pag1muBich zHp8~S98-B)Tx2<{_)nz5g%40N2Qh=CVA9nrS!%kIXKvS zuZ2yf)gY;Q&}_Iz_a279Mf3eP{hPo#SAE)+Sf|*SwC5FBQKL{ok181Fc879U@dfNR zc6>o4yufR@9-7gz64Xb8-9QJr*7Ap9HfwrYPyJiMp=q8`+n;-)Q!Hkxp>HcLGG#5D zq7FeZqfD-4%mK55QNn38IzJv^5ZEF24@Xtl4x>in!&7(?m*&I#RmRk*_kLc=mrXJC z{j+#XT0@8&u&46Rxj57RlOfg$+jLF6L&HG?n+B*3ROF1TPLE8n2z z#j#%Q^U1NY$jfwUkz$$%e8TXFI>p&k1@Vq671M^RwM*^V$gK2Is4LHv05YW=_u!8q za2P^(p6+?1u`s4IQG(@C?oZdwViZsL%YNE)G9L`!m${^5QyjQM9N4)tA44Is>zu{I zWaNx7;ZH6`G&WIDD%a_m?>8i!TUkNnR>OV89#zGNzAsS?WsHcsf4kzFj~9b@(IL8m z4{cfxrFHx0AYydbIut+43fp6NswRHJobDUomh-;xYq(Te5slTl}M5&x9)IeEm zD3{inu4`c1ze`(fbz@IAX2`mnflBec^l-`O!j_yRy+Kzz4F2ivdW{>fhYr?69yUHW z@(^@_HFTLhG%sax8urk$cs59|g_4MQI>QBwpH576WIXhQ)oeV0Ep0OC?O`|j`fKMA zkz`P3oKD7<&SAXhD&x?W2D#5xKh=P0AyVgYZnh?ew!X&8NhkE{#q@kSm?d1X(MAGR zU@09>`o{;~Qo1qX_~OlUF*<(+N7}1o0@2tbJ~>~!fX8pDN}MkM@X0xZAT;lM!Ff0w z^yT^2tF7`|RfaLI-KsV}SwLw~rB zCHlWaVZooCN{u=1GK&2ffsp&DUJlvf_&ou(*@cs2a-59g&X5G;`+5t%IogrCFhOyk zU!tw53{F&eB%>dHiBn+5@5l(N;y|#%n}vh{mFy9{upF_1(K0 zz&%%f+LpK{-={!3VUv7FEkF27!UP)uH_<2wi{jm`H@F#s3}c~&zxN< zJF0d_x}j57J4MHx$zpDMJ+ml!i|!l(JO$T!N%w4pKTML$QTCr=pZSEK`bR=)xYXt4{)-0>;RKyI599N}d@^*t9l#Uw za5BX&8-6f3aQB>VCqgTFU-jY9Ljv{|<6e9@bDCebo%50N_zlbt&i&`FX=~4U39p0m zzr}l>MBx(vTReo{Vdi}Cb}&smi6xMjzJu9p0zWOU3dYf7wmpxSKKd%rc@d0Cq8TVF z7KmFe8=oT1_&U`BQPfS82;~DEDg{yXS_k?YC_}~l1>FKHVt#8sD9leEC1vQt#$*4n z$%3^+w{SY2pDW+BP0UZ6B%K>%3|-Uw28xWscI%-@?v)+XCBo1{^^aAP9o1Pq`#?W< z{P0jri&-t}Ku6{KKrudKrL6dyAq-{rXhcyHt8!b?8(@FpH9YyT{rjun>;?|>D?e?^ z1Kmq^MbTcss@)_otem5|{PtxGzr(W4Uh5O=aYwzv%guvvHmIm7PDX(zlwi%MH_`e!Q*9?!K;U#EN6!WUNhfqz^m`=hhiJEWyw3k zSdQ^5V%krXM+&byHZ;sdKEy(73@|<#7o`eI1O#>Lej&vbgM_z63-XnT%b5^?W?=ao1ZD^PS8wDFR`ih z&`|IE_`0|ze~wX<_sB$oyg#D^{g0v%M;a`?t~N$gR!c<|IdH^ z^XqnVui?R9*xn6-J#y8MbLC&V<;!84Fo5)OggnGnDYche){Xp2@(#uG`E<}(Nayzd zEu6F63ExJpOr~5)6+W|LZEVzgYA$(vG#1FxL7%>-*YP_>T%p+GA*#_r37RTe1y$;gHf9{6({x5~z&LSN?Ovjp^j(z7r0^z$ zw3qlj=Rf~}a|G6r^WlOVSy_SW|K~qP<0mK1eJAVL-tn!7&|eVb=+6_4%T>|eUF2R#op{UagJ|2rrtgO> zpW_tu6W;7UkJ|m=dH1IqP$zEuv=R*asL+|-U+O@^mH0lRec#^?3(J9N$ZNK-*9L7& zb}b(&XN16f=R^}xE|tAV8}e@YH2YkAWjtR+wR#Ot)FdrL2xN8Rl~hW3CFyB~ZBCsf zP$TsE%JbA#niyz=n~DQ%=k;WiT!>lCfq;Fg`LQ^_Ix5fQQVSnv0Nh=Q&+FP2@#XaqUO4Gvjw_#1p6ER+6@}u}PQ%Mv`ymc9#UZ6u$=xAU*r7fT9fdC{ z(smo?2Bde%mN$F+TkCvuKDvQJstF18m7cb>j-s+{XdML!f)Hug4Q}TtxRLWNaj_-i zm(b>xlDmZMY$GvEUtSpb>^O)4bDXFwp^>B=q3rMDvn{p9F z_0)a#`*~oC0FX$MBC8}-)dEqMFm3r6fznd1V`lex!r9grnrsW2a z6SR0TV2c4cn2bZh#G9WHy3JotX-MWRJyTxo06dw&Y?f#ZeQmtKAqj=|7? z@Re%M%c z=_`?Oi2f3m9d76VJsq!wqK>>-hua_b-W<2P&*Gb~>@Iq>@6+(gO&2CQd^qCXFgI56 z`eg}=(9H;TyvTh05(5DITr_=9`V4J!;01%!rn^e1%tHzMz|!`eE`zlM6?=Vz^k#`< zXl(6}m8Q)a5FwX9E}}MZ@Y;~67BXVe1@d4(h>wh#{;~W##=7zBo`{B+OJ!SD9Oo%K#@(s@N*TPwd4>bfRP^UMT5AUCswzG-k`mAucYlKkUHQ5Oen|OEf{o2~_SCq$ra8(2|Y^ z3(09*pU6V3nCA6=|6RyJ7`76K+$Z|BfE(37Fw(V@hL2|bDCY2HaP=iw7f+!u)QmhR zf6c*l5%l?rX#|jOTh50n!3D=Y2_SJubR;1I_c@E6}SGFWpp$ACnKz{#38jA{w2FKAFU-X z(jPVaZ+4*vf4$MTK@<`_zbzKJ2?N9IWkZ#fGHqxi>$@;+=abPXRO^!@#f_fLB~2F- zi}bat^0j!re72Lc8%cT@C5D~mhEWMR9973lM6#1)9qM?RT}FLQbHF5NJyd#}G|7v~ z6XU2Op1NY1fg?$|qt2OJ-B)KO4n|b!ReO>arsq*-u+{m+5l8#gnGyFXzF=g}QO!zH z-z1veX=z34JG~9nI7jaq)s&<(mCa8|wJu1qlO*3tI@fO9QLRhvAZer4-X!Tw(wDx% zTQ3!b`O|+8eMi9OaMB^nWH9CD7Fu1ye;#g)Cy6}6;TG3Ns6$U^Gk8J8A4A6j5@2W( zbmB!guLxI{?9Tb?Q7z={$bG^s_qZpEEMROrK1vTCqu#nC$+byq-$MP24JftC+o!>p zx$I=t86_c;R2s(dk$9*7YjA=dNdKS9&*32Y`002YeEiG_Ox$}sk+hpJTAxe89<09l z*&`^%`VWhd48?#k*6&B%HK$*%_3;f#&6k|{xf*|p zfhd`kQsT|5XH{KsS`jW8fsFZBGjMN#v&gICbw2<0Y_YtV;LJ;2?TUbWb#_#MJp3yU z{XsQVl>qs)GfQsyKPNbIH6p*(|Nff16_1x=3jsi<1wOgs1@-=S3B+YdLFC0F-9S0I ztyC%#YpUagIkGXc;*FX01DRPwrbk4jBPSaoV@NcNZe~>qc^7gss~gLPq1$TV((idK ztmr;M;hJ&Nk$^YT3e&eO6xNark_o^xr?9*JAF+pcH7&L0v4$#YLs?Qn08BzZuGIkY zP;aeL_8bS3$FzZ5v6L#B5iBh=ECE&kjv6$9SAUD|udZqEo1<+v0}VpdEC#POsO& zh?InGaNyD9Bco&BvL9hBZW(X~T+wnV{hu$y{m7Ojupa?=ffd9s=-|)^Iu$8stE+zY z6sFczKYI$bN=l>F-=%5-&ZU`m6F5s%av%<7AY7V%my2;EDOFHnioU5Mt~!#Gdg{WC zFW7`>2ORvi`}X<0-TTkQM7XdOpst&satXw!082evcG5ym0dWZcnWnc+z{{kC2_Q-y zVo9qhoh7%EoHn_s1btLTlG{nJ$rM0T$MuB7p29fO4pgwCCsJ6zo2qK^wY2Y&wA`wX z5*%@5TPxp9z*QY?P0`8(woUp%T|PN(C-9n@70TtcNuzNpYKU>I#$cLVTHS4OSIMc;Qgxvuae6vt6#|@Qgim@_4T==1PCh4Z zi7HKU!Kd|;TT6~>PdaUU(p1%wAoM}L$X!N|%k(H1d^vgK--Nw<*{faAUand6Qy4EG z0N@7?Uv?lmtfkhHCN!BzbZ)IHan38KQy8YJ*-LB5L&{>dK}(3Fk}XS(J*|~u*A(<5 z#3McbvgDyx&Vzf?H^1%fI>&MAD;TbI=P7%;8qE8oYK7ryYy~dHa4n2JgFsQ zg&R1P!E7#_Krmt#EpKMuvchDN+rki-lmKcYegkZ_6u*IRYgo6;cn;tF`gMkeUgQ}z zX=EfBiVe(YgF|g_s0Dd&6QF5>Lv3&78%M(Yk7|r4!8B(waZ>@gF{_jfqsKS?RD$$ zqdW?3{m4~Sk^87UPyHU!*#2j3<_!*23Eumh zg$4|z{=RyMIGzS(%M}`}Kgs)dGUguv~v9JqZwm>7Jf z)lqx)R|}eY?U3lamZ8L@f_;;yy{wd*)uQ$e$lgZY-UFn^$BQ`(h4jEZ4)o8{!}m@* zqv>EWCdg>&c!-uFTbdCAdZEhqyH7Tsp~e^Jg;MRsFz%3O<*c*loswInZ+)5&Zv?w1 zWoh(9{!4<1{vjELC9~l1ay&-jcVT@R^kdSpWU1I(t9Lq)R4TlL=f1Xe?z!m^zzbz~R9@VhyO{V9w!O1DVogEHsEvnw2NJd{<=i)#lQFJldn zBeZja^0Gf0SR@W3qo750dZb;0g7WD|M9(*;B=?sLlXLF2gFVq9&QOso)paR{O{H_P zdPYj@N-bGP2&NoPK=!vPrHDz?q{P0CFgZ1qMM1=j=C>j;N4Ig&KVH7K&u>T9Pq4ZC z)vgL`RHtYm*!Y1@G`?TUNdH0NAu<3P`(`%5M2l-&VHbDYZ0wsY@+W896eBlrvdpOD$wm3aa|h>Iq6#^a zl}e>p{vM1H!CI_tJe-`}5hUmDz2TtuAwg-_MXURfz3@EFHo^apYWs0K1hnCxl*xA3 z*zF=3b2$F#;NHPK$NPUR>!$=zU?I+2Dr!SP2#9FA{x@p;b^vm?g434EPOi7@xI#gF zwr_c4EGh(vljayzM2j6!d%afP)Z#==J5ntamB-cSCWV>=1$P>ee0~{t6Jg3Qv`cQ zE4=i^T+8!n#NPd{#@>Cv57MRx-Cy63=E=m;QYM*n?w4d@rwP_QnSi_R{rdd&t9zD1 zz+f-v68+chAKTXJz1>%PFRfSi9`45EwH2TFg0T_ade~uQIPJXcK!o=LGkSUJ;dwuv zb-JDYE$h*^Kj`pnHNhr!jj3qz$l5(n z^qh34>PpCWg0O62Ju@8n`^u+oM*&KyWbd(&*Kk?^q4(SGpB@~&d;aXj4M5SCzS|A(&6t;uuV^x=IuEAw<2L0vAwMTnLZwU=v3DIQmIUJ z2@w5ltnd1P%CYUJ-O^LdW{2qM z)BwQd>qSIFbYs!F=!d=N|I?OB0!H$svt^ozoWe8-3d52j%?Yp8bp$R%lBF&3#YCpL zRuuhTN8HfK&Z~fpN_w-#NlnXv6FFVaI-X#LN-Hs8`aKEm$UaLPL-twJV3s``cl>lj z+d}U27RS0?k7FH`DYwUjD#tpp#D(;th*S1jEwoXRHtXM&ulgbhqAHx8Ph(3Q*lhl! zcCJ^eeKl@zp)UFk9HUucx%Ap$Rv~>wwIVw!lB+vC@rhbc6=v3HjAtL;|Kb#SN<5pP zeVeFY&u>}7!3XFp;3uS`yQNqFS#7@Ejrp&$Tnxa2)JgkEQxY>t{O8wd8bvtFDz;JJ zVlSDtkvRXW#u0hg1IZ6nrUjr8?n}B@!#onj%iZOPg*>(mD*!o+R<0dqi_LdgV!%zC zM-{iV7C4aO+q#D9MSyK&Te~NBP6o%^aOQfiJ8unH`5I2YKHgz3a_;&2TH`W*|M$%c zaFI4++Z8w-f)ucwkiS7w_jo>;4~Exoj9va}S2V`5wL(m=HXF;(QyYzvt(B**FCG8Z zY_Ligbjo392Ft$NE(pg}4qCM-cG)=$3mf34kNcR7L}wdf3n^R#BGQWGs;>(w;w z4WLfGAQu^srPSTAqe@lil|e}#DmWPy3N0$eWOVfoaEveud<@k-iyZbVHfAST2C3WX1TT6*vNo^M=RV#WWM%{}+>4gnu!0D>*w9Uwc^hz$y z(AR@HLV0c|n?Sp&Dw`Noe#aHgAyp$EtHR8*huu-1zfS~e&8nA?EGds64S5qH^k#&q zyoQyLtLvQnMMv20C9fP-tr7m9R5^_B2T$an%6>V;NLxF3uJWxJ+wY4_2o={@I?T1n zK%486CvQOZl@rLiq5z&^_I0k?w|2$28?vJCoK~3+jb2q|qLPXVN6Jr+Lu;R1&B9mq zn-#ARU4IRGdvU8}7N9nxWrvJ~5(k?9Cpgf8uXdn$I6eqp309<>%1XIe&4G3>dj?J6 z!SdXRjsk3^w}Z*BW8Lq35}to%A}t`}cg_dnWSf4T*iA71@8JA89KRx@fp8G(Wbi4r ze*PpU=&SFuozHgfNzGrDm{T+}tU6{=x zlBkXA>`?LY><&L$Z$ye~L_kOZ*~B6cQouLT`ydfTex=Uvv&>=Q^U^S|cM@FmbTYSg zA(;&0Ii=+VaoOvBxHFka>Kha1dKhVVG603a?cLeykWBvT*-rXlhuj3Js zA69R2`*^Zsv7PN$;5dUZAtr(q@nxVOv68^2Ci3|-9!{n3ZOrX;#&hdGty4xM?V8`g z&?kuzF;|pJ$O%Ju9LIfYJ{b36>u>G$9qYmD{P>ugoXqaf*lUH1+BRuf%kH!`8_zOk zs@PP>IhK9L27+ehue`7ggMZoq+e`Kw(KGs>n>Fe3zL zUDQEUtXT;g`d%=j%OqMNUx^oiJm+(WD`PedAc^b{e{JZ~BW-l_*qP?!4A~-&<^o-E`jZ&{Bq9B7o#y|5ezD4pPN`cLcNVAp!T9Df)PdV1U$FFNycER@M? zF~=!I;Px#GgSKx?C;cAZ#V^GP#$vO;92+de?tMGx&XmIpy4bUL)*lT{CN%h3?Fk3< zXpH+3KgU-%V95!UZp}&=9NX;{JMOYQ0VN^8B&oQdRQjS2FVqes9l!A5J2Dqs)&-Rg z`kSNZR46akVo-eJDI9-P%`Ip$KF)s0U`Cidp6l(=!Q|}q?0)YiTu_(2+7-=-d^S}+ zDC~0I^W;^k*c0Cc4d_A`5;?F-D4-*W8A~`)8@Q|rCAoMYnU<~tPAV`7x;tI8G;4%| zV%oSZrzN`*$?TL{G{r%w#a@`04*2lw%R;Y=64f|tT5Ufd5`rB%X@Kl9G*d1$E)l`T zS810zeKxzYfDK{fvl%}cjE@Pm`S;cc)MYXF(&-Kf^-vQm*j`el*pJucOvO!jv^YCb z$_q!zi6K7TR=GL&K$hgG9-SO+w_onx{ru#E+kLcn^K^7?aCuMAwOzBgwIpaa@fsNr zQs$UTs#d&4M%{~n#P>QgSSfeG7y1jsZzO$&!%s3)U?@3kd{nQ}pFGS3L>l(6)6>a9 zjDh*!iwqtf9FK81l22z_JBZ3CACLNNI&L__>2lOfD>^|^QT@&<_GXi*V7j_DsPU9_ zt|;Y_%1?PbK!1u;CP-W<3tIVxtJR7uXyv-n2P%iyQ69n^kadAQ0~V%PlAa_h-uj(D zB9zB5yZUkeNS_qcfajG=Lzt_>SbLjm2KnZlBN}4ed;a~sbLuVR*(P&bHe+_?Pz4r8 z%2H>@IJcbtVf|eumiy5NQrdtQ6bK{4&i+JH0DA$M?*F~j9bn2Yc$*`Sm75o_HyJY+ znMad%Jbmj&Q<0b`cb6)>gVShqlIZg2lVQWu%l8hqv4B4FKa}Y9gE06sJw14Qe{}v} z@cs@L`!CYj{}Y2;V)61O`hRp2x!z~~KinS9cV8dex2Mtj^WDjp?!4T@Qm3)eBP3F7 zn+~@>-yTillQ%EL2XI{-I7RxX^XtRq(0}-GF%Az#i(gOfjJ#!3+i!4?1)ci?pKXhd z9lo){JJ)1~ zZ(h0;8}9G#rBk-=#t{Ew_6i$A{EC~bjUk@kGKp`9Ok+0(;@qqu4rI1cAR02n@BioD zrlU&_Rt;6&VIm4-7YK32aC*Cm1awP4KD9~I`zRHu@Ul|oJB?#EI;UG_LZRC!s8_u> z`03T_-Ip((?*7KnZWQz+^K+*c6VKQ6V~N@8xqK!hkC3*m-|tbahpSG0(HMTJAiPeG z#!^9r{_zzN2nZw#e9u)P7;aBa=*CGbp_0r+TtGyG9!!S)czo}4FqF`_np1b%Ep^eJ zs1fuCJdkSak>yts0N4p63{bT&YlkCAtGiuKA_Kc*v6u2=zAsIy5bxnuR-uhOOzt=SD-k^$j|w@5DTUE`w{W7+2Je+(Iet||q z_zmg+Iu$36JMrI8@#=KXt=lLRwB0<;w~mLPMyWJ2kX!GncdzeulZMMJ_S&RpYkJAm zog_8e>aCP_p&v?=Z!hd8y_XcLS2K`9-FAmVUArT513j0NcjVoAH-VNhqJMg!w4&SY z_9V1-{2%`zrH>@?mRQL-N>;Z~xhI}~Yd$KBySPm|_W2=AhIG&w$=$J9>7fh;>mU+~Cf2hhXAPa)1Z zAg7y-z-{f{zk63SB{Cl0kHUU1*@=>+HW$loJNMoFUG1J%a4BUqujO1CT*^ul78Cu(ki02F zGBFb?;VS#_!$Iqf^Lqask0aw6sOh4R-4r3k%wxfVa;OrZu)5CT<9{A*6$&AvNf+%| z@eSsQ(zmQxJfzZiay%I#esP$5JAR$ivEHGawL6_Ar^LUdK8W?gTsg*@R;)MQLG(&H zg6HL+_U5{F%MW2vXl7-#kh@IH%WUsNCY+wz5gW-CyS^XTVt7Te36TxMKlp}e^%c7@ ziZp|dZ>b$eKM3Ru>?1JL`eB}DrOw>T8eu&rhD2h}iF0afjuazk$FENBSoSe)U&)pz z6S3D)f-iU*RJlmG^dd*ewupF3Voj=T6p=Um*<{e)QDak4d`>6lcRtJxx0kbraoZX^ ze^pZ!-~Z+13plIjsq`%j+7nW&j1vBYj2?n#h7weh`LxGk%_x&%f!AE4E@LbXNC8v0 z(vpEnXijF5c`eVYQdJ!no`=a)b>x9pO*L0BLX;4GoSzg$K)zp@Wp}4DQIeCO7k9_? z@>%vuN^&>OQ>|DMKZ>Q6_FQIw8m8L*e%N+?M6DDL@>fJgD1cXJ@=QA0!Q^!B(~HNa zH)5+@@@iMK)vD93*leM12d>}#j%~J8l+~P8Rx2^SF|@u7#ID;zZ|$cH#p`-~0XkD; zuOIqg9=%A`Y#>mD0QUN79q%0rmYAY~O_qW}5NZT@!{U9htJ}B0W+JiR3)HF>7->8*!+gWNhPkoXeFbfgb|m%!8Obd%f?<$A;>Zw&2OL( zj)h}hBf*@D(OUWj*KkYsQ69_YH-M@j8k^TBkMzy>e%t4IzVUSFxBM+B z)rBb&7{fDphs;_5V>m#fym%7n8&;~;CW)vq472yotQV4Ipk>r6-cGY7ayt%hUURB} zGH1zrNz%QCc2F{LHo>{F{}A#Ng4RxpO}-Dd)3%wmKq7Payjp0q9O1yWbAe!kh}&b4 zMz<}1l`gdM{0d6D$1F82;k3yV;do3-7l|KGbtL<@{p9#`{N}gaUE6K-FCP=oc%FKy zL+$*URITv*jk*`(`4^i*gXJHgQmcbB*Q^dD{ZPaMq?%8O_>)P>XZNIuK9PkK9RTE^ zFg^ji%T;xvKhU?TSx5AjH!%zCO-B%t>kiRLx~7CSg0j-<8bKf5M7z81rhE6las-vn z^A{RH7kt*28b22vmA}`Fpui`eLsg3=o1xOu>l#5L=Agat@ylmlIfBaP`3sGp3qS2k zjiC#U%kl_9p0kA`q~#)VSY5^R((4&Plf^sHIQ&=qb=ljo?arM9RD_K2ZI*y9ZqYOabmGc(wC zHz<1<3KB$43H#c3;t!KQOQbo^jau(~YTa^6u7D=!+jt$2{V`JpYZ$sw5%+*7>lEIB z%w1H(0~qxQH^5Kh>fbQ^PYxPHj$^PCGb(-M2Q&t;BJS_Xji7Fek?r z@}JK-qnMbIM1NC`J)N8kdK|gS$oar9%WGV7tM)JNac+>^AkwjZ{+(y#CIW5VO z>ysr{3qp}tkIQOs-gFSkey|@I0U0Z(jWj)D3D(uf6S~#cTQ9QxW2$1>E+oS?(Whif zZM8_s2-)!7pc}v6U!Gh)+3@mLyCT`JI*ke0&~62`jmMoKPznUeVyQ}pzq|MlN%HuZ zJlK%ET|&%kJqNZCcntHvXXR&p2-++G+6o{p`BEwUm0AFQ!lJ(-Tq6G0>yy#cdG<^e zl_nz~(Q`aIJG(;^!zpo#y>Wat@0`Tr-nl{vS4BqbE*DJwXK}ok{}@pbo=ZGLThb%t z=3aJ)K1>(+dubf=Xe1hs5pYv-A`RJ zmq!-#U}cdhot}!wwEj0~k{E}OQk1g5+#;8(=y~U@zIVEu^-j^-o=rx@(f8CAyLG-G zd&mD`x?&D}LUdKkhoAS}y_S2WAy-JApo5BlZ5&Ot)ZLm0(34Ay>j;%h4 z6vmSUIh;rUW%W8U4q*$if|HNInzDF(X7(5L0g&bn(UFK>$8%y3Y`bGu`S!~P?;k$> z*Tc{6<$(|8qOx+NUale*lc*JdvvrON6a3(81!fHiIxuj!Rp|#`>8)7_y02bh54XkQ zIowuvj&Q(jV4K^0ZvVD|Qf_l&o%!w43E8t2o#AcCnojzw{_WmmhCcYG!E8Cdog+23 zB`J?k)U$3$R{9mTRr$&}Y*TcD3xp|ssQ}3mbWaD*&qv*f@<~)(OfJiy;bM}c=Or0< zM@Xz0te)+mReazd@#qXIb>*XLc6Zd;<7V$SpP;7GH<=@Ir$oaW8hE!tVhRKwPuybi zsWe2(m~>$pHyhM~#APc?y+K~QBI0Ve0fd>fo!Go95$4eSdW0Fxc+alUA4p`cvQlYA zf-t|BTQSM|2BhnoFvrK%`NSFmk58=5sm#?Z0dpy2ddKP){EjW!q(8B`Xt&(5B%j}M z+#%b}aKYLU%SK@uQ)fJ|p6%Y>y(MSQPe2WzX?2|yo6Lvt%V8&O&rU`(v1ya3OjP47 z>;o4LCn)G_UvCP@WV>O&y$bY0U2O|Z8GNhKx3(Qd4yr`vJJr9T z#=APV=aI*Pka__Nv{R-PvJ>XDO6M^*kc)UkO!#s<6)l1zs>)e1fZEB=p}iQzL29S3 zmV$PPVej`vxD1IR^*NNE*=KZ-vd=-NeOB@DX(cb>lxx~o>~m8I~(B@6TCxkTw9$yAuxX#gDqw80)ZYe^FI^|td&k)xi$g1RSVdD-?RX8l>+0L4 zl*pi&nOV;zn^wbMyBN)kU}Q9yF0fZEHl5gI=_*|1ay1b^R0`|{^W6Vx^W3+6Hwaq3 z?h(jMNH;GP+aod0Ur%6QSdWl%gp(l&Dg}%ebw|A7r(o>h@yFJe3DElpw+Y5LYz=Ch zP0sM_^S#c<0%x5Jmy39K4z`0=UC;(9%swcRhhX_!0u}&{_tW^?LR4slZ5}^8arOrf zPiF_T?>n%~6T4BjoD%~omqo5t$ZhYHMTym2EjFy#q*%AGvERJ56Y9@BJ49!u*7a{O zwi}q5uzBvw9CZ)O?2Fno8mHfSPiEFx%vLHMF8F$3DsR0y?~mhi=pXT!^=h)@Nxvi4 zdY;-nX_#peKl&SE#F|fz7iS8@6Kd3CN)yyTVU|TxRYqBN(m`(O1b&EQZl8Pt&LW>G zgAK_n$!m9q4dUVU>0l&31Oc3k*>B+fdCfpQIVA@xbLkyRt^vR(dH%DLMP_Qbi68N8WcT{6Hcos-ow4Dd=V!vL?8)*ImI zgU%XsB@5gRoDgj%fCwLbXOHmg!|mwyuY=dor$^UsfnWS;C09+pK9`RF7%rSYWzEJi zKmht=GU?}*z%8Zwd_I{R-?ARbn(nkmAbqpAe}{pe8D?sz7ETrf!hR5w)-YGT)}+-< zv#jU73zK#Y;v9apN$a)94vgY$nbDMP1+r3E%?nQR3t<_@d9(NMX=Fn9KxB zhEFxe9#zID*1%{wjM=iyk$6YKT-YZVux#frXNPlZDfDMX{$L~q3OQEUB`W8nm?iI6 zg*l3)i#Yrg>GwHHGwc&hj;=FJFXVe$ElUOJ6^(a0;WCd45jA{ zo>OPMoS%N^pu|X#X4gK%hYZu{5qkEGoaVa^cOSfd#CNaNbd1FBS>hEr0==51W4b3d ziNa(*7bCNb-9|y+vKy$_j`!o9l_RhVJZrNiHv zv6y2`OvMruP?3_?sKSoF!yE2|`RlDyg^3eU)F;x|I#GzCTfrU$D=L@DT$e1Yf5s}h z0xJ6Qbhvvkd;ad1S2XWiSz8+&xiyH6>A9^|xUr?xArR6H$fk~dCfb!Y!2n z>EfCiR@N&lE8%3CtE*vlQK~JlFDEFMekiT51sU{x=cKbZSt1BcY^-zz^uIc4hCNhcdARbsp*nBjo~C4&a(FAIesa19Q4;<&cdDgluMcj%BxYytY0KI^tHltcQAI z<&;3H@62l)v|uuOE!$>8A<;t4;r7$h>G4VQ_y#a5vG-&_94?Mh8}U0abFSt-{xfV7wZk7w7fU#4__tmu4B@3(zpTdt=AcukLhl7t5D;N9 z7Dyn$8IlkekOML*3fh3KH%ztQ-rM<9mdzFZbPFn!*sM6s4>8UmfeO5X~Y3S2Lty%h;eE2ZW|ya-VX>AgQHy?doS zhzT#@9*$0c+(-8o&z`+HJvkdb>jtM!E-yxjMSrw7J5pJfj}{ zENF9yC|vDeOsZDomyNm?!%?3~sK688xS3>M)g_qK9e7cQVMbUM#sP6F#2KJdCt>W1 z^C@}vwl0=yI=m&}0DUY{@jf1nNPswC)1}x>x3)fYh<+b05^0_807GfPsa@o>MVDFJ zjBq8g3^{2NMpLn=$~~qEMQL$#lF$tGXVdUh_}<~RAK9O|)5Gnv*Pl*5`zN1C&t6Cqr@tL|kDNy@?Ki_u55N4YS8igd z(?~E$Xs7D@!|l(vN7MM^&C93P*n!jG;r5631NZ*^S@(JPoA<o0pZ-p*%*PxgN0NyWJ?=s>SACG z6{Wo@jxxmvQVwzDt*BN@@ZGI+4$$8qdOSU6z99vX6*L7nrCgStrl_!WalzHf?4xf= zt8)|WDpnlrfx=-SHRIKtf`7$>xM+N)G#E7 z%)1Hgw{?+u80r7#-=N*XMi3jbaPi13p*-Q+qy=I_gJsrg+d9v)S|~QM1U1T6eJAn& zNhS4pdMq{3@&Xwx^`vAX{h~11`dyk~P0z@7M(iQTH?@k!bLaZ8qtS zv6a_Z$!RIgPCZRIn`pkA#{FB?dm&<=qXqZq-0BKrxsNHGYK4HWAnEcUrHVoWolg`e zA4ElRdnB zdl%EF`KxE`B_^$fXIUSu*&!Zw`VODQ117j`to%LZTmJ+w?A~eo>HM7PnllOMfDX|b z>OUNz>7IHO`S`5E3C=C}!@wMbZKS+oB;b@N-EXn=_eVr(a5klN*d0tn0uee}posC% zgPwH8=kfIuhCR2M`<553><|&}Tn%NId7zH>;dqk6$#`8~pDvY0#0`b_;esflJO8 zJKA$=1!oM#@U9oR2LD*mFng2XgrM9ldv||75?5H(zbp+m6tWk#BO#ZNu-hioI3#3u zyZ7S3ll}AV4XDw%m%iE+oiEueCLR|!CBPwJi|#BjNOE(nbo^UmmR!x{^5-wo7Q!&- zlqwoBE|#$ciGZ0qZ?;ZkXX4Z?Oh7LsdzoZ(7Ey^t8D*iJ@>6_ zvexXRR++|c7dXCeutFo`IS489c;Wgu}frUWY zI-Ss79Hd86yZBk>eOx@K*r{q-^l*c95yJWsyCpjF0n^uVj>hr4(K=?IZvCgV-@&(a zUW}F4hZ?1i71~O=YqepLeUa09HeoN-nL>7lca9SynRuCAnLg*ljt(U{51kgpe|P$~ ztw%V%JEvCQ?>MqfUMsE0C!cbeJnglxtW8=`3R#)tX_D;!H zgfJer&)iq1?)jVE4KTGYeYLBa(fM2`hIAXfNmkYGSdwuqMO&hGGvG|^#63UXi5tKf zq1-{s@knfnnqHtU6=EKgN@Ye(DB9ntE%4Wka)K}tL2WgSf$SL1HMIpEWcHr*W(-13 zfeDluCLyH-R+g+p1zb{yXTwfog$5E}0-Yz!6Ou@vGv@Lu6y@Escu0g?e0wpu&GzcH z?48bQG_Rf#s}E`IxI-MAqyy>3o#g_J07wpEfZReBB9)Dpi$KcU+WCj|=)sP~O{u%F z_V21c5{p(H$|a*;jXTq#<~u+-8SGa{(vU2-71=BRbq9F5VnIL^En5(NTadLX5*NCp ztF9Umyr&^Xhhanr61N|BE!P)rvL6{ngfNsY%T2y%Bcej}{ZWjF1|$xSuzy!9x-~XL zrEfA@!>}Q~5%H|B<)R=c86f~yTP`_b4!6;+p6s2xaIW2gxctSgYCvRDrPvPm+w z!XPP~xP(`fO1W%mkh~pCkiEK(<}fa{fpV)kZD2U$mqF^3C+1x3DckHC??^1n_!Ucz zfOZO*`}vccps&6!vNgwK)QV?_zDUhqmT;!;SQRSt-7mkMe*DyZ^Iq738+_t&$)reH z82y=AB##Fpe6jmv-xyD<+t%G7l0nu}z|=XeCh@yU*R%E&DJBkP&q&bl zV0rFDM}ch73L|Cf^$fNrwlQNuxbm;2ptC>=Jz4FQ?NaC4dzd98OOi>Er(!Z1^uTQ= zID-)l!WilFC%3G}abMW@q+BAx8?X%d@pE90gM(c>zvB;s7*ho*Z)XlaKQU0yfE!H_ zAA({FY~1gO^H`w?sJY#-UQ7};hV>ToMe$ST)=$skS${M*Q7PW8VxFYC#EN|aUBRh( z3H>Olm?vVeRE-nK3|5111muFl(P$bcsPjtT6k($_Kn=O1G=xKTWtTNh(tEl&KVfAB z{!y%x5@ujP96x5-DwoU_;2Uk^3_?hgUc~-mbTB!4J-dGs2*;(bc0~gvpG_6O0rTGX zyjp-Gu}4dG%lG0aG}wd4wUpE)9Z7nVVk4QDWShv1WWLfRCK3B+GcBzk5;l6!%ChZ+ z;lRw5E?Sy3!a<=&G3;81U`H()lY3x_gHj7gke);U{5L6mB;~B>s;h<>5kc<jEPn*H7)^7%W-V3@JB@%Rhpwk{Ip;0-~IgLgWG+yc=L30Z*W~vMpw9I z8NWza?1r6}xnz^7l_(>l?!_FM_c}9DPTplT?~`6KG0J(3lShaaF+pddOyo#zoJx$* z^mMWiV;~P7F=9OsFjY|@>UTLR~pDl|D#9QgTjWU`Mi5!NN34(vxJxmk1rH45`gKCnK_m?5lLp`uF_%d*{?!$aqt~ zj|Su8iR^bfI3jzCBW0;Gi<2yy`Lz<(=X>c1s!g`Z$Aiw zPt((b$M;9)4+ihcF>`rXpMMi!eN9c|dY}3KaCo<{G_cPC%E^Kuh!EUa%u z>m+QX^XtRq(0}-GF%Az#i(gO7us%H;(m@4>f7l9LXUOJ?rm1HO5ciq?54V2_=kNCB zFQ1%$c=8Z8{uiU;jL^f)+HtkDBa2bN&`WnzxrZ)4@>Nk52Do;FJ7okVWBfs;o%-qX zi+%TFIoR$o-4Z6BPSD|H*`m9#NX~ou4=|-(UbnCY&24cEFR~rD^4Z3Q)k?d>> zr^&{nQSU|h{@%F_y7~u#gd21f;9!*?-ijq|gRY8FKRl z{hz4y*PyG)0Xs$Er$;6`HW%ve>qx^1f~p{_21T86$iT`-14Wa_c($5fH4j}d*84w6 zKgap{P+T17hzyETT&HWKADm`jGsV^tDprfP)J+wyel{@BaPHFAtwDTW1xz?8ijxhC90|SN5+P zzvD4vAfD?SpX?C*nZ8r|cdP*qxk`yWW{@BZk`!rlSRN!n;qp`E0Dv>5eI}gSmc)Bf zDvzAfTQZV}gDsBvl4`(DeyY?0zJE-EDpqHbJ}Qa+c|PoQV9dhO&dDy^3xeY9&?R6Q z9S`!sa}qNr!vij4ABTkcrbFEcj%DzVtXI4D_wMZx4#Y_AO6CFvC(@{Qte3BEQv*fp z&Nv()J8(4P-U#x?89FK23-F@`r*5UX; z=L`H^$;Eu!v7W@Z;H=%(yRZ4W-S548@;CXwi^nU)wj@?BUS6oKVgI&g-FtKJ@XgEY z&({O%ZD+x-R?QC5cHQCTtFcG3_+-g#QAWJT0;+)o_omRzCpzL7-KzXRh2KkfALO`F z?kAsIuI1<>sfCNpj9-ROD1pk!5);p}b;NhuLb)^8y{|eXf1SeaCHWIfe}Bhy^3cO8 zi2$XycysLDO7v|VPMjjd7gqs-SK2+fGX>_m8320C`4b(WMg;nYOdT4LmLOnQOM(LC zww55!&pr)j{`;f5*B*a#`HNkV-kZ(!&;otpv%R+M)n;BupZ7OT@6E{(xy@SA{nIl{ zMPwdE?>wJFjPPaKZF^Bf1asRC?LgOmQ$b2*sm!WL`0{sbLxsQ2+6(95!AtHY154$u z;GK<6fc^iQya~1l%CEJt&d$#6%qP78DcA{iQcLry?2oThwSB6|8i%;cwyamSmjuZ! z$2cgowMTbw=i9@~AyHen%E^6W?s1fGS^*qz%?8g7(XXjFb@%Q)$v-}~)GB`5n<*1G zZpmF6?)RG#eDjcCunmcp;0w1M#M8 zQUDE@f+#*3dv!KNyT+g(LLHILl0|WV{)@y5*{-A_cOrc@xROPY+|tb%6#FgtH|bZl zNma?a?^RZ?$(;BhqnQcWcUpwDMeIlz@zC%IXP!RZefHw`==9pni7Q;~iuOb{*{N`5 zA7cWoUo{bxGC6%8_QZu8+CmaE)W&^g{}RJcLKQQ#G27!{LtcPgxQd5az#(*R#>3QJ z|3oH(1>ufthrRgu`0SH<5~=WCvXQQqYmTo_E%*{HREFZj3OFBiCz<%Q(0e(_g zNx6j-5(t5er)%rWVNEG_P+XSqp<853<`U|x+AXOq(iDEShODbPhueGI&wl&K;@A1^ z-stS%$>jrm8qZS!9avruw?EcdOi9&B@QzV;<e6C^n<(VxudYfeSP%-!SI*))oCdi&-rpJzeGu0@RK9O#Zy3!m$ZTLOQl?ttf}J z=JOg8)&=%M%6ZecsVbHZTsM{Q*58YT_hueD(=IBBY)41aqY+fNEp*^KBEglws{i#~ zoe1qp0yQxp>byExsm%Tf1n~fA%fbBA0%)~9og%0?m@Q64T~&!=iZW-v z1umZnWkx{QnW$>J&k5IV^o{juLydO`mgdQaq5>GTu+&lor51O4pyUFe^|XF#<(J8hwydBCRtzKanf_*DUV+6JZ{7 z899fE6zt=&+wCczu64L=y_ijweT!5UA}%6+4@nuVX{!EAi-wcsQE~@*(cyTXYgx}} z0q2}sFFGf98pXAzj5)JqW(ZAM8i_yFB?!BW#6lVoi;l@>jd!?!IOYlZ`%E zZAsbe$xKyp5M(03rNiGEaoq@l%Vo=4$3~CM3)!d#p132+)e2@3BnzW1Jz-R?1lH*~ zs-!asb^_0wFtX^r(upfpK>4z%74QhY1wz;Dxk@->Ec{AAT^4({8{lvkRqSH9X?qoR z#fRS>_RoWp)?JDCT;UmTB_w@gFI-Zntc3(en0DDQ`Z~3abK8JJ6+=LlUA3H3 zSl9tugm)|m1R+k1;{|+Cv*>2#0QF28py1#9L8~-ML+t7+3pK!fWu=z&x4Kq)xa+Em zHTg|M7r~$sh>EF5p*srrCj*#MEZ3wPrpqoVg`V#7ZVvgs7w)-xd$cTDCRZ%ixoLHw zSTitZs01eiVb(OAIA7(9L2(`DzkqeH-;VC-4v%crFIfls>L2YcvU-Mfu(vP&YPFPq zGs`KY@?z^bn?OBpDyL9@=gda8x7A{KfwKpd;9uU!%hB+c8?X)l$}V@ct6B%y+$A;v z{0NbH7n#J;;-5n;rQ_e4QE(yfUm>F!0am0AOW1#nw1CgCuh$i*QSMT92naeIaua-R=Ho+j}?J-FrFcJ^u>Sz3x24vXyLO6UnSvYh(rL zZq&UP>Ynao)skug?U*NnUe;(V%?!eY_Xkas7PrIngwG2?@Y=3feX zFO8wi2+M8b%?MjF!fp_MCp|4lud(O4h?>odW&Hf@{gW?G-Z2+u&(8qQ42%>&*V9BOo-Yn9RUxg&t$zr9`-jxX_j)Rp&tG;RA%BdGw6Mbt3m``@#Au)@ur-cH%W?@we zd06Qp{aP$5)?wkQ8`s#%%`Qr`)%ooJ$AuVMv+C+x@Z3wQu(IFeDicYX(UIlcT*G&X zz#A{btIOACRNpv5SxzmGX6CJoFQ2Q>d(0Mbf|4cyNBm zm^}RKf$Nafj(-5T+aTSC+mmqoWw3vL{|1osOJD7Z#zS>VRmKDRBNA!-aOuH#d~N;> z@!utwwF%=CzEKeDQe9?xqjJU1t_A#&^f+kmc!j60WQ~*>vF5U=HPW3xgYOMHOoD}P zEJ3QHd-L;mr^C)`*+E?u7WnH56TmfL_iJMxdZA?X^gJ^Pvr5*DJGjOZqqiI5ncoOjl65I(~&k0Gwl$w-9aR2=Su5!J*{#MP}jmc%M+qHZ( zx5fh=VF(yYwz$6!;DJI?BWIrD@7SfBd6!+XZ`}R8VXG1BSHA~Ou$Hgef`?9v9I%0J z2TBdZg%@ye^yvBQw_x`MR?!Pzt+GW5Qyq#|s5U+gMvizk6crBdd+!guUAw>%#o!XK9OsoWYp8BF`(t4Y`0btdsF zxVz_H-YYFZ?~Ah|HNk5MXfA)vdJnc?7i4_86&`G(ZswoTLgG@-xGsY8IbO`4KmFBo zPszgz@%>At1Gpd;%J+1|s4+l1mo-$xEtObSDH@e?H%m8M zt(G7mTiM9gI!A1VQobjsX%S`!xp|3A$0TXVJ(`vB^HpkICJR(~jBVaQGYM3=f7ZYE z-_<*KA%#+}b;}ExY%3X{$k)djVkCenRs9&@jtGNpk@6{I`fKkE zTTgD^etZAd$q%l-n#DTx~o_xEFAH z@DX|O>%^qC$l-47BDOSP`H+I|U!iJ)tIY=SxSvZ+Zg90t6zGk`d2DdCahe^7weF9| zF0{ebE~cBL@8`Y>?GCi zRe9AnxZ30jh@Zl&Hn`einz&-r2)k-GZg924?~A?&T%u57ZtPk0<(Ji*Ut>1tDKmUeoUEer-rIGDKPS_6H2fpiz zpS9Gste~lBHVS-6(+u8|DZ1mfb*i#K26P=2Fc^j;jd7$623A@smB+`{2C!Sg37-yz z=dU}x4?9-B0|9(_ENW?d9Ca3p<*7B8Tf^n2Tk?Bx3Tg0cLWV1=b85XO{R^39R3)Hn zF04d(luBQO)H-@_>a!$tAuJp;(uJm>$@e6Bew1d)aqXZ&Nt}1zb)B~_{FVrACG#iE zPJqo#=6i!FSPg+dJ5MCg|X`vKzxdIaN#)*7t)-RHMsuQH@Io;VAJ0 z%8>~!@+~-JjQ2g0`Egs8(`G-s;}s%XP9S}ENtvIUj%%!l++B;xPjl=*9<)~j;9N-L zr}^~j;~JeGQvndN6?&in5;z#9PUy7BcYSx)j^5AW8{t%5@@iL%Oep3%CGryx!RLkF zukhd6XvUJs5boGv9vWRT8Azqnc1WS*BiM`AxMVUIr81=^Cd0RGBxE?MG*`<66bb=p zV?bO^>Y&XlN|=j}I+GU!QQLuVkdKmTA%i40D2C?E}B?L3a^~nXO)~m$iiLH;f9F+U!Y>7)anb|zrVM)FOg4KWP+MO{sgT(G`S({ zxGfLuv#>>k8LYj9bHCmD<;e}K!yTP(+D3exSRAc^l-h&OBlY%pKi zZ5CKzV>S?(Ds}0J<^VyNFCH()mQ7aC){aAB(Wq{Lubfz$x*gh8Y9g0S7x*svGCfkI zoGT^%d1Xr;&F24lJ~>#vekBhrP0mwVZ5`3&+pM;iTWt$VReMy$L3%r1T{qkHA~i-u zg6QV9>zF^A?RpGMzc`J_Ad>!en)lP2?Rps#$(FXv(8zlzr61R&4KwP}^GxBJ?Rs3X z8ossAN@FQcW4%~=Nfz$4JT&B7WKIDM%hfWshOEjL%x?nnsdHowDiKXF+*%$R znR2jZjblp*0A6Y#A{~h$T)vac&5@#1Zgbop^kT?(|LuggJjvsm zGq)PvlPoRXzivOX-+lh|m)*Uie?975KBThoJY@tsrE}F8MoHC5dOgF4%AyaN!z4>M zyzVQlSaTQB`qe`lLyrf7*skNs9={v}dGS~novr1`R(F-uuk>$`x6e&#*2xE7Lh5%j z#th-s%^17*7$br%2|`K75RoW2bMd6s&QnF{wain$#qU1s?#++)Zhzf8RXNX(F;AIO zg}`r2Dx}i>nt95Y`SJk4gQlp1pZ8C=@-C2f|jeRA+>@@{u`_sz3sovY6Uwdbjy3k-fm2Ij1=S&X{b zSZcOeGK^n0R?>beJWm%gR<3A*2vz7}CWr(`QnStkF;yUtQ%GtmO@+E?0M8TgT58z8X=I$<46cC?Hxgay&EQfQftv?cZlx4+k8GJp*0*A2N=vV2W*W1b+Uk;QrSWPUq4-rx@lflFdI&*?#zc74hAftN3pUP>v%%;jP4)u-+CkrJ;|!TBCiqO*2QAL0VuejfY@kvw4Zetr z>EDLVcuF1W*f>LU zhUcH<45^`Ta7f3UPdoEd5&#O10(S&hTlR^3K}UpgE(Q}(Jo@i)Y;p$Ob7wj2u#VKP zf|M^ECs2?(2vTqE&;P!oApk`8J-&v%Y^ zUVLWJCk*e8_QhljWXMzZ&y*j8LF?<&ledEi^#WlimxwP) zZ6qrdSS1M#hU?R^k8}Np+RynK`fWSIBx3+;Zq$aS9lAaVy+o*He3Y=N#etPs6{fkM ze}P-ozflO;xIsW>laLTGn`l-W(n)R2{LGc=r867OH0<1Q-~Q5>iB6APGn3vyNV(>+ zg{XF=l<%ju|K4)8B&ocF-ddbkp0g`CfmdI+i0R!rIJX|*D;bPc9tUaEjGSdEIcmZ| zD`iK`+y7sKllkeO|4%Z5y2oEmol(DcHsJfF6o~7fJ_5(gwL1Mz@oX`e55_0bN^8DY z_SxZN#eAN z?!@{*#Xeflh6w$2StdLrF#>{&_n$rfSO3ZI?F|GOU;1iS3^J~n;Uvnq4V61|B3n%z z$DQSHag@eWW&Wa4@bPyMMnn=7;i&tL69v8}5jCsBh{|>838S(s5k_dE$cRYBMNuMh zcF07^#Q%RKYb^;FH5m~GPPq}9rBEs5vZtJ^v6ddZ*SS>#RfK z>%O%hqcw_C@g3`Md?)+wCz+&WcK<19)Q??R02a^3Sv~|gIz6RY zJnjvKYM}@#G0pRi{W^I0CU~j>(O1MY`@-k_HiDWHC#P;()Qs(=WeHo(FHl09>)UyAiAV3dh;MGpi*vKR8X{z=FXYL_j@xrOY%jYqf8M zMyp&#*0y*)xH&Z`o&7-j!`-CaeUc&ZDk7?bpRiz)Y2;@mQtx!bjv~1 zMY~I_!_BHhYa=SBTzad$jbcePhKnn^(jnMM4oR%*7uZQpVNmrLY$Zi+TE zIsE>0_stvsm$$v$-SOV;v#W6y4)dG0Csl;~`z1 z$8{&cR4b#ZbeuV(6R6*LK*aGtWsB1Z3NEtIDs&?GU{+1y9P%kjf1TA>(BiL>j?l)? zW`t?8v>A`OB|1OyTJg^6v^} zoJ^s{#{FW0c5t|T7Iz!xM3$4xOh~3{k6Ymn2rg(Q2j3YM5Q=Gw<#x>+Al zHd+PeS3HPmJR5Zc;}qkqeq}oT^uE)x?}cZQYB}px1F|=IHBGa((sE& z-K>B6sF8KC_R6k!`*=F(n3GS}@#vazDJ;i*x@EN!FZ7iYZ&khm!bL+jaP0tzaR;4N zPq&k*gTW7ZnlcuIe&{wQQwV5N%OyTFBJYBY$gr+4TrJca zCul8_vr&j2!)#np($qK*(XhF3m?j}&M_88RAjifOX$Q^1EEnpHBmYFZ?GsvCZd+c^ z)HHGWeSM|}AID)xS_j&owOTxu;+f4$Wo%`hSv%;X!*2(%uYQt=fg|K#;AdO?>;pw* z#r>B0qp#}rZB?LZCExD*N!{d7OMN|1l}G}?3Dp;j(gTxTX?`@Hb>g865;9-NPmg32{FJk& zN8C_8`=lM=r<_);fC=Vd`jx4Xj^sEQ;rggenx$T-bVT{Ll2NPbCZjJcN$xFqW%|h< z$$+f9II%Pg2YunwIlPO>8N(GZL6MY z)+vscUDd1&oYVl0Uy)@-1a6&>|90$X#|d)RlOuUxF6*nl=_DA=u5;Jfr(qciSr&XO zbB-^PrGo4W-IF^fgX1;lt-(gG;q>c4YIpyhyQklDCaPT$-zJ;VuCu4zL{<;8XqbE1 z862#tS}is4W`U%k`bR6pCO9g0a^}j-9zu>DH-JAzT~XDv$%qi3Nsaa4g>BLfxrm&; zpS^vy-+Ooce({T6t>mB1(b&@QZ_RIAvENf|dPA#{_kYdyS*D%Nr^#%vh_BUvPZ2^5 z6-q29yM-<%0@+cZ+tUdn4BM(GWk_m3uz#!7!aleR>PVP@chdXyx7|m<;N+L9qmH%b zso&ex+Br$p3hHRo&Gv!XY*r&7mafbC{<>n8UNh#X!-EbFPQim@vC))OQXtVpCmXBQ zfs>x>uda+ZZpM~w5N~hBmQsAmfbW%KE1UUlfI&PsV*+acA4o~qkSvkK(#6jbKoN&< z7`mi7(9HdsQZAOm5BQzJ!1L`knGan~6FI!DrLBIoO5?`5Ws`Zbslic;mcn@pTI~2y zUv2??vpDAFmC!yycOuLu*@2snIx)p9pJ;wg5{21xY46vj=%1 z)hhyu}cwnLKH?w{|i{YnD#_`^- z&nt|wm#0sk_n)7i5AJb}S~P*pl;^C06O_Z!m^J&8v+;1!=@Xtl?$0KJ{^2@g;dnB$ zcGJ37M;DawL$cPk?X`lS4I7^KRi}{U1hf=ItoZ^xp?@Qv3m93XN@3-$El^L?tobN*p@IDYiN+Ogt0 zA3C$%V1770ST2^GAx!moXVkGeAC^|qZ~{c{%A-P+_B|HhDSe6<|Pl zi|<5y+1%l?yH`7W-*UViueIYBJNykBJ>4T^d(*!J8(rSNw!@rZlXSV=wu!*r|J8ZC z*Nd)y*G=fk1+P}xU1tbG>F{RLeMxBnZIaHU1=7WpnN0Kq^0{9AteyX})DZTqwjw2& zle4xkbH;2U|MhuxdQz(aTlx-uDMABUMtoaTDU@#&xhUGQYF|jNK)f*6O#we(Vik9i z;|N8maS3d{K=Nl@7Dssg_Km%}JL&s-Uxy=9&Qp&g)LJ+za0H|7#c%|Dfvp1*avSZM z!32HZ@5#yq5o9eXsU-n?isvMLL=z!M9%Rmm3R|GyD=1Nypf9ChLNSCk11vYUHv_DB zfZZUXu$VEGtwt2mnB-M-1|peULYIk$^}>KnVN3GDWu>fYy}wqf*zWUD@vLgX9GhvKUCh-sO@5nxunyL@cNgsomWcr z=1I0PJV6+WEna>bnpNe!sHxJ_;E?`kZZIe%G!UEDmW#rHH_^aKq263n3t9nqN{HoR z_GVGi)VLOD1)Ce^3Hu2UkU=3yQPezJH84ScM>z12d!X`T-%#9LoMq4R>KrU4$WXG?qj+ zWNxXF;fj)`#s#*$X1sxedJY#54j~g@%hp%6YN63M*n;$%<5$O9Ueee&@a{F^4JHv> zH~I*FLD+(^sP&vzDbyQ>3m;hBb1QFbUeMGu_>*fU7NX}#(@p=nP>ft3d|Yc_Ezd@ub3b@)@sMs5g$6oS8hVKIpulv1wjw&4c!^!+jr|5ldX& zLIa+u7V1sIiPgqdX;r!{FKB97n|;l?M~b5i02F&2vj~GyH``n68_YtzX%6c|i?x~5 zW2NP_C~0n7d(C`@fd@znL5g23oQ}LC+JLJmHJazZv}UP5s{)P{{|fA8ngMtk%T<~h zEDvx^Z+gtjb_<5QW2>!(ZV=$gEEVd_bwj>^=`IO+%vKg9O^xH!Yu;XJ_>t$iytll@ z2ze-lscf@RQ>r(wW-ckWv!+U8gUNN*+?#FYUtVMnYsY~aN91T>!c@wP#$xXH&}}PQ zBY65rK~uxv2CT7Pm?@<>@;YO@5OrX@H8rJr<8U~zqvU@EH)}c1yxfu18cof`gz}pE z5f48pk0UHD+KSc5RHyZ1O{v~oj3IUf0U|6R1F;TCsMcs|F8s_jrUct3WkrRUIia8z z#&)EpRBtYWkT27@hz+qTWbak!-kK^+4aN{$vqN!s+1WfXSlI^wCV@8S{aY>78{BS5 zVs5sH5`NNxp;BpYwMJ8O8JBA&DGx}A5T1Z(z$avek)gG=TBtV`@-*9t>O`6yZ*Ew) zW`c@*_VZ*a$07~cgDTJ}U%{(|dc)9MVvu8$ROk4-q`7geHRc85PIlvf8jLWI4c{{! zag95_L?1FNDsv*#rZjq?CS*O+RIZU?-T zIZ};8tOo7ns21vtgQWrpz=JOLTVBxIG>4Z5<&^-6!?*wr0x>n1dDTL_Y2bI<69#g**Ybj9Mu?STuGN)N zy}8Ux1QOVhWQ;X!$fBgNc>pWT&10G(u2O;(KseiG6l0~#Xj*1@Ra%x6G&PLoubEU3 zrW5ZsuV5R(7}Kr3kEs^wO=DhX`xnZ%FgqF!u0RmiXl)aW6QqVVhP9^@h241K=$KB9uF= zC~0UMGi`gFQH3jm4JmgYwCE(6XSL-PySiMeHxJ+>!UKyvP;OpP($qW;I(hR$<&|xF z5m|}sHrr*fn6=hb3-!iDzWB~+CY8LPp=mIAyfu%NHy00H+h@6TX%~Y}x8iF`^~TY$ z*mAtI+#b9Z(8J2BTT`W>!8`?b&8`E*P-W{OL5)oc468rsYN6iTkouyjTEW2bf~KZ1 z7}iWHaDiDfFcEBK6y_IWH(xE(n+9;ks|7m2CKwX|d7tuFtk!61EV#g$SB4EOeTUyk zo+03zfhAT8^~SOvz^uT!FSju-XlxoIt@(A4ov%Q{0*v6X6P;h}eW@wc8^_j2q=9+V z01}9x1A>&BSyQE{!Mq-8Um^h-80y%@{KrgWzC@K$y}`itqI@4zdNVI*Zk)enR5J0% z>xmDKUTHBEV$|i^q-vqwG+s5fel3~m&BZP+Xlxp@NArXB1W{!R$^?x2j_0OZG}S`A zX|5A7?Gmh0ZdzW@+_ZJ}GhKx@mh~jygy6e|`CKhDn#KkR@Ca{irEPghW8+{GH}{(i z;1Fzzw;Yp`4US<7)|Bea6BPXFd?Pw#&0@zuV-{nW5n_a!qU7AE6zWa$Z3j~m#g%f$ zpCcCIws5gyilCR0mqujQlq`7hHaB3e9b^r$nstlNknh)|Bq*A6g4WcqbwaS~w zjyE+6PuChCgJm21f~^g60RW3|)AD9j%JgPou@JT9i6g+A*YbjfhGAx|Bkr)VP#dm@ z-3;^xW5%tN=?#Ntf+5A|EH^AKXlR(cBy09*EhY@CcNSNfJj9Kpby>AgZyMVhJm89g zWcFEJ(9krYldqZ6c-p*SAg??~9u7P{t!dRly=k%|!yc*Yw!EaNaiJHj!KDNfz;D4D zAR7#}sfN{AQ>r%)U=piPKz$Gu0Z=OMX|+Z}b9ruSxeDyj#IFQH4PPB`0{UYXWh=8# zZ!ExDz`I`c#pVSKO%o!)x}H?hTD-P&2xvcz)Hk@eVriaHsyB{~_L0{T`&x)aUQjdT zRjV{L*vH}0yw-+{N|~6s76HS+vN|ACE!3L}N{CtGRZ+COpt))5)(jSU2Q!?(h~tMf z0zQxS&eW9ZjdNup$BS2mz>AWG=HU!zubG~qq2TF;TL~!#4ivDS*1Bq;-Z;h}BqlI= zf}UBY@z|Fq#A=O(#$sScYpfP12y_PKyChiT9WWM?YN6g(GFiyN$huH&WKq)8I9BO3 zW01L8oPW4tpolPqz@yc=vZhpT9ud&&^wb{8oVfFXhQ?v?AYIfLXN?2K#?oibhB9OO z%*0yTs)c&fgmo!GO4Zl6C~0UM*h92t!IPLJXfiz67Jv!QF)&_{YN_5l>1gmwo-AuP ztSY9{$`h_uX=*U<>6&E_A`H7Bq3yCO$Eq{nm};TkT$#7oP*seY?08edfX>#~&N5Xn z6M{YCz2uRxhG@^Zrc`el`!t>vj4Hk;`*P;}N>5g+G&LBQc#TC3f)Ah~Di3%+L~~Go zXR3vIbJ_Q?DP_1+po5~Msc|9uAG#iJ4-U1$IGE*JC+^pxa}P~%oWYFWI9Pq7b^nHBu1b)FnGnd zS75!olv~Ng9sRh6bX*w3Xe#oQNWmAx9$ImHMgb466-RJHp%ur*6HP>ZBJIfWlV(fM5(B2IL)~|cg`@gSBZ@T@63|o(*qMUa;sVf3 zz2YFZmFp_<1l3axcxpUlNo!=Bd}}fqyqEteD6|W<9xeVhx0X{$zBH0p=WPJN-L`GaCaE|#Y>|#|vxh`WJON8(Gakq20t-h_k;cI6~S{iV*>@Cn+ zn!G8YR@zdZlR7fOVQZ>3>MiA4>S}$}nvS}+Sk+3dmfi{NiyRSuBz6P!M&3NtI(eUL zP{+TE_M_bq{YlS_t$)bTV=}fC5~kVXymQK+9(TIK$w`v5kmqx~+7N;*9?u4(m@Lu_ zR@EV}4yp=}5S-fV*~qQ{9(RYG4|k|om=H&c^Qmlg#$uqT#e5O>^!*2!s|Uve zeO{Ohhm*5-mYZ2bsIreP*on*&wj3ujoJ?FG@IWO~V)x{n&Y9k5^B@sQuP zbJJ@h&Q&rm{u1I`>`6eH2$+K^B+%sHHd*KVX!>EA8AI=CdSJH&}Y6ENKbMO7-5 zN~Nm*GyR|LBv4rog$ZolK9u;_mkU{0Gpa6o!y#4T$}!pHDSJ{-b|dtI^BQI-(=$Fp zKS`-Pf?2up4N+S47!1-v^}&Q1!J@dw#~k>2=jOw-c1r)e{;-()Pn9@ z9P?0;0&qeEKs74< z9R)7|@cc9YPN|gZ<4@#8FdGN*Qs1ffOo4e*Rel%=Ym$V5Q=XxnSeYmwM<1F>yMUaZ z0h&K}MZ3XxIE7wx6hxnggPlPIL52BUZkz4pY1Z;q^APEJ}O z7LDNKWXswcSTAI2nzED!<8jzUna^(63#?Yq>xCQeVz=152G%LUKGmXdEc<0>!8YCE zcxa&oU}x)!ydI3x3#|*T>_q~TIv#C}vAxJ49M#*;4*74_3jUaB6lH}ItBpoU#aFQR zT+@R2Z9&rJAs&CKah7&aujdUzQCks$g3JmUw$<4$#hK{NJ2)QtyC~W~E+gE>bW0TbTP7#^- zT;aY;Fujm;iovaO_-6M0@XhvZAhKoec1=Wn2>bHbU?m5@-M;UJB!>9GYuFiRP*7!drW z06_2gr93a;`-@j!+LKq0#g6*DzyZB}9>Xk#NQRS*M|Hg0EU1C z_No*Z%qeA%^B6(4?zV?tObnRwjMm?OM5;(#Kk>-u37|dNO4{*Zuy9Tv}t^>hkUFED3F&DXW?WTA$3TO!|7>gwTAuvY!FQ^wygK37h)^Bka_DgP!L+U{9B#zu#bsL zR-56Vcfo2hJd02#PFZ0_|FYiTa@-y&ZIjcOvMix$9B^4QwLYT8)hA}ws25zQ`$br4kQhn6Fc;cU(KWLz$Yjz=Fb^>lG_l)9olZDr|58&$6q6yUI5m-5ABMC14DE(ftAj3^YkN_L4M>!X!I^4B%}NQxiN&Lpsw;*5so$##L`K`=}j%2gjRU)F!98hE-Qy);!^2G zX6f9Ig6a048}^1{sm~BeW`_>w1E!S>bXo=7u`fGff{m z`tEA(Nyu_C_4D=hUG$tJOFv$Gd}(( zrR`rP(~r^M^GP3rwOsIF_F?uK-?Z>#6n$1FKD>UGKi-deea@?f^+&2+kp2MUZ(hB) zI@W=h*}>t@sm)PLL;c{Fq7cNEBmtlp zqft#)ye;2LRny<4F}kt6SJ@M%%at0V3G!`rA9f=#VqDg6>gkt-1YgP6{gw8=?jZ=7 zN@Yjn*ra!fow?v4xU;+aX!{Btg8TcA9_3n-4HA^m8xkRO?^}8ZVu!8@aRp4k@VXL4 zzj^SWH-6Ln)V~ExiE3`SyIoTgo@@%D{ucfb=qkj@Q$nggh&|d=KOrf4jdTsk+oN$o z!=+p@$4J%?lXFzTHlvgQi(y-~SFK^%n!kfkg0H|d_DNz|E9CIX2?p6oCRPm9L>AyT zqq%#TfI!h)fRhoYm`bYSO}X}UFr5;fsO%<&{>iR84@k@P0B%9gdj4>Gch`a>>jCVH z4`89Ga_aEaOPTf8aca}92%R~xj1c0M6h&~Ym5_wwU@!Z%!J&kBRkU$Kttg2o4Smtk z>C6)EfAmGTwb?Eqjp-_IEh>rIc55p+)Bh-LZ9k!Iip<7ByW|r*T!BFgS!(mim*FHj z_rlx25zF4~nm8g`vy>qn775bK{CI|RL+2%GZNv6kWBn=EeqmV8^izp)Ot}2M6V=!B z^iJ4Njh)Wi5>>3kSGUnyw}mp0?1vBzu>ti*Cw&|X?nzD(A)tpQKzNOBszW3@E+I+= z<#8@rA}N(_WIDu;_C>1|hFjK)sAqMfanFh-#6gV1Hh*%)Bs4CGhDKD2*>TCe@%K;D z#^*l{UwvHjwZ(yxOQ_{MI2whvT9U=4is0-d?=GI)>8ok^O8sfrCO>`}gyD(A*Jq&b z-XVZyCmas~LVX9r4eOBv3oZ{>`J-n~muPT@0?5KOc@SBT#v zchH>p%?4DH1qdg8ii!=)wp?$^tzz|CLpU7y%YJ|-%z^9bf-dJ8`?@OV2Mrb##!Vaz zNUb5ML?Mzr-W-4I?>R5t+y;bO`fk^RaM>~>5Uxr#KB)X9mZqc((&A3x^qq@MkR?RL zCe$qgz&?%8a8C&m0ReO3Z=4^>h1S-cHG24PpLr>zq?Qf z{oB47N!3s5I7G%j*qMnhBIv-EOgVWq49v>KQeJMnKYz?NXcc>9DrRETMfKbUE#7F8 zDLzJ1ahuZ)mYHt;{QUf3Gypsji`v(&M6@E{Z4RUA%lF#{+t!C|>-DRnR~cHXY2=1w zm6+lz1fm^^N_V9@JQ7S=QG=#gI68`&8nP(OqiKla4_#-=C;l)CA0o5L z^3ez}V!6El#C15r?S>W_NG7G7KUd0Y%Cro4UQ}v$ya(+zAqTML^iEXsC()oYlmHAu&52U~ z0xvTgxLTI*W!F`*twFh$7Pw2q{5d^`*{Kbg@$_`o$ESZzUynCOGgu1y-Q&&o56;JL z-~CoOs(J@ccA8V@x9%j0NbE@kpX2W8JU=v5l;E(kY6X!7acRAMq6IqInzTFd#CZg<523L@o9~R zBT=YL`nZ?JHB~q(HI?^QTC)>PrD87_3xZQks{9^Qgi%taf81nTQQp?9|Mi-G{`o&N z%OueOKY9N;^}^Tl8~XLl_55bCsu|Dx)=NpUFa?tlG^8hEoP}7Z{$1Jd|A+kto-tX* zPWt%DN9Zy9_M-)9IS(N@e3>|j4=MZ0!)$9?8MJ?nJOrq6eOeLi|n>| z3Z3^4norI@On&!fuij2ybszSFZ|(y?)%4& z_lN{cB$t2|kg0+HTWl1~_#mc1&Lk7hNeshY!Z>t4+UTvc)gqdrwX8_FN8m~iF4FIv_A`-czhCh z-qeYw0h1k0LGU-`b%_b(X;R7}m~-NJjxJ{$JR$~+`1CF7h9}7vgqEqQ`;N4$da3^*G|s z+Q)-vyVV-b1~3RfpP^;}8THgGu$X3pR%m(0nzNoH5L4-*EpH5ozt zc9PIAgUIv;^%kn)z$3Z7nQIjsNGksEW_|Yl{kzA{&TheGx%AzxYqKmba4ZYr@Jezy zm1^Q{+AP0HX=;X$kq@hq`6T4shDgc6ODB(XjSS%AQ#Y1>IL^(|Y{Osg?)fP9gq8ez zBPk^nH(W!!Mbe$aSZg{CIi=EUch^|Xy8rYL()#xMXnHDGi*nP0$bfv5W_$a+m%(6m zCj94+n%1742Iq?SP6_(+HhbCCgW|59U^{}^{ebv;iT;1uu?Q0T`&j6!4xhYwzjK7U zQZU$n+bRTLSWWtH2H({~nX9YZnPDCmKOxg$*kPe8?#zlu2nPT&hTN%F#57-WCxUj5 zW{tgBzFp-{L=Itl%Oiw5=QS)Z;d|pVU%M0Ct~*f;!gw-e)-eax8_JRQu=V;)r*bg7 zg{>K%;pOgj-L1L2{^+gQgCeaW*4MkaHD8J85f4ryCQJDiDcaK{`pRdqNOTDKb9q;Z ztFnU@^S%`1=2)VWvsSX?GW^=6g{oY#pL@uM&4@9J^7pYGDLT z9_%uU;$>#TY41*$4U5}jJkSd_S2g10&mxb>%B9ALV@0fCW&W#PIPQ*Q`KjN|8^28F zRjuA!=OrnAiWn&)ze-9O?pWvb%h9$te1SxM8O#E-4wE+dbHmxh`sGD5X@+2?UVChP z3?(%k1i%7?!dhTK4-7#f$GD+vSnV0Qt=Z_I)%v@T`7|Sn;YJ*h)lMHLWWvJOISu=x zi6EQAB5C;7dksFfDi+?$==`-uTYuOj3ditSy9bkK@1Msc`qhzA$M>X@UYPL4zbCcs z_GqL|wvIQ24vA734){rN{m5kiHbTuZY>XLCQW}a&3#1iwlHiH9X&#PF>~!S|)NG%V z0}&Y2@L|$xH2f+FD>N?4w;h`ala|ZX3oV;KH+j`ml7_^=O+Hpg`>F^{`CyCW6gA_| zT|DY%gUPTnweIiqP=3C^J7V$@Rs+SUY2IisJPlWf3%Vgc)2%z}-%7S1w4=Al&-9h%bxfWP(3-1cYuJx=9_=%SlhOt3Ond6joO+;# zUdq3;w=e(YYN=)O+p{y-zVFm(HcMcgxHYHZ!ua{>N$~#T)3@iVNAkD?rJ&9jo>_1+D_1{}W-j(n>+IIuLtKSX$9_!~I zp|lGXr%`QGYLJq#I}>5f2xX=w)Ukz9qYG4a{W+b;0j;bDxc&jIm070zEbgph z9>-m5)JYw}hF(WI?bd5m7pW}mVY$%gpj+MWZUgsrgm!zet+(pj9QTrCRn&v|ht)`} zVq+6XN~Ie(AmG4BP@!U`9|HzL1ymJUu)&X$5u6QUEBF+)BzYw`bQ*P-kgz#?oXq$+ zZ1$oqH8AE$8MPC7z_`)J^SF|;_&U; zLyB^6b#h8LDD3T=U8wL?9Hh};Iv#!s$5WU>&)Ko9*7!mw;R!dH_0e=F0@2vAjBu5H z%R2w;O4Yj+Vg59gyK;(QL9$5{FjA+a*>yd)Hpl8(BQCChJ%tv?Gr`!6L z$WT;07yaf?=lfoPUiE8eqjGb!fpmZ~xyMSH?Mrzrn=WPNh|$Iyl4(!Fvon0k!cjCC zl2Sx?<|`c^N^j4uLLlJT!Z}~*g#1*LxBRTLjqwv$K^=5eC;}?SvzpgoldHF?s%C}K zdbOqZ0K!y4iVa!~4Jjhpq zvVr1STS&EOQQgTWkxtS-={NUQrPWZ~2Ms&>T)kbV(`+H|idRCvvmy0_-5ceWx}iLc z>hIwG(cbgtdk3n9bcO*`?b&*K6#IUq6F(nKdv>VCro$HR*@5a!pjy*sbT9Qt)Kz_X zT<)c6W@D*`L5~y9K+R;dvbVEngav+cjT&v;VpW~dVlR#*6Y=;@71n5A;IuE3G!#8LLOf34f6(+-5db!gxN(`|Z?t zQ?;O_-X^I}_$p;S)M_C$>rI?ki)ix^>4tD09ZyC=w4K4t__4-Dwxd>Bd4W1qdK;*l z3*zy`wBny`Ap+G@u#hq7`<@% z4Lxn-uE{LC`sg6)65*Px7>znFz8Xj>k7VHVBxtFDXsUtGJBxnhdJ^a%lRbhi@iO#G z_TfoZ?WotIhbdk?)e2^;W9e=XKW*r#m$7KtRcW(Y@pL%InF4T0MZQ)?*K~0yL((*p0HTdY@AZ*kS3|+-Dl0Y35*;y!+wMZ)( zWjk7Zz0t?+(eCz6YrOkd=!8wbhXfv{ z0OAC9g5vO8My~*nZZrZ)e3VxS&MIHW)f(6fqJtLFDpxyS`GsxS)$to}e#;%KDF|lm|j;ZaL+Z zAr5;TB9l+n&~mLnR_c+<6v%9MN_nEy)bX~h)HSifzMx?7o3>H}Y1MhGn%C-`#(yhA zy6lr)$nJ?zont%noqxoJg{mvsCB8#S{m=$M5QfgB8Phk2){V! z?$|qr)7{qsf1=5#7hI^R4nbbRz0FDk5MB^I@SmK_bu~i=H_AMONNMG1zw`9_ns3K2 zuJYXt-?%D8j#pRrt6thp$thWprOus^x@w6m)u?+lvebew zVUng66bg{?2}LTq9Yc9@(o`9Yl;RYXhUQmA`CVp1&C{mBk&FZdZ>p;jH5Q3e^Hi*8 zuvE=CRIGn4LsM0lNO}2fJIj&HmiDXE0!688lp8nWKd)VtCtY zQ7rHD>UhX!cXZdumihmbV=vS)XK0|Vwsb|uo9CGCHnnKVs-(Zgyn>W+D34tf zC;TNVbscA9+?dRFQ!33S^r$(TC{3`bJO;sql$@TPUQjouIF@{JBhnKLEh5{bEZff& zB7R>^SYx(jDvR+-R;wvmP;|kf5pH+TeNTj#-|e~@)bgsP+?x5uBwHqH z&yR2C$IYfYD9mlu zamom5Fq~RMYK3jGzaAaE5;_C5X=^`Db3azfdx%6W^qB2oU*CI!oA*c~p!(1s5b?-UOSoOm(SWZ7fk4?B}0T!_iEa_{B~#R^a@~T#a4*C zy+GLsuq-YW%-16RgsWH&MIlq8JED%3f$by|11Cvr#4?*tuPKhz_x7h^= zN(SA-(a{qj7qCMYQ;r>aIOf@*gKuB*)592Khn3tJee9xqvQ=mSrG$V|U;T`0>elqN z^}4!)t;=!o=RUSNr7x67R2oqQ_#R7BxEpdtHKIEc*A#!ka9%^|GW`s?Au(C;RH~Qy z8C*OZxwCo_-AnOpG@rSmZs4lhC*9ILRaJF!`ayNEM876OrTZ>%4m7Iz9(P4`BHuYr zbxzlJ)h+cIM-Q7i6OROO3G-TuM@2o%<*#DSXyr#mkAZG3ei;3p^ovl(@%ZTxt?B7f z8q9bSsG-YG1^wLmb(9A@zOlNQ?x|=0?Qe<~LIR-DJ&$?IK-gX@uujKe2iDfivY7W@ z${4XwpN@xQA^sZ6t7O#Kj_ayBT#M$X z5oHVY*n7XdvJM^{S?{+$yt5A9zTSKD(&|hfT8Ga_VgReip7rM4J4^pLf-iPwc=5%u z?!TB3?gVRW5T2cq9m~4^EC~DXW8Hri4u;nKBly$glrDP`hON_CkJE9br{i!CTKAuD z<;OrX^#+4UFPIL;Nw(|3a3BRxG)_sUo$~;Klq>2)xf5FD4*No)mBKT(s1tX5u??GA z@%g4@Vxi`Qcw>~P)Qh@tiO*4ltps_K#-y=j=YqUl8T4uEg#1sFkmX#zesMQB`67=N zpXi3elW>5WK<?qB3dXVqVR)N=)CLE?9X>s|1x5VQce|z{ zK3)Gr{a$s*^3!nSQz{v-<2PP7|GiPRel=3PkPXSt6Yu3C)g@DoQ^94yms|(+5{NWQ z8L${c3MyGcMj3@QWK5y|Uk4eL8(KMBzTRxUe0F-UJ35n3eL4Z;~&G$-)0|_=xH6!Mx)_)dh~=c6Mp8UQwlTu8Drfs#8>*z z^qr8;QG&YpD3xFo3svgaR)R?^-zuTXQ9rdTq=hgEh^GZ5wayc;tou@)m!4u-Sj&q_ zKGOFDv8k%MKudT^wsYIo{p9gf60>RK%x83LP@xm@eT!=qqGA`LPqOoM2ksA99u z<^?z1ecg>%gw-;t7VFet#BydVo~w5BfW>F@gI2B}^TvayC05l&JfQm8vXrRThy_yZ z6^C}WuLf63?|S64oT|ohk*HbLQWzs`>xULLFa+-0`-sK#yg;wtN=~;B4|;B(%dcUP zP&`irX4Yp2dXJVD*9 z0bQzXyD6x^cf}wr>Bpbp16`Sin@wOnE*J%cnL849x=O3nhgKzX3dD}-L4qLZkk8iRSm zXkO*RUSy?y@Xar*|m1oPUN?+!S3E-r8fH0oZ>VJi`4#FA+Eb`&*BD|QFW`|iPf1(v-U z(||;5Q*#c}fcDbKhz`WNoto$hxx{oG{8PeYwf~W146i$C4&l=4)d0qMf6`*<>gG^o zdwcKw?*})YLuNZa)f}4VQVQEi@ogg}k)9=SdBCm~(^|-+Gv`HN3hAjM^HTCYnpKx^ zD$OR*dYKybNwkySUf0XidYRhPVRXj4NMP69-|GJAW$MoYFN)$p8TBp_ySUmOjZ}be zHl-XJNiF9p0Ue82M+`;&)xP)=X^7-pDVej(S#oDtN2aHD!o@oe7C}OnSp+n?D%lYX zbX6)f(A7NeZl>~8y!|fC#D21owHv6-DkWnq%+Bbal2YkL=F!)WCX=(M z-yMdt!Im{WCH3IsJS4AnHyZb_G(g#a(!JdlqiuFv;sLQ=9)|UuX?W41`85v+1ezjN zXX&>MLse{!%DG;Neh`-+G7@@v{b&NF~TKx-)OxeBmLu=>g(ApmJ`y?WHyCvx-A6m=} zwo6Qz&xUv*;X`LV9ukcFVK`iQlre-&MRzL6hy1x zF~ikecq&YFTd9yx_3Z!$1=-~|Ywodm@I+y5mkq`r!$A~St>IW)p%?DA?2@4c z=E8&3YQcvwoK37AAw#`T129~eWq)_-+ty2l=b`$g0S*7TCAov8s4-!`wSqPoiVTjHg6; z9CL^0N*FYf0$<=^apPJZ$6ib;;Em3#pvlC&UyERL!xbzk6>BRv-QNE(0u6D$Cj z1>Ku;BqZ{f=Xi~yYXL4uZY>2DJj=!H+TXIXT*Ye_hM&=efnAj)hP|UvB_Rx~=$lX9 z|9*77xBKxH;6iQLyImhH6xUD%7u;&CiXgzJTkus#+{z25zc-iYMJ@~mr7tY$=;3p^ zE0&A`BZ*UCPf3&niSnxEve?`b9SIYxQnLvJ@npIg;-Us8xl-Ov&dA%?sJNhrpxur% zQx&A_?I<{j2A!egf9SI*O`|Uo1rkSbp#Qi^D7rwE>G{d0s7)Bj^mNvTy;oR!1uErw zgGq08you7`cyo9A_a`s^2xiX7;k)U_*5UGIsg>Pxz6r}vY$TX$fo|BSdo^ui9zhsp z&?3A%-c+N?c*-VuG2+eKQ*7=0Dm*6e>GR_9jO*1(Y_gPpO(PO5$jgyS&aqTtK35h? z`8TcD8V^UJ9L(FWE^X3KH!r^TG6t;T#92@Bn|@e#Z<4EN&m`B2O0q-+lM!Kf$VDt* zB$vRaf(5NTQ@;;R`cV&rcWpU5sbrk^&%I^@!kqH;kgg4p4=hJS8!L3; zEE|7i{gFgy6-P@I<^$03i_axX9oHRi!tncWG^>^?u~Rz`0gaP0Pi83Pzh zAM{6I_wAb(D;&X5k9MD(_J2Lt>kfCTE$8*@^kLtd)#nae-$>xK-W7wK@Scx4?QI4(^2gDZfiIspqQ(V#`wzBpZ9yJViqW%;s61 zV&J9LuGn+v2D>7ko0B z+;{grLlj@Cpc!&pZ2_)+0$x~zZi`^oJ>Xw*3i(+2sq z75Ehq>41(G!$;l#nlOb`<5CvEoL|vwI++cIeL}X+!okOAOU1)9>v$10R2p%=8DKc_ zY#$pF$+@vQaV&&iP%BHNy2SKWI}-d_4}z8yv?>^jPyI25<;nq&Yb>yopWl-1;o0eO#9Bh^KE&Lu5?+2Bc z{<~v|a<-ivXP=pokPfYaCCXJSQQb(4`j@qPO_r#yG_PH$OxmShG&oC%o80|J?q2?x zZ$=Wez4vH8PdTNWN)i?>k|*paQln%XZi^&}OulR+HP!$^`rMieTf*j}4^Nz_|7P{U z-OJzXnugJIwG(q_t==Hc$DbR~{R2`&d0QSyjtWW4k^na>-;Elnb80oW(r{VH4D?Va z;lt^tg>;gylS<-fL>|3Owi*sJ3E^yjPCduOH`kjg z;k21s_|09d8nA-#k=M}{rLQ|Z$4gG}z}Md~wN0E*z?76{Sm9n708*am6BK=5p#d#O zQLRV^RPLK^R|UObDpd%<6;DBa4{KnKs{l_QKP|zwZBPhC9ohnCKnLT4BskNo36QOm z?KU~jFw(Lw0Jd6lxvE@~z;f#N6`=yEbcDc2|77J;h)1>Gkn;_v1+ww&raXh&rYCr` zf@W|YP^d`yP;J*~r`~GP2i?Pl)0V!~>T<17zY28sThe|5mvxSt&1%|n1?(HPWOVfG zn!-$VRa#N#D!!KUg6q{)v{tjLa^J9}ke02kk~6hxji1m8Dr~3GUR}=288>O!BkWy% zw$upZ#?zA#sJ8+c=W0j2ML9!Fg%Mk-x1qYPMv4K)l`y2VUdM%pnsxf1-*~+pBSKoM zwk1HKVOQmS)kK`(jmW)%I^(R*ByXf4Z(NN(tx=PoJd4Z(CgHU_-!z0MQI0xztjda8 zz>_Y?g8+~wqQr$zJnBmhaFVyHh$t(Lhv$olCugDcsx`HsW_DyR*s?QkT7@c>#gqDx z)}*6LaGHJa<4;nHN5MTx1A9g3XO<&2#(=T7QgQAML+?y*~-xo%Y_mZPZ`B9~~U_+aJRIyPa^*s~o-FeSX&6 zd)z&kJ^t*y`0%Cq;nVc+w>QssUUYwZ|GK&JY;-g}{q=O`H2RhP7p$BMZYXZ&ieo(+ zG`GIt$GO@sO{|8o@D_EUq)_JuSDJF*Q)9}h-;F89nWU@9gyYl#XU^+SH`m#4>noUW zc-&QJ=VzI45EIs1{kVk*SIiGp0R7DJ5V7U&LF;F$mrIq`%Yv zP`|kWpau-L9k`h?nGXF;VgN`=8QEBb|0TfXt`oEYaf1`<^(;EInipr*6E=QU&q??9N79tRlDtK^=vy}C*Te~BnAJ`4<;aRK9y zzakBT688khCU6NN#D-n>8j5Ku7gb#LW++rIv}}sg`IkgRX-qtVmHNge1Q4B#Uu)D5 zeQo>}^Y6Y~%JuPAh41C$VrWlJRge#&5wYXtOcB(=GSpv2->>g))3ZceSfFwLN{v)^KEK}28&6DVkYAUt%13~)A%otWM!)^`=H;)$mCG%xA@sA zCjfFf~LVqBnRM@DgygDqed8m>TT0vk=mRkoi? zU`wfbp=A@c#4m?B24ZpGX7KnAlT)?s3E52^v6M@>KK@cZcA9_F;mN3@+a2Hg|GQB* z340d0S(~(RW6EkM`bhmD!ugm$283IGr;^Oh}Q`n)WU%l18ZzZLdvegi1(M z)B#LFl&RC0aP-4;B!JtLYtv_ZVDYS+ibxA+? zBIguCkYsckZCJhGCpk3^KZU*7Cl!J|Yn^Ub=V7mBoehTPM5s@)7;s|F(=N3;XvO31 zg)r6e8d{%1um$j>JrP8DH&EyI#0gY) zP~doVzd;VFS{3aYJ9YX-%7u(^Jp1Qx?M;V+i;D}EK~Z$&fZ*HltUC)XIGN!-<%lAL zp{-M8%#my5%99hS8Hn!|6kcTPE8!7!&V(cL6<&U$3iK!<60%4)iW;x|ab!IMT%zc5 zs-*PEQ{p&+vX(_8x9t#IVIW+~$p z?|7@+-+orcJFn{Tf8DL*<1(_740gPmDO-LfbhnzgrKMtZCZxAooEH&$a1;%;^+hyd z>O%4`g~HrM_=tqh(BW-Zg&k3EamKE9?|4)7Ky)#Q3P2r)IMpRRYv+305#5I-plfw9 z8NWr0LsrnvT^#1rA)z##T%2zdkH7RhUVh1O{@RY8zX9%lX{8)jUQ6Opl7ddY)%<#;)acH5vhNVvFJ|Z za*LO+X}yccoqk~*1TAGf5kS##ym|5otom@G zuL=Bl`c9>Q-F8Q4W}K9UGgR31;{L%Fc}>EyhEq3}$>>$%^U~OxB0t$Z}lE zAzH@WB1|m}m=Q<+?+wiSlv&k(R8>w5S6;FddQ42L@x(iB*4|z`uRhs-cMHe~N4Mqf zc1`4zZA1z&Idz;3esl^!L?^>S$g)&m5EPQZqQ{w-P8PVFU1}o~L)1mTIaIEP!aI~S zd>M()WttI8fVA>gxYk|zoJD7^0t8tJ$Pr~#He+*Ig6t%_K&G0SN8_`aH^3IUIMBOs zbKO*pmBuCKYVH|xx~Zg8cJfR&^<>sOfXrnaZdvb7*(w#8F+YmA4I7R3Zx4STJIyB7 zuhmt{W{3dJqtkcr?C6v&-LBllLU{M$b9c2YMsu`&SwP$)oT)|qsAS#@U)w5 zSO-y;aJ0bUdZnQ#I= zU7I`aPQBgRNF%ZI-L8w}(uGN|95zR%u7d-90_|4?@`SBQ=FMqpziL&2@CZv%xy)I( zOzoFhD$A+fQ?I^>&hs^htBp>4aky0(%d?jov?x?+u39L-zQdSeCqS zM^#bBcRw&0FKWI!SlMb7*fJ`O0%vx)lvfgOsfVrTfyOu%XovIW=9(fPN^&}{yplW_Z+e-#aCGWWgU(J zaxzg}*HJuVk`bb99fhsa0gAWog(bAPxDiUl54zywg8KnRAyX0KCNW$E zN9uW2B@%UDk2Bp|0GASI>f%O?MakXrvJcN$ySW6tN%&K_xyTpWu-R7YaBlEPLjtt< zd%ZO~8#Her&%*L|yXJm6XPr_TD^_Zchzvg8yScGmNp({Z!{zZutVv9Gx_3%?)!I{EzZ(d_(v z(gnBRyK{g3nONYOVTac$gv=u&qK5; z zsyjBruf$;M);3(9#jKQK0b&?&XEt&y++K2k`7c9~1049?v)1d+JBRIOmBa1elH>qT zD%?U+h50>WY>zi__xtSDNDlCz^S1hY8aT&Tj|hQUmZ{_b3I3trk*gmWef_D!{lq zw&9f0R^5rEB+;o64~M2@a-v<>34Sm!WVu={fUZ!NuSe_Y#PxElTP%NL#Y#zg<(kXn zE+}pW`R`mpI!v+0_|Df~;ap+rj(vq|*r#&nsbRQ&vV!7=qNRg1?8D#TFBE^%CG&e< zt8W*YqUm4gF0zWadX*X772CU^6muOF%9NC{cjVkfo=(4<>~=?6M65n^RUYfqNi?)| zNUu1IhLD;LjtBR5PlLg1a=OzFE?$zT?{p{ZMuUlU|5?xt`r#3!?bB!+^sQgoy5`w9 zfCVg>0N{0Pz6z|fb#5gTc`NVyQr)*{&(+{q;%y~{G@oUHa^)a~6Ue+=z z+LA&5XX^M^W)_@LlU^_#j*|qA#N3Pj3(PfBGDPOcld+veohWyrOb1jlV>vPgrWI?h z2TC1RkD^F#d{*Rk;{^w>5Ut-Gyk5dKy^}g%#+}+q%#$QfMlu+3lU(V>~&y7f>qH3zq$&HkL1ZklaU#f>CZXg~g8Afi{ z7LOQ4X0<55ab=H}n5On5VSXig^-4g%gZ#Y=1w7BIaoPj}r!&n8Sg}aLP#LY&BLX99r0Q>b^^QfX&%U=e$$>qk9XHK1<*2n%*qsRaVDH6J`gcDWC4$ z#2Qy}X5owfO2iLDw`pKbveFP2w7`=k<(Qr?6f>Ab=M%{$AA^&kz--jUuf?xuGG$Fx zj-^+rr2Ep9;Xw13o4Z~?-p)imk=Lblz<}eEaRT>11ffhrkeoAm;QJfZ+aPrL>kM@1 z$Y0@Yk-V(}YxEW079J#3gIfLyZ!5znSA;n7`pO?~?md9k~~JuK;UV!F8PF7 zZlD^rB}GrG8Z`b<3h3&5IoLUS@%v6&c)Tz3H@{5X_OtRcK+B6Wx`&suL;g6oUK1wj z+)OByGREJVmX#Bcg%%kxin(AZVS2-62JWOwxjulUK~Fkhkh`+`@NK zjvI0w0K!k`)R;1EQ_hDzHQR)B(9%NrgL-broN6I6)3dhSRve7vR+8x?cfm#7mZ{lj z$*GF^C1|ze{&o1a<#F0Vj?iqVx!dySLCbIX%-yU&8t~LL%tyJVt?s~=AoH1LQs-2^ zWfJQeHEtNRM6(>=+9Ky#EqUUkN2JV>mV8M7MHVj6Ec(8DMlE7=38Jb2*-BvEl2Js*24tzY)yRan>-S*T-x{88AUD|YIw1ZYnisKu&O-{)T()hChZbAsu140adlx3tDFnasQlNkf^ma>>qEc zPZUHIjYRfUCSL4c=wZ9x5+;4kXx~buHI{T}d%l*U}~9yX1+)ahyMl zbz-GlkQgQPG6D?rJ&6Y-^>Z)4k`pyh zX{mOWkR%4NiBF#({R621CR6d7Cv}EiXa~b7o))C8)~Z7}i9Z+#1(FJ_Au}f;*jW;b z=xau-!}#3mUnGEJ95rXSb)*%prE*m&SR<>SBETirtr0$Gvs^7ux{7XXp-z;-oZ@M& zuREBpiZE0o!7=i7ZimzJ&nV^PWExzp6O7xG&;OFSEv|OK@#bFBel;5g@3yzMUsfNS zJ{+l(6RL2*`ce2sE^R04LOyWM&;lihNLMWd?iqEj7Pyye=Y_aE6C>huc{-d9ndh<# zvX3)VB`0&k9T8bTMdN`!%F%fFAiGN|Ufpkf(hqvP!2jGkzqds?Z*^~#Bi)PyTrO^J zJ+-o!(Dl^%>eNCPpOe5?l|=5YpoGkOMP#c|TDqE5$$4Af-t*s%e!JnSRBUIxDw#_N z3*hOj-;CE9M-V$v$Lim-Vrx7cN!;2}tI{pHDxVGSL=;`J7x6*Nmqg|BD#Nebo?k9B zJ}zu#I;HKcfeM^csvf8%#RO|mK`a-*K|-E>R8YZ(t`g~%BG=3ZE>}w;&B}wrv@J%i zmFIS`PTX9%;&O3o*M>do$uHWiWEJ~K5Oq2Ui=jZ@&*WbTFuV5TE&{4_YnY>Mx=dUw zwzCFiDc*`%URDpx?!e@}D);m}0tTJ+=Q37mkRyP1agbZ}=8ih}jgw(c?Bl6%LeN4S z7Z#(sAa3;AjbrLm5CqkN2X7bKj7 z%i~y!w@Y0-ZE|hLHItT|QuRX1rmsyCT9EP6-i1o?MHIgXUP{a^x09biEgR(<}EB;GIhFPFPv;xEeU@tuD*zRV@SGt52NYv=F{EhABM9JA%~NZ=xQ&H zH~;nY*QotZl7)L(M{L@^+-lXaCa0vRJq^dwIezCZE44VyVUd)i#HPIoaaIIT)80gy z)p_xE6*+78+zbD+*qiJICZ{dW-NMBu^|@bDNd9ybl5W){vqq(XDwcWgG{gkdAHRF> z*?V2T1ryNHce|zu2=9vGlBaAyc3pyc=h}eQnNn|V|3Ra66(!kacm{X%OYw|*b3D_q zne(>40M8_)(hZGgemS2(b~WgBqiJCMLN@T}MKd5%pmhNa4l(yYjmdc!POWBlYW0Je zHJJ^%XF!)p-@+_~32zYf*)nidkmOwLn$Es8=a_pY1|-9!#`QO9qO`iqr{*#r<8K?% zA7dE@&nEtJ8=iuWIlQp|%j7R$B<3KLd!#F%RRS6%^VOURF@(u#71}YeMDD#(Pf$>b zU2LC>sD7nht+9}BB;1u5`Yo4oeT-dtihIFe0l*I29fp&B6wDNJ+E5#Y6I-uBD%$L8 z=79@9x|3i6M+42T6lq8tXu-3EsH-bM4f2aovFO@TX^2hFfEw?^wqmRHlxqv#a9%Fe zgt#p&n>8$UdCG2&>z;QDP-E%4T^DMU7Xt+~ScAy;DzVZ9w~~8XYp8J@s8I~0GFR_- z^@Uud`z(-%Xc0lPeIU1bXD6jrE}goVo=V|b*>+Fu_$IFoa6^2(ef0LpQ|ry!7kh^X zAGYH=;>cMg;=aZY+QanFB zyWSMlkHn=YUr=?OBpdbYfKU;S%<%G4f`n+d`sPTLsB@uerK5hffn&NBKRM$!8?;{a zo7Aj?E%9SWfZ;JoTqz$0>A)igTILm|oVvpgip-Dsb?Rcq3{hTouV{WuJzbhz1K zaI1dEfcc%awB)uq?|UIX>vkvq;7&%W29pS~OLtPBH6;ljT}yoF=~YL9pXh&+m#K@nzV1ktSbEQs z)H`izR63y=Rhcq1f&!&CjcmkP73!1ZRph|yLh5g+9Mo7R4|fX)x?CYu{w*xhP`O-e zKxAEGB7yKlDJ=lrYp}wQ0-r zbmcQ*O3R~!;_gmRERV<;dQ%B&vLYpC)o;nSR+sfhca-5!rSX!g-h8OK z)mCeo9&Bk7y{WpdEbTQh0@=nv=Pl`Wki|NVbNbuW>BtoHP`)2Z11=N~+h z-)wR#x}%$9JPTc;`J|gN5tE%m;s=s9-Kbh;P__+Ksp^6(l{}V=cD17hKrOUscbJ-) zV6g($HCGsx3gVboZ@`0a1=Vkr0i7d{)$JOm4S`H_zb}Ka zAgfl~-TDqanEg zp7cr8RsKa)whm}rhz_iwRdGCm9qS*PQK$3r~_#IKu~Ed#7&00Bv> zk^uzkm)JWV1-riu-aoCko-zC8s@=&XS+TF8c0WHqe;5SOphb)uiKUJ=j~4~9?zK+Q z1Y6s!DY)X2AXdHL&V~Gm+IVi^B2<5C@_<8Fl}F^d;X^OEH2D8WO-l4>hfZ~!>{W*rFHa5Pxe(y>VE09Ot5 z)m0j;i^zbKrAAMPMp6;r(h(LJ07-!9SFHvLYwH$#rGXH!qS_RC95A@*xvzfedkJh7 z6^9IOz7P4Q(0#U5Kl7uMKh8!>!))%Qo<-W{NmWmT=a5pyjWOXF{f-bNaz@w8j~hMo z8@3vUiW()k3Q0n>pydhu6dKseb9FjMA!E@Y_)b&AhWH(HC}YmhR&qwmIHG-^XKGy3 z7{$VX?kiMvlu30){@0C8nGVrDd+V5;?ShC*xS{RMoiX(GjtP z@lti0yIX#aI8uP~vVx3WZEw%8 zRBt?L?B|hs5n&gj8?Pj1C^1GyI3{R&Uc1*7V=iz0>p|1rPNT9bE=5V_67i%U#+1UL zlX`|2Q>tZ%F{RS^V$2GHrW+OC#f!wRvgcvK;>O3D`(GY6XNO1r%Eg%FZ+1;7Cf&tF ziXro`BV56}x`MQl8c_@9zc(pnQKU5f98lVZ?GqIcEyn&1k=Z9}(PM3XDB6rxC3-9} zKd^->DYxFJvwhdvb|vGqm;y$RHydx8NBu|6uZO^t`?9y^5mf#-K~q9AjUHAZ4ATI1 zHPcr~Lh^rePFsQZb|@tTjHymMP#UPu0o;P_ovvru= zoS1DD?;B0-T67pLz6DxX7_r|U4kGv)G{p%?=aKOU$%pBLbSjC2ksCvOt4l9R4OF=? zaS4yCR5eSjq)W?j(~6XO(Qgjbv+#?GVY#>U?=<4M3b{JED&ZW4*G8!6GIu64R)tb? zv|wh$azd-CPvk6t0eD}Zs3cS;Dn5iQsNA9BO&DZgNBed6(Jr1H4?aZCy=PH>`7F+> zx#9e|GWUDYRZC0|M%{(;%1Ggy&Lm|nFGR1I`m}j@oYfHXV)amq#a%YQU@{^}4CGx< zl`vnN4Ttw!%5jUpC`jJOaodcLA*L`*R6G|L1V)Fm1jmlVAfoPb<-B6Xds#3AXz_TH zn|!IdPtz48E7>h@>;;(WEeyZu}5;!W?v{@}Ckjf?#&yw707r>Z>{k2gO* z=#Rqg+cz(mP{ofyz4$!m(^a}A8I0YY7wo+F;I!Y2KOaQr{=>dEtN&{nbpPq_N+pg; zl7_QdvnO0H#J{e|0Ok;n=p}R3m&Tze;RPz%0QGM%j4e-(a!JKyxJ-%*@OpVRbC4Iu z=kooaI3x8|ZY;S8oLMi=DY3CZ49M^=`w6d?=M2JMYYkKbxL%&OzTNW7mh#68WoMFl zLZU8>j1{sC#5U2S4s@MW@z*F)sCh1Dzv|ffpVD@z7bn9Sbu~N~9dDlFW3#E9!4zg9 zy3??_mSheCP?j2)Cv-RnVpzfc1QseI(n6twu}sD!^TAK;$*JmSUYK*Nm%A)3h?JJx zV3A_Sc!}zjjmTu*8qS20ulx!*lXNv#>8tN<&JYxxl1+@x@UO=HrQ&H5t6Cl#BCm>$LuV)v-m(h6ES02Qb zjyC&9dn<}luc2;ebQ`R?SWlP-wvpO=Buo7ak zbN!ZB6xovCguay4IuIK}l5GEjK9sF+H%Q)EW4sIIHwjxaeF*0o-ynBj?<&T0 zp@ngZK7UNrE`Ow@s(ldj0s(^v%kRZ%f>*U=B;MPT~CbW+qF)8D2w#GAqT3VQ5bh zmbeY^81}s_XMREhQuv}(wu$TWY7YP9)_ruj#Zp=KbYwMa<1KC1iFhwJ%4H^SuH?>T zfaEMO`p+8j;W$;&p?ie&kg~#toW^{?&!t2MBS=c71cB(F3?1JP2Nx)xARWe5Q{GDYeA#-Zy@a%%QR2lT zU(hPkzLS8f1RdZODq;sL0COC7*lEhx!4u*dV=WQna2Sp(yCP0IKH=cGOb2V`nV*h% zMlBl^*x#!S{9r+#vX{SoRUb9N*)8nlvV<*lx9fV+l$S%E3wNPftzNH+b6Nou{7B*8 z7^qG4naB@C^%;>#W%Y}Xo8J|*XLB?#oVwidE${<1Nu5XpDC;CSbt;s`;3Qwl3(6EJ zbl3e1OoNrz+)Flc2 zT3}u=>RwITnc)OO<5>ik^DN(`ObFMr9zwFVC~W~dWiaky3pb31VtAJMFIGz=%$Nkz zNW(xRl%^-ggkG$=jQy}Uan_UEv>z77=j$KVU7O^};>7Bu{#70#bKE;{`W&cvE44`BEF#AGbffa@)1tN%W}L#g#cxT(NiJN6~I}`iJ4+LHMzWXZx?M zoF7GD+z{oeH}~DSV%OBW#b4sHtc#4?>%&jY<|I1noo*kTJgpidw*qI6zh=2k>`Fhc z2`t;FS&c5!189=i<#h`gJ`;aPWtqG+1nAZ_e9;-L5ukxO@7m^gR}mnx&Ah}FMbKNW z7BjQV*S^MWrZJ&&6?|r;q`k6qyG-he;-0?*N?m`5bH%^A_93oODq>y%6RI(}vZM4e zV9Q&h)GYJtG6VMmqtxBhGl{7MFT5#ifi~&XeiIunW(0{O5gt%X; zeT+XV$g+34u4%Ho8i{GLQiaW+5huMVBu3YRS+H2%IX9-nX%Qb^2+N%6R;8YP-zH}n zZZ5Es+cxe@^%9q{W0o=#ZQXH47Vg`FXe!P!_~bzq24iQ$2K;C^9!;!wA;})*uVZbWlb>x6PB<%Xv`QOO{SvaS?1^E>W-gi_}cMxVc6V@7_q z&ZAz>5+z?WV9`O`CZsS#)b3wQSbbSiUvGb~wvUeX4vwC_I{4e)WZ{(semDqNzQ?nP zbs9{N+sUiQialYyjRrb1rkrm=R*l#t3IMA!99uNpBQfUGs`=JM7>p+tCwoJZA&$A# z(~d~>H25jxNBbcZCB(!r4WbEzB^QeODCZ!JWfp7Kbt2)oEN6xL^Up$LrE&zz1?90k z7aYS}sw6s}Br4Bi)&FW|7KKzAVCZU2Mx}^IN&H=ceaYBq<-6f6UTPp!res!2XVoE< zm&HwzPmNdc-N#Lq;nG5!rOn361Gnr2U^Z?P&BhE;GWjku-G61M0Y{6qiYrLPTdg!2 z;)b3_saY!Cq*O2(3nR#1;v=JTuwLslavEFKFVBL>a8MC%e$-ahFXLzu&PM&P*F;tT zWgc)D3l47D(L@$5GV!t^wMbzX1uQ}5r?QrHz~fSg8WMmiP*p&rl*Maa&?8B2a~uX| zDhDvY_pA-Q{uMHAthI_-t4I({6~%)of~F^Vzwy`cGHRU$VGm$SuH+&$&M&s|3Q${(8N~2^2YI|4N~&(-+?2yA3osKKS1X^o{b<7 z8a#*w50E&*!310kSr09nv6Z674wysJXw1B6&Y~WI3(k1GnNl@A)f<)C;!a}`=M(cU z>a(kLtVgHRMo7p>iCWg1XqugtjjkDUTQK%t}c2RRFIU34R4KdKjL`BUd4&)NySrTgY!zOxO zmC^w=c0bi@z?s8)owiPfoz5g=HPB+GEKnz3(CH>8y5X>tI`8(Rnvy2)X9&l%iH_U3 zLOABW`on8W6rgjdP`?v3X}v;vOwpZ?c1G&pj~_k`$5pSpatUhrn_XLgN|s*a zriyQqr^2mEcv4AjiuK}!^LJV~>XQx}(`~whn99)xkiqsK+9qd^UCf^An5D9!Y6$Wd zK#m2<(SxUNf18cF( z(*R(#M}1Kx)Q%ZS3a&=IkQ7(%ASanZvwQ&~F;~;-I50)l$P3GLkaqg+LI~@ESz&#s{xIHlX!itd!_+}K0rx&ObrilL5PSmv? z^=3`2uoa2^zx&}oEEEG=k4-=9nZ4sTfVzx-!MpYt)#^t1ghA?h+2YQ-!sBsfvd{OBgsSSSYvDO zu54aZ>W$v{hKZ84;Sq<;lFp}F02M#9lJ+kE2qF4W(Mg;StX^=A zR5(DLF&W~CGm&)|NwbZi4gKwqm31=g2|4b-YH?;9Zm`z=eS)f^j{x~lscyG-_to;R zJ~ceH#;q{E3>o(+ep+493!>G1H0<|h@wPD=S?n6r!ygOnZ@TMjh}b&dy74e*H>H~s zb(4#~qf>4Vxw+IIhYw|zEWj|HqPh60-V}|kvaIG9-HLQ(#eF*7XVqyspttMN)}Pl- zm2rezm(RMaNxse9x}cZhlK9SVo*wSKef@Ip#m=j@Zw@4_o$3Qy2~vpQ1*|%DgZ#_+ zK9oz1J~;k{=alq;MFlonmB9?Ceph(}5@q!~{q+sh*9Kg@f%>`x%#&$z3+n6j{7ER- zSEEm+mG544Zy@V=K>WI(K3gSpoL4C~`>n4|PIbeB`m%K6Z@N5@FR4;1El;;@u2=<~ zqwvc4WGFKh<67bbos_Z|=gbw`y?caX-cjv5JRZD<;YXC$ZS5}2W*lwFiVq_>tfRIup> zQBP@_+t^~+^si*JP|4)JkwPF=I^|{dRLV~jx^oK`c}OV)AYbUJeXDrQbsq2#~B$jFu8BltXFU>^xuXt7QDr;47hPFfD1|{(~G*P4i zC#f7~Adcb;?$Ml4KP4p}XP|we6e_2rR1(bDp-@95rIKHqI;tl12fv_C=rfb}hhLh> zWG0DA{y1Oh{$wybZ(=nY4bQQLNVb?2S;je-S3yci2=JUs+okvdRrEFc7cwm=N!~MTO-GNS`hvX0G z(}%$R$i#MGwCobpKwp(LnQ;m*UHX3~(%NXrO1k>U5%3gz9`B9X!f% zHhUL(6CL3suLE6kI5>$f$^lKvG8}Pl2g0UyXz4~NQgX6J7p61AzrmEXhj#6ucmWT2xB4UW`x71g$@j}=66 zYgBc8UDZ}MiV>7TRvK|h+FMXoL)X%{%u|gB1!vMteH@=xH+A)3>0^cL;=$67s9W%O z_Lx{a3}oe9Enf0!u5#-bM|h@A66FvwkZCD#Yj}f{K@(HI44UAs<9fqVA~|q{h+8Zr zB*1my;!c-UrzW%Zd-MEQW2RfgOv-p+lf|!6so^#T6MvO#d7i)js`vO7{Mp4Yv)tXT ziJHo*zP6P3E=dONuBAlfE0H)Oq20znuR`#NB(B#yVpr-_^0-tm<`=6a)^@Sun|1l% zjmlz7H`a?h4D%HX%hn46h7}qvz_3E8ES?&M$*L?#<&lcjx{HUP9IGW?uh6lCtfgyJ z6Hh{^qtd_PD;fi&@&_+T@6I^{;X$_dW^Vlx@Fy_e#`2NZXte1T}KUKj1g!@T#idY zk}tbw;qPa^eLNFg$#M`q56@5Wp}ZnMS5?;#{dAh4- z)hMWaZ4HA{0p~A-!Sq3kQ8}SZ!+QHI!6a7lX>mY-d|R~>*HPkzMpF7I^2$Oj^l2TWqwW}s zoJ!EpQj->5;&IT=t9qivocLO@^(4Zq7J_0CMvs8%gFYXdhi&T^<8P)PMbJpRu0&qW z1zZzTfgj!iUp?}iYw0?*9Me$d7UPg!K(1H0S;sx{cm12Tiavdux-2kTviBH{ZDOb| zgl2Oiu>uz}-qz~`1uW>gbVPS)QFyi?hTX)!%}|=;M*0@~+X`cujY)>iQiE8^oOM~u z@bpx|vkleF$tWE63CkRui4v!{8|!SrZ@tf9<>IJPea!PFV@JGimVnyuN5Qoqbwe=I z1(&o)@ie8a%UaNq_1wsE?QwtkF8Np4>~Nq}ZasuDH$&_*q4g0?Me!(_o(>5n#Ti%o z@ZuH(&Y9v#Bwj*ep>41_<6+;Lw1R#gsvQ;X+!r?`34?C+f=Jwf zT{u~2ul@N~Hyls%1AJvE%|_(9pTA~jFhBpbHaOAcuC>7m>BPuOW2+pBDXVM^T&7UT zVpq4dHdt?^1`Jtig9TrnXgoL_o){hzv4dqURI@Dl=gdXrLa@r(VM7ZGTKri%Y^)tN zZ0HEr4jVro$?)JoIc*KiM6J8VD#fJ!Bfi?VHe8J4KYS`?qt7SZjkBft(x_d$=lH?phnX)&`q8_HRkn z`J-xs_14*fvKAV;uj?AAQB*Z@YJqw*Gc?E%!s*E+dOUZB$6@8nj!A1T0IBP%7p6_?wdV8U^6IIMAY9lb8ha#uMuFgO>c$97?2b zE(S-g7Yb47(Re|AqgTSH1) z6F6RA=Mt(FtUb*T8tybXQl|q@Z8yaz+-_Ad2dBq?`Q*5UK|eXJ=8kK*Yik&ekwus6jEXC0@N}u&9M;_zNs` zH4M$gXka4#s)WH9p6dJK2%fZ&<<_7z-SV6*I~NTsOpNQumW&|uC&#zhcL4RfIVCi-%9NWY(njJy<%+>6bSpl zZR`^9gAh{UN0+<&(FI;DF zeC?|1#c%f4QY@Q8a6jkU?pKz!xFANkn4Yt};Vep|ar{1>_1GP!f)7QEN6v4{p_1nv zk@w(9`*CmBrxfFdWpda$CD#-bsx`IL83QjnvMmaLXb@uWuM*w>3{b4D!+)AjCU;Av z(!HPG+%Nn*DvdewctF>j`G0!iA#uZqAd8fnmhT=%*v1W1yZYOpYfU$LaZ_j~t(~P+kZePZ`Vu^AS#8Xo>Xq@Vq0-(IK6s6L z{M&uJMMU7rP+L{HbhRbmUxOvk4qG-=QM47D_<)l~nzpi*Q%gXqb5SlJ>Qf`&VRZic z@4xR>lGT6T?>u0ie|reikMiffLA-D`gD%A(9Shf?dq9^5}c~K zx(i#VGVorvP_Wy9@4<{xdOD?6W;xy1_V7Q;oTK0Aq*2j?U5VXGk!D;}%T5M%t?I07 zU#2tGuhh&Z-k%fZATD!j90av_~8#M|}!U8htqd*g!rOSxbpGsRbGqMe*WMi(39 z;FHoupvlU|CsRpSx_vr1;(f{TgPYK5*LlCvV7`-533--S4k#<}^$5S$-s#M8T9)ex zzn9<1`Mos09wECAT$>6#xZc`9Mj zy8^BbR7)HV$nnjUOQy z33(^&cE&|Rz0;~I87FHe7+2D$7<*meq|EN^>!+{!aoUP(`Twn*?ImINSVmeULc8*refyWAxf9&W~6LlV$+z)8PSst0r9eeMz!8nSxMVpPGEqXK>sLl za#u(oa|v;!+MhS=YOK_r`VmThI`{UR2U2)9(~Gi#Tix&2uPLnlErJ%5b?Xpr0V6~` z;3r}5>fzhd7uW9Cyu#hC8k5gOwhF~lxxnqTZ64nDsbsFAurWgM1Vs!-8QV;S{XqgJ zCGv;1%fYqIGTe^Je%BzGW{7utYaHYLfVmVfWg#v0D<3?Q!KMS2!n!$nFFzJ$b*F-B;#N4HP8hp%$i?c)G zwiHWV98;{C1yUUhg{+q_caM+w@^tTg@NCq*_vrEO)ED{f_WM(@j`EwW&*8bYdlo+} ziP0u`uZF^J`WR1#Jz;#3N~KBRZ=>(Ue4DQQyVSHiDCj1V>GC@sa7drRFzxoij+#)W zl9^eYpYl+)upkyw{+tg#11#W~so9nd&CULM5gq-PWAplJLv$S+_Fr|3>`R~6lMZ&?+#TN;zdU*K?q2ud?C8#i z@)%3A7M-4shCNk>@?huFt?_Am^!nwKYn;K;(ZSB_#jE4z{^z$J9-d6@y_ko$$KJAC zJ8|u={%l+JrB`6_o(jwW{s}Z~4DmN-|MGY^@osCMoQHig` z0^PiCwPx>k^}Z=3bfXiOG|-#V+c!G#87`y5({d?$XQLA@XT7-ax!j~PB~vsns}rZ1 z?K-6x|JQ10T1!FGNb88ju0~f?dBgHe6izpAJcm?ZOj(kQDbS;POskDzGYSk)>8cK) zDK`)wS$Zs=!n8rO_4<+7ZWwk*pZVke{0sMZ{m??8S{epN6(!0;slgfed>z?DHOkJe-O7Vz8mvO49ajL2`VSwlaRjj8F*cJLI%#SJ6gtz&f4BV^>x&Qph^Zi$Q&#VW}Us^x?`uvw?&)!)--TmeMvnTg0u$b0Q zzdzc0{_E>!*1!J^@c~65XEBNJL)MHGjeI{n7LOVEH^=eh{cu~I@@W`N`4i_4U-~dA zSbvv9*wEtg5B8q^LoPLl;ZTsP0I#NqY7Vcy));rok!X+^;BNVqG#ZDeC3~hB-j?t< zR(mF_p=ME)_bEYkZ!gtt3)AG9T} zDf`hY!rPU}$@(D}M>uatl^~K+>7~*TSGFP-4$G6AGqYs_%5jJ(UtV2!&*+=b<&nZl z(ibtl4mA~BN)GJohmW4Vi*G%@f%tZK*}GjezFpllCD7d=7S|SGUK8j}^_NlrJb{Ad zb3K@jMpJ6fA7udUmb_67D_GK!J&Z2A? zio~mBtk5?e{GDu1q0UlaC=#U$>rTPi+op=dbQVFLgR5}*-xh0J(s_{tC)5Ri+2jO8 za5joY%V=`Dk}+UE|9tAazw^PlE6;N{XY$Su)r%yTWE%empP(cJZFG`L>m)9S7yiE$ zxNZkVq`)!pwv|?%K&q2S@^y$v<})}xpNfQTISFJf(LsdS#MD9PHky8n=aXKE9EYSZ zw~WdLEZ@cSHop3pcis;rO*&zj;I6|{!Ar11{-m@e z0nYU_OkvL};Fdk#v33irUC%GDc0CYi#rdrQ>(~qAHzwG_Ud!FqdaXXafa>)$P-idD zFI#{QxomNZ;9|E33PfZt(0+!mU$y`r_DCSsM!x#@{f9OZ?Xt$~n7_L?!ML~flQc}I zIRynw)trLP-$W(kdAg1e9J-E_rAnVc-|tFG5D8&?*wHY$eIM=gVA%>~94+UFW_DFKzV=*mmwexfmMo0FWf-Vv!&Noe^F4v1l+|RXWlO%gA%~ul zqc3>^*s+cyY)K%A|5NlRXr+}Lx~!U<3j9rx!A$L}z*cKj*L(Nw-+!Ph!iMVVsef$C znWkB+gCQGCGu7u{=YID*h`jUHPxtP3 z>~8<^Xv)Uxl$E6<99*kNCtWKDNu%$@5R#gDr=cXXi7m_)kZOX7WVthmUyhPAz>ngk z0VTN}{K!s{ATPmmRdJF!@-id@nqV|rm7afG7?+q$L$QK3>nk_vHtS1afLE=rY!keJ zPz}6GI4ca#z(vaSElx+v(aY1{{wbDO_Vm(CFU4f*aO;ci3?lUs`<`nVfbG&1#O!p5 zBJmED>lh>|hx$dC&`__fCT-N;*Fl{j0;j}4!66O%!~iey@S1{YR8(6PvFiB35ccIX z4_N;W#GIVQZl8Z!$+hk!ml9T@Q)^$&wjM5{S>O8i&!4!>8PTU% zOz>&G`1ikmesvrz?o7{LJ$?Ld?s2X096~Y`OBJr)vZ2Z*P_d?JJAE*OQ88%yVxBmy5MrEz9eeNTFVTzTof|PyFL<|- zvG|?Xkfu7Ka2=vPw~$1DC<3>QLtstt!|clzV_dTm{LoPp-mYu!x{m2|S~-Y(PZnaN z(hy~5OC;vr0&z)EfK#gTQXODmEiH1ph`8VmpIPn+!4QHku)hx%qS`o}IpmSE$vMQ4 z)DOsyoeNhgwSEUP%KBRGmvd_l4?H=5az6X3wTPqfwrV(v!E4ZRZmF}~RwbUVXs6;=^av>P2IU^)2cHKVVPW5&d4B24n>M> zstCXj9CRECMua*WaZM{zT*OUUK~)~VfkH1{EQZhCNt8C{na1 zXP;h8es!Mtk60?%+^wEBlvAaYOwM~3oII{&OkC*-8^4{0l_-9@p#9~3J2?M6C`E8t z7*G5#1YrTyZ0H~HyT0!cSolZyWNU%IBp8T`psdYvY zd{R#8I66C#;uwAFaXi5cp(=ye8}lW*pmnr_3>t$(EF6%;);t~!hVx^#FSbVOr&q`E zOB_FPH^y>ROXJ6-e8uP`3BrLZLwnH0LFP0yEq;Cd^v>RG>lc}21N?m)&9DW}qXCmR zq*XI7-r|poXJE6{bEQpf9CiM_d(rafn7-TAQzhrNtzVW{qt?qIhou4n#3u7$ge=^) z9!{ry1SD_&B%*rR_PzMybhH#m7J#G4LlOHk>pmqK7U$b6E9#|*s1$J6j3HdBi2@oF zh)Kyrp$GTd`yP}xYq!c!~@I5c@d4kPi3OQi$_TnQ|@y&JT5?I5>O z9K{W>=p9Z6gSqHB$plLx%He0M^y~nr(L*X#K<7iO=ce)96&y$zdaMamWpRKLt%CqINm1Q9ejQ@-7XY|8b)p6$GN zY2SK2UL4+pMRd`-eV+!)S}t4NUFgfJ?v?1Ib9Z4!xZVK2ktkA*HXY?zl_#Tb0sR_3Gt~tDJW?=JpGr1`N^jJ7F7M{LnXQWK|n^ps8ejfxcxu z71b+=mkN5gKSU*59xz-%#BBn4eG&q_xNxnafP|Q?K;`8kj(b8yNDa}#$X47RK z9nZ+L5-@&ON_$X+^!9D4AWhFjA4D(wOo=0`Ue#ZjvM=`@J=x2VTeVS`dJa+oxx}`)_`^5ea(9yIs+zSlyJwr5F;v@Z0Y9>rynLc}eEClBd8V zDBx>%I*3~~f^s)KJ*s2$Wgjy-H%15%`D_`b7(^lRjSFigoANqUZkvjH;(T}Rzk0d% zZtsb8jDs<2yv#q^Hq*coJ#$x^7cf0(*B}4}+(J-;TT&{Hn~UCAWe9F3Xn+ z+TxN@bc~oTQk2OfCF^(=5B}?5D<=&#TseJA`_VLlm+r?k#mPJYA1=A{K`RLD5D$$0 zfUB)Mfm8)}emZ=w{2zv8u8vY#o;*0ZYSR4ebT;arLC%$Q={3gWr zLxuS8yLe99B~clET_81W#PQ_p@oeUef4hEheEGXwQ5@IoA4(qE)JwJ7wfV}47pP=N zf0tPwIM|F(5h{EsR3s^8Wm@i)Q7f2~6KmP@FKbyDt!>1dDiil5Ss@*)2MR&ALLsH$ zHyFNl=;HNOHmc(OL1(+|B4)o5dv~G$>7&uuyAP-5_s8v{`)A#IFGp`4wJ)Drk@{tvOnFGLXtDUN5ep;c(JS^N@(G-$I+ zYFBf181-Gfqw_maVew+cb7rKYcfq}%MTKOR}7}8|AH9h zT=i3!FTh+1{3T33APpwam`m!(_XEiW8VuPxkj>BwY%dgkP;If*wDLpKH&=*?mOKGy zie^ihY6F>-wMwI8?-baqln!EMCY+v~oyoMSD;{yq?M2)>KG@;RGAV<0S4q4R`VN5A zWY1V5Qi~<2n2^tWIc8*XjR~aLM^s*kyFztsMdS?iCI9X?9-UgJ@l@6x$ntoGAB<8Q z^8dU}`l;XuBde=c8!3I~*G~@2)@;_#xEMN&ZK3SD1MgEphP{IP;a5=U@X|Y=F8xY@ zn+-CoaP9pFsZN2pTHBP%}F3SD8%Z3oc4KNI3|3M+N9?ME|z(hNht;fj`TV5 zB8|d+hK8NA@!1x=+k7RiPjWN&P9ic=CF|}(w6QNN)>52 ziy$RfnBnUWk01RSeTu!=?_7SZcD*74;zwmArB+O*4Q&_Yl{$zfSYu6XyVA-Wr`q=W zsDC~o`;nrF)U&V$ApPZ!j<2e^iTz>f-=98kh9IH~#e3`T5b4C$!H#;&|F9?y2kXS& z=%aSlI?@7VaQRmBFw^PTH7FGxdgp^bW z2qdM^6-`+qCEZlcNKM*IDf|&MWy(4Qu;C)vS{(?W{NP~cwKux+=w82b1Df)s?{;NP zxw6ZMrraT4q}}#~ki4W4e5@E(>lCajaAAH6-TbOT_XkcpB%Re^!?yz)TqLBMhDX(@w(?L@v5VF;oJIdJx6XzG zMCX!|m0Qv9btS45Cuy?Q`d?%%rgc0k80I>?|F!0rK*vKVJtdbq9nO1jzaYqcNp4&z zhoyorTzAoE5pDDJWE&MOS{z#xPmH&%^X09h;jQWMHXKwZ;*X3w@D(Xn)?#{oGPRD5 zr^}Jj7vQ_UX3<Hf>Z2i=gbwjlL`0 zj)u=65zQv3Oh&HhcNfZ8ym%_2kzuIP8cA;qZRB4$y-~+fcr2QgmACXXp@u77Drd3u z_%_SCl(SfxpY^ZQ_q4n(geL=ou#NT0MpjK&L?D*(6}v+mJ`30bp^&i_zIQ`gHzl=flJ{^6vE1Ku9US@(^40C$@lto!{QSd-C?5?!)1m z`KMn${`RRn#8R^2b!L6mPM~Ccj$1DtPoF<}{cLvnxcB?<+|2qk@)f_IT`RwzYhzxlNyOvDf~LDLxY$NzVrZ(#a_L5cO0o3CT@Lv>K@xaf>Knd@9gM$~vihC9;&V zOC|%~Ci_ge2&pMHgPxj^&E`}Hnk?qmo_Qp-I6!KY1RtW%gjEXNwCJrSu9AagBYy1(Nm*qj$?|+Z}k! zrABtSTka4AEN1p)Pcb?-ssmDoVmBzZBpnA@zOW{K#IUN~HdO~6O)QWLizVTHi&A~J zt!L3^!1KQ>=W_}=O3LayCiUVNJfwSD4 z@hq|)^WZsZ+=$|bs!`h5rf4n<=dH)fQxb8&ef}Da$4iCkd>qY>$3w}oPzp=(E6qIV z{l&6>E-sa)`**j|K&WpK(c{?q_aoLF3BeDBWIX@ddfr9XQY-C6bYlHq3n6b6m%mmM z8y>f+l}##YS>513RoeiZUez~H)l2$@==y>bbzD@AUCP~;>&$#}`o@FcLHmIe0?Se> zG=9OHH4k3fH9hj=-1>e$DpMgDQf(^S+Z`J?za5awaj^61&HcgscOUxKt^@G?U-oWS zR03*t8kO!$;U-DMWDR_uw)`qo1=324rWLjB|1^o45l^snDxbBrHyfW*)p<5k*>NJj zVVcl{_7lq7OVMAcH*`!*`r{F0fRXU&o_u$@q+;~!G@6lPH=0kalZh0Bj^@_E)`P@5 zDYtww>>^2|9I*8^njLH@)xhb5&bIYavD3dn37O8$D;b=R|M}tm$LHtc-W^#cX>N5% zA8^AC`oIbpN=`mqxfd5KV6bk>*_P{kVFeKsEmEUUbkS`>s*_dJFr(ERfgspyL9huZ zGtv^&RXyGk@2Ki?uyeorY43Az=e56gfA)Uyds>S(zd*jv7KyQ!xlY-^u)wPT@qXPS zNxD`*utwjDfnZJT!oY5n)mY+}$I}HvQh6fhigryMVPd5?2dn@9jUb7Dy+zT5Wq4Go z=2j7|b->lyjOzEyn_Vh8PsQ?gBk1b3ls|cwVw*B>$1ZlZ!3dNHX9Iam4f@2+olflE}~qE z!faEq&=TLBvHXPk(O71JH7EvCvfnphdFm$uktZ(ywh77@ASqyt@*h(-sz2ZdKp)o*53;mXYL|U8jk33 z18}>Ce|9{0AIa!gC?yD{1~-f0rOB?yuPII3P{)p#+M z=f>;&b(hdL==a%prw?`xb~Z7zg?MJBK2SSbgK2HlU6dbo^hZce)%KsrsfllDUsF&oqaw;+8IxvQ&~#QgEo50wRVAe(M71 ztpOH_gN8W+2}@grH#8EK>+yn<=0wRH*6;ZAa7^%LOSpnz#-%(}tL3Yb1xCilWx4FZ z^|U)=6>GmyvaRx*;-r-yo0`)qjqMJ)-{4=p01UAR0t)+b?Gmf+=%S zibunf*m_OUv32)2j>pmVFVT3|x1J7X+xv7GTl>*qxcxAmZd*MNS#v<6^JVYYnhpEI z-e|g%q-0Q9b1<|kfYrzI@SYvNdAEPU=rX&=08$YM`IeE@Nmd(Akt$=JPrX6I4A9)3MgZ6~0wDTT!;1^PO1?<=FG24$Tbx>k_17^xpgNr#u?{mtd=7)J0bgF{{Yr!UPKfU- z5&1NszS%4NCx}rv%Lgj&6$o>0I-4y|7g+&5C9L6ZzHPmY73kSIo}Q_CBU#5W(mEn7 zYEF(+pJXYiUQ?9*8_uk8JO&kwxF&D*^mG=}0l@g}JIlp3c`v6@`wx~8I?j(lQlHGX z)#V>Ndh+TQ>+X~Ld(ZYCJ$nev`)DNf0~f=|GM0SXRdSuGBLDAtbi#^w%LSI`IhIm% z7>EeFTwZFjE#hcBBxk@3$Raie$A>PE75urb|QZEDByZ zR7h(q^#${dOF97DH*G$sDQ0pwR9cMzOKV_Ln~5T8Ez-?AG_lfvt6drfEMvHiVqX-W z@!O$P9@FMNH;}SceoAT}6^RLnV|4KTZvW$>-|tMM!bJ9>7-ovSy)OGx*ojPu0ueemG-M{l;Radh5=Ws75r1%+-G6n`6I<=(aqCXcMWCr|F* zd-%@k&!RJHJ{`x((kGipjn3Np2pgA3S?dX=bM(J|lQouBX}*@KUShA5Y#k}pbpcz4 z7qKQClyaseQ%BN475xOJPJ7qw?D~FT>U8)$-<+wlNe6{}f^GLug9j=7Khwr5MnMXM ztxM@L`eJRc!@FJ^If69v0@#;2&cTlR;bCt+KfIBeUYEVwmD52hTaBcHcECM5fudRY zYne6%+W5w_SoC`(|Al*pT+0*!uQ^!jSgw`dlA56&?k_6S81f($+C?iHkgl@3mrhl* zkNn>B_0zq*H;cQ=%h&eEBl&gLsgLB>Y7duAo6fYX2 zLgYlVBWf3@j7Enir)qIHA;PGfkEZ87wxrB06sM5OR682s*rsd#aTx~Z&GM4+qKR*5 zv%JO^ShC6T%Hml~mgE)dAuK0-Q4CiGiGogJU2CsRP2_+mkIg&YrouAsRIWo;8%1Xm z3thQGh(uxURqV7NJ>F!TwL1277-#T3IBuu2DteW+%KYGtLsR+bVA<>R?1N>u(}PmA zH*h>YNgU-9UKx%D4xfF`QGH_Hk~3Ovz$fMG*Yu@1mW%YA$k`qCC(y=-DQo@Y5O?nO z+MFNtY_8mNM|Jl;{aqPh*Ny#~9(11|1$Tl|=jUPn6nipi3E5HWXm;dO=nKw%e-xj* z9_+n6W>#02vuzQie;S)n>VMV4UU^Q))^1|U)i!?hgb^GIaEDRaTz}_^gG_L0t{yC< ztb64%R!7vQ2XnCKJe*_NwCCsJ@oa`BO3R-ZSCecrli#<(JH#Z&HfcuHI9Lv;NvSR) zpawOEA|0|jmgB%Ky6c67mAMJ19nGLM=_etd6$sN(6h|3+*lR)yPI=ViznE&>qm)Y_ zGwd`Y@voXgQkt4gKy8jWmTjZdzol|~PjTU3pUK`zkmD$xW*vId=?t1ZsOHmVqKee2 zN7qmGVF~rjhTSDhIeCXw0kxa7I&)4PWb>u;19ny2goJJf%x0==aIFh!4#rg+f=7Uf zzW6$zNm^;n^~$PMBB#&C-Kk0|=MQC(>ckq#@N)vRJn_kQFtl>zP|zr14{5E&?`Zo} zrV6u*YGI~2SCMWC@9n?t?d|pZuY)f$sVmp1H>qniniVFs(f48|buNHw4eCsRy+MQ8 zlXBab$%=8HY)>0za$`^3@^0vOX>(dzKAXi=sMedsRkL1i+?YO@vW2a-rCY5=TbhWf z>)OzhdY!iGyB$gk=u5iV%FS|X+HDj5#FVx1ex(N!gJi2{X|Sjz`juMLD+9{7-NCO0 z?a;A1Qa{Z`lPbH(nj>C!M@&+|ILx_~kghLCUz9%A3_ z_-%p07XudgW47u8?)aV1v)xVzPBsW#R|rhad9C(n8g9^PwVQ_vVOpC7K*?&G;d-uk zM)J{C8}-3D@JRbWNidVlG3E_TLnHL*HV+L4XuIQu@Kf40K@m^|h9s}H>Vsqbv7B2i z!A6>+D_R3JH!{As8_L>)jVX+DW#|v9vlc~9F!p^I$ z`tUkzTCUAX4Nyc|vL}>>S?$p%lw|Z_bM>LW1kA$v)aSC=s1Me4I{_t3 zLMh!~KIfvPVQ>I{aOE@(j@f_)8(Pk`?+d|xS=Llq_2J=swS71}C5He{Y#^MB<}j;0 z8iz|bw|Te-Hl_n}sN=Lk9K?{N7B6_U(HJaykL`kw4|r{C->DgFrANbH5j$Zzw}uz# zvWXJ{FKU$`Cy7Pm0@X@e`60PA*y_djfjyggLwvRe^4Z7J>)BoPeb7_i@%nmpjK`Vd z;?=X0<9+e}*)8$*L4IBwZ>wu|JaO5|(O6wEy_V`Heq`>1r#ruvx_XP_oqjTop1Mk` z_35hnCF9^~Tpd3&mrri3I`*pW;V^tQiiU4G#|>=vo{-x%xrdwJPDAKWC-K1+Xq(Z0 z9c)degXw5AJv-QMF;3;G~ic@<3+G2|kOs2*%X9gEJpR zZR1z>6(ee->Ma|Q8#-)$gf_G{sHT-4ntP8!n6Y`*cFYrR^WxL&D(Fb9q zUIqz@U{tElmiqL&as;qQ2q23^UyM(Vw|JUp%4!uR{aeZRmK+b&6#})$ZMCo?b+v)& z(^W^37bLlkgnxYMt;o53b(R$3Ghd}3dHqx$>CYI{5kEPiUf_0*Vtum1Npqksk-V?T zd!O9Cr;ew$H-4%*ON#PbE6U499V)rr1D4JOP+l%Vpz3~i0uN2v*g7-Tf2s^o&%^h2 z{UCS1Zwl=FfCcti2s!w|sUHJ2!^aXj*m<{n-#(o``EV1g|Chbnm0@0$Em*<4JY1l* zErF&D%t?4h;hS%5}mxTEwg-vqSd%R*?X<4 zz{1|101LbCJh8i9=EJUBr{0HMt9GsMVHN{Flj zd%3B@yCeTCJG^x_b_O5|2)UFxUfPYlSzakDY_q(w+6-%#*A1e=-b;2vgCiR^(*OAv z@@f<>6!0Svd_8m zTHl9qu=@h9rYec}HTB_9B^`gI-l`9eHkE1QQdJvoC=S;0FsnTphKrZKv(EXdssaJ@ z!C^=xZ`_%t8(V4y8Lj$oafUiRR#||{mWLtK59RUJbZHo{M;Vtj^9wA=MpEL?bBOyw z5TTE^+NckgU)X|0y6U6J8=8jZv|H<97lh2CM$)tF8Lr@W^umv^DZn z%#~TRG!0Ke;A`Am1V%ikO~u>}SW12~f+g2AaMzU@^}(?&>=x6`gJ#aDx?6D(S9>%L zmUvC`VDZL1kIAAiO zo9B{3^t{9n*@#{*5>VgBtBv|_+4^~(5!>ZK<_(QQBe2vwG=vJPj!>A>@|+G)Pkm_B zMtx|6V&Md08p}h=8=8jJ#*4gWXjH(JP!wKUAFl`on6U(_t@_~T&636COKhq`h_Ct% zS9>%K7oBO%yUa+%b;JLNOCoG=uIsC^+NuxNaj4e9p3e&{X%mFuD(`hom!<*p2Cmx~ zU5{LJ*>psa@LW$%d}vo1_2IImBVhy$QF-=xOVi*aXLIetCoUTXPA32XfnFXUA#&x% zaqV)eK0Jw7Ls;N;a3-|DD730k{F*Ln2E67S4;5xW6oeKUB>`z&2v!^Q;RYfyTcJ25 z%>6QNXdD`G;pVv%d=2uC%6TAm*t!iMQMFMY8uAb2Lj*y2Y(-1s;Ml#I2Z#E_!}OT} zGF$;dU~Cx=y3%S4k2eORg3vgQxDZfHSd}@|E{y|bt6no;F%_t1&iZ2GB%p6f#%ilR zU}9~k#=Ldq>C79N2It#r2Pf9Fm-sPW$XV4=9ff(Zz=p*FMKi(-)P zIMqgDa42wu@a)R!Bb z(l|KY&gQ`(_yV?QmJV`SD%TqOOHHdjJhnda{zzLYk1lU$8XUNrHM$}m631gykQV|r zY_nTv54GB;4-HJAn9vogG&|lruyD;IWlL@Oeu$vL0|6S*;rZl`wA!c-3`G*84tzZ2 zIm;WGhSnxXvF3q_!jD7CWkU_n({SBsV?#|Tx9Wofr4P~xFqV(4g;1*j`qgx48ZbNT zx`*uY%A&?FGyq8GU%I)>a7&HGaK8vE$@aSLF{6VLy$^-HAwVaQ0DAG5D~R>=9dWHEWlmS zs>2pHLgHx;zS^h{&BGYLHVs-8&rQ+NI5>>V=E336>fi<@5Qc-02W_~#tBv~L0^DsK zkfRkboHsNMjoqYqXux$bMsT>X_c-u>87`4(qcJozeIEx;MGB(}B@K;3Lq%>L8oH^h za2^gGCA$;6re7f1*m19jrhE@)mClPhenb? zD-eKrd1!e<)6m4v*gQ0cQr*~@u=+~y2XD6VL~C00!BwLtPD6P}wswwhLbaEL~q zF8ET)qbpjPhKG*2=0T$af+!TL1Ih+AJ=f4^DvieA2qPhLh#yo2m$x(x4uP}gvC3A= zKU8#FhaF5seQ?!AeQ*TS@V(S2f>}e;(8MFq{DQF5uz|@qkk`aRdD4egZPbS*{%gEk zC{zHj**2^DOSMPiU|na8~pxMt!j4Vi1$yHgfc?0DnA*mav9FeVPcJG4_9C!oZLb;n?} zQ6Cr}T~a`Z43K_-ErK(qJbBd~O@qaczs3q?lr7#vlqcC@#A?=NvD&B)RhSt3Eh>397dE577p|lo7Bh53{CA(|~G|B4`2`-*QED=%@Yv!`jrD4E0 zku!3kl>is~W;kcW$soRJ*B7W*zVnD5w2mgWFpP?sM@Fx z4y{81sp3d6$CfuV4h<8#d1yr9Bp8bBg?j^kvtcw<8}*@y@s96ZP!1jJB(Dk+s`h9a zEXvfHhm4j@DGd}G?ugCBwy7=2YNI|_@ih{_=u~hmc|*g{xLaqVst!VkhE$EWa*!L*vlaCXa#$ z5y>JRYcde=)98=6+NckW7oUkouPqNPZ)hAE9--zp8PP?2mJCx8pRfqfy0w?1+NckW z4FLa^BpsH=Rkh&69|)2og8;6N6jL7*2^UfrUlX>hFBHC`=A zs}j%z9`&|hR}3|$+Gq?8i7qc!#h(svGHGZUn&35>=Pki;1ub#75naL+qRX|KR()`I z+;}xHZJ1!(%@V(_Jl|@U#sQN~yJo=hq6tMk{xmceVkoSsLT#wF>I3GzZn+4`$}-Fw z8V83EZyp@}O=N(wSx8UkfBH69ZPbV6fDI5lQ_U3SEscW%jM+RmHa1+4B%n)5pDahi z;a}5g3=d=mZVF-rl_#9HG!AbqZj-nsup%^TOk|;9FympBMq_X!VzVz*hn6018Wh-swrd1!DC_6ZTfD52~uw+%r zkg1^7bZHtePa%CljKs-m&UwLHs1hfIW>xKj_!6)>cxpI0#qV2eI%xuv*2+&wbIxDB zvVJ~`qghPK^!?wT?^<{E?mzzJ-Ro!nF{fb9d>$ReDpQ)aTTjSc?YBL$$NN2x4A$&> zn-pL15~dm8-JVDOY@bAY>!<8$iX=7YbG+w}1N#$~vwqUpR_V^!$~g84&hTi}~1On~zvCdTBw{VLUlZGQ^X7ZCT)X0tiXw(EVVU6B91>6X6Yi#6SkBti3fGpj3!)$9rv%B;p8QcbMJ|5?hb7Gl$1mlMlzEyv!q zL%2;hdDT#v{#Cg~PtD?e8Ov%$$BH0;e2Z`?Dkx$=oH_?P=g!%qhc6xuZXmDv(s#RZ zUUg-&QF+z)lyJcdAI0~XS3QeWF}GOI7|~)0!RfzD-UMCzJG(ZoxNW}QnfNDyjg5$L ziMp#m8ix2UwaPY^u$^b}`3I5qA0OTM(7;OW*cYQEpD&jE;Z&+<=CuyivCTY<`&38V zS0^Ip{tA3Ecv#4;Bt!}D8!aF-z|qx47F{a;QU3VD6y%yNNL#@nlPVrFGG@1=Zki(g zi9bzm&&LZ2oJpMt`kO>DqFnx6sf?+<`48*)021OM#TuzziLz@we|8_;Fx6%KzMR1c zc7!I|1=d9yS#}%&lXvB!T8r5-M*b`EA?jXcg~}zM3re%184zGI8CM8s<5sB>o-^rH z)d05v3WH%fWV!*Q=XPf|po(x|Isll!tXlLAr-Q*f&e@tJ9r*Awra;DQ1X1&0Jc+s^ zp+ZW2&?utxYYjPb(J!OdpC1;=)Yfi^b)j$em7bU;okbVIQKQ*pnOqX!J>`VGd!nL2bCYuDs?6VguIY8C+56gJIIQ%vk)k#-o=P?S>65+&I znW{0_4hJ4Vwy2D|AUlk7W$#;AoL1v|DLe?wsK)f-Z+oFmd!A`8X0P_PZ}ie7fzgvR zionsYmk6ffhGdQhExm&&rL>OWcFQ1kPrVt&?a0Cl4c2z$=`g!=w%b$%)`{J(*$ZOa zjx0Y>dA^W4M*@og06fBQM4k*B4E9L5)ffPfgl+mj_80a0L#g2wjSdyj`{8ghphf_T zVn|s`b{@_%r920upme`4(kT;8V`SH2_&GVG_OU-w%KD44%b%T3`yuS|_g{4V=nI&6 z>aJ7I%%l8JWzBR2Gmp{tV$3|+nl-t3N=CuK&POEAl>LmEU7VlFq{F1R5WegA@UxU> z;AtE$$6fhyCPX{D%!+V_|K=n*z1j3s{G29jys#Y56*P_3bue5=Pn~ZESJLW^dJVXS zj+Y_kDX&h`g;j24CW?1{vvyf4#(K0pJSSxS>o>f|teS<=^M ze5wix%`%H6DVeSs#bO*ECAIOQ@#$zdI6q9!s@W;k3wf}^NIsGOa8lT2TuU|f4Ldv_)m4Q-Dzlkk<23in#qc|@GsG7FQFcPn@1cmqJQygYJRPw{UCr|fwt@{&e6j{9` z?lTd4e4WL;VLYSy{Ad)7qkc5EPT|!4IGm2+3Flh<*g6tlXWW_4trCBob{^dO;Qspd z!zU}UkgXq+zH>S6XI1lS67}N+SL-Vsl)8UX7A&Fh?nTz|EFQq`oqGj!ckE6d=iq1> z^$&J_R620U^_28-a%AAh!cD7V0;-KX{q${l884#Y42K^p-3YcSZv^YOXfOPElcjryY{{N_wUv8Gu)f^==4D5JM$15 z?oFW4{L@vqB^_PZ$w0ioY$K#4MwGSG%6z5r z^jj94iFrDncG)B@714YfcdfnC)4xbClNTAxu1)nB?V%P2d6ubkEc7C4Z!|^=lKP~P zs79+4AI>%9XmJUpTG6h_(K{$o@28))tr~$^=sIU-XSbtVL%S6*lG{>;mus0`(kGFX z-A+koZ(@l$XH6)vDr1*{4@IBr4YPLCFRl^IW<)NdnFYG&-f)h|hJW_sRFt#5<)`6j zIJHixUT%$+V6bM|G`!LEzlX7?ez&ckUd*QFNcO4RoIzf%1_1K0q63hSgUmOT07=5F z(Ea(MTtWFh;asJR^d_Qulvj_wwkp!3Mz!wVWszkf zeuc_`$A^D4{S1|((#udeDy=uAa-fs~I!BKOfGyfGB||U%@&-*VUDBuF|J?nOyiKi z+Dd%XW!LTkUo(#5+|XubR|btlq>Q4H-K~U*)bPfhKiK&vSb5g$9&kxQpNnVWf5%*aWjoNRhdlPLfG7g>mTzGlEpmBROA5;c+}2$h=pX8$to@ zNXVi_CfNy^vH+=NdXQ@rBN4ZmgWK#%exoZoN?9W~rmkr*rk1n57&)|AjCiH$-j15Z zNFo!#!o-oeW;JHITIzKxR%6j{3AIz!YO%#?YqdCNtW`Xm35wzCz9Qt`Y-3n`^>?Jn z?b;&i>Py|OLdOMRFmFrAOw2NMR4Y#)H4{hEkFj+;J+q?G$eIo;aBS$+U6kx`JnJ1> zXzL)u&aELX(&21j^^W7{RMI6R_eK&W23%{C$8yO$7KywDWdFl0!54TN4YR)5TD+4A zK}sG=KbF#eph=VOxIwe;EkTl|$Mx@lWV!U)HgMB^zav;o^Nkw=yBLfBQW&OTlB%L^ zB854#3kQ;rRor~1_)AJU<}GHg{CF#t6$HVBesyiwf&Oy@Xza=H#;un7W41=Fgn`FQpqvvWQ>m*jocYgRr{SmUmvR8N$y-)cY!`}#?R6(qBZC|)3p2oA%W>4Q6)C-wUw|25|Gi|0?QdRX@9V*a^h}Jh7#*==ITX)DQsyI|q znCDWo!@Q>yj7Bpq$5rsC2v2|!$$W}Tdx}VNJj-@>ZGYF(JSrX8ZN4##`X;VIEa|44 z90*$=#_;*IsQ?4-JN#7#J8ggV++TctcI{q{E8OjhZVn8dHa!a0XYSOtrz+#d68O z8Ck0gBLQdpOm0Dh&;2~I?*6i8J$!3De6{!5+JF7@#ojN8oD@|rb`lw+zPl34&pa#2 zfjJ|K2QbC|kWuWpGHq8xSCFhfn-2RIsPFJ0MG>GP*XFS(L{sFT}5 za0a&&i4>qbp>As%P9@luO0ArqdZ=b-HljgwNApCvm(N~L)B*F}b23X7%V_eM7A@4o z*)WD!^Al?^v8G3&kMXyRZw5-K2EDQM)8m(q_Fq}M)}#Lz6E?Dx#Yz_Bu?4bec5aPG zh#bWjuH5i87rIuh1g8IrQbIw2vQ8p6m%1R2%xgs@ft8TMhDwr@DxU%LupECEt;F;4 zAkU^$c_}w+G@P8|>1EDe7V80XCFEFe5b-vK0xZ6}HxFJtx^ojc2|$ez{r>74;hG#ZG_NlgQBj*tnXn z=tynSgN8S+2MNa{wMj^WE=h0YEzMd*sHA$(lQ=rwo8u#WKAldV3q-VD!(mEVU!dW9 zP5h|nH?|L~OcLoR6+!JRQwK=|n5o~CPLO3()mpZ8JqQj})xFyGRIAB5m9Apy6WPqu z$y8;8u96=hL~-g~rYciu<^1#w_+K0WB&+-{j8UVgck*QTF`hq-gk6C?=FO9yPvGC> zf3y|~O)XczDVb>tLVxqc`6&Ke>!Lg!T1U(C`QNNZRy4Mf>)*DHmqL9Np&1RV7s*Bb z_IFWj=Hh^lRMV-@W2tlhk>FDC)Qg*bHtmk4pDdXi`GLF1HG9iN3`2*t`wy$TJh#%3 zi4NyeP8eOP5PwLLyn}@D`-bF0u`^2;M$-KAE3arc^sl#Dbtd)6%B1I_t z+Wkq(iFjgGJCs^5seftMnO6+5P1)!3pQt&MvQ`;Jk?k{|Opl{z6d%o^UF+VvN6#Kw zzV+_;>zCG(=Wp&?x2#`ZKY0hmT;X5pbvl)do>%wggLl9G){kCr^{aZ6zNWsFXbif# zR8p|29a1k)<{#0|O`DQsRXx*CV^Up3a|B5lX>2mvD4$szWJc4~{`7P>i6;6Y_qsCQ z2;rQsaqIPBdP?r{;o%`owJK6F0uwEK_ZUCU1m}&l-#eZP`d73QV#80PIl9=~>f!Gj z_KEmf(L&)9UQbZfW?+NQMNisa&gb!`?4-Nk8OS5Dy64s}s7w!E?%lngwY}SW^2oY7 zolF1)TX*rTss<@#l=avf%%-=VpYpB@=L>$swQ529D0I_GgiETMn~-4#d!(ufK_Cwx zuk20og`v8n4DB;d8DzM<>ybHBr0S8kjRO6R1-L4R)*pd{>3-X}>p$RWd^?cglB4a4 zX-ecXNVzkq`3?g*thqZ+!bgkuuSdV#gzM_Ece|qIRcwl)a8a^DD#6#1Kz<*n;kEfm z;*u7YSI@Q_nB4qb1o3~$i)yit34%TG8{s=3%U4M9F(|3DfVF050Hs#hQxgr~dIWy0 z_r>0JJKu@(rAA_#(WGjJ8&d|zw3iy5B4vt4m>uK6rK-R_Y8!uYHXe(s@){{N! zUjjcpOX)(~jBBo+q!*j{x<F3|xE9mfW?hm6yMUJkcEO6~(qxm&SynwkIxeiVGa<3_hH`xIejf z|0cxtW$$)Hv7PT}N@}-xZQC8m46nFJz7MH=A@B>;rr25x%0v(y6~;s=oYfE}5}Oto zewCn;>9|NK@Dz=et}0Ob*r!fuo7?MhMuHG15?U)ysmN#{uQCvx|_3Exu3?bQilB_4g8IkEGN!Qb1&q#ING5W5|kTM8H zG*M^L>Xo#tgB?Yp$C_snznoT2C(CO|t!h~H)PHjVzLZtZ;NRob!;TfMKJ}&CS0~nM z?E1$a@c)oPM!s+#(sF%eswcf!VQJplW`zl5Bw)F8HC}RsW!aB6Kq3aBk0pGe&6~H& z*tyl;im*SUwB<-PRPB)dTGS?LvUw`Yp&>0%+g{i=Om$yv;h8a@Z`|22bGxpgt-4-; z00U2GZv;Z&LjQpT0@jp5j^gx|0u~?>l5$ifDOl~624587!Sq_Ndedv>=5?+*mYcGx zh3-__On z$#L?#I06cv)^2hI{#N}4q;+wWt7ScsBW=~&r%$IfU`*{OR9Er&PDM=BzeTPo-?@_? z1EpAoeVelaug`Hm?sLSCU69~|0e%BDZVHZ+u1Iy(f&3!*6@>S0k%)Y&18hrgkz+&}m*^IV)YE$tqN9tE*P=0L%n$gIZxdwHA~0n;h@4wpw1Y#wFvK zp5#KZ7S$D6YCOrg>GjomjaiG_5gbO~3|NOifYkeuj9*edCI3YsHY9%`_PH`_%zS56 z^6S#Pc4fKO#jNFgKY4G|XNxmiU3n+uz1DYtR{PO-h?)Wui1iXbw)C(ab5}@SS~aMi znjU$rve!F#NxS@%R!g=CseZ`M@?F(mpe97xn8IMd>qTq*6eee74^Xl?nL>5nAlXXP zm6Iut)$!!My*5RSBq@&1plb)X6y$-l2K-bx@DEFEgYnAK z0rb_$$!v7B4KUd}lZ7TvA=^&-$+^iBNfv?jGqXX6EFlGI4!_KFb`2z?9f>!X`LZxW zmctUYK?%uMDMus#@BsWZA`&VFNJPSITTW}&Y47?aj>5|V2~scXhJrM6@PB6%q~yxN zSka;GHHq0R($x-j9=v+?-g)o!ZXif-;k#A5`AI;ebojfb2qh~_qVV-0c(lV#$M-0% zP6chFB%YYMrB>O75|vgqu#E!yl@wS7YHL;Ci*Mg`$IIv5a4Z>=S;Roa0JCgaQb~OA zbTz5EH%h3Qyi7XiQ1iP~@dnpadhI7@+FL9;!%UTM;yiom=iL?yH|v9+lM=Oj!}0QV}wx zk`nZV-JK*#k{3%8z$Ceq9!Gu zqCjE^Dy2xQz;>^Tq7@7}e75W!dSK9&`rC<8n%KbpAU!LYV)dD*prXZlcB+giRjtos z>$G}YU$Z)wV%OxL5bX!U5^}IJj>mHa_jFwRdWG@HFAIV$SAtJlba&&DTHNu{dJf{%@hw>7 z{r0X~h>>p+SMuQA-3JCqY90K8g_=L@3*kN#tmKq(hXtCV?z&fnl{8;|J!${L}rPJKf-y&dFzc z{@__fr5x_X58nOy>fJ*@(5+G_6#>hNpmAOmHFIQfodSv@#G)!o2_2(}0k{|;r?1Ah zh_f)!Ok%W=V`}FtV&XjeLv8ShHlxY(YFBN1@PIk|JXK*7?NBG;6*c;N+?}d{XVGH= zKqAJHAe3y(DO<=ta!oQbrLuJ3`kK1KJo>G)C4GM>{th9 zs=eYx3y)q3p|n|G2BdAXz%EePRuC>Aov@6>V{U-CEu zA0HU*rqE28ue5T0`eyA-j*cShA&()R;KHK8yzQReRCpazH|cBzyWVIzovVMhZ(lok zUJdPwC~ju5GY4eDHQ9Wf4z*}2&Ml{dif#I~q$;WKzPPppk*`e^wYC`|*+eJ{cbAdI z^cN-0lNYFMht$5X{Z7zwS_eC``*-Hyt)tgB;BUI{-AbXz4cG3~QjE$vNO`MGw@Ur6 zY}E=MT-n4mWuitR$a(NiE2IqbcrrdbnJo_8aM$|j>GON{?>>LVm*JJOq|SK>2m z6-rCML{d)yokam>LUlYn!go#+#8uBDh?ZRf&`pAM9JD1m)|Dtyrv);Qt0dU21m7c9 zR|rG>Qz5EShz;$*pq+%S6og8mMwK)4Biy%yEZpfld^a2P?BLnB;#vCuEy$$hzn!QA z=5B1L2a^5{GyziMuvm@xK^CEAMEEQdfL&@41(L^RYC9*x z`SK)Qs5j`s%8pv73j3A>FcUCZvS<^aYZXM81RfQbwJjwXGT>jbBr|MRvdrW)x=w_b z6hA0M2uaW_Yfv1hYEUgNtigCWu)LCXX@mzwh1y#lsqkQ1)tN3pURPp^xp9zJ1vFI` z$NQ_r2mhM-8g%Mlmav2z>MszxwmW!&Ufo*$dcXVd`t`Mo-|hR<*VdA{$h-b4ZUSgB zv6VmrxW&6}kh)OoGYH`Vu_-gI_$!M2TI_7Iwa^|m+Ct(%Q6(@9ZK2jBL&3-kBD;p4 zddIgHA6J$<(3G=V3hXQx8L}$WXD`9j035Pyl(rqzGrq)1S}i9%nAHkbXRV8+j(F}* zR3luH7x3mfE_6dUQ~;yu`b4#rADSx`t=MHY+JmlCLxaLe;7MK|H8Di_{5ey02UG}N zp{jB*Qxhq)Wts|AWJ}!8qUE+iho>Nzq6DvW@2NbQ7RTCyz5kaI2`R{L&rbz^E+?zMVyjXcBef zTVweC#<nHhd{ksJO7aSSB2c9W52al-j z%2P&RR6{V8O%>&qHr1o0pk{%}kdUkhoZDPW9-q2aC%Nyc)Q(n+)%Mfd`*-ve!W3jl zACj?!g_bDP$}URox?ZWx9#dN}h+d~^>nO6&l5*<=K}EZ*FRj4~LC<6lDvE${O`vjy zb}Qx66*07Dk-y*uo5~uqDTJ_x>j7zhO@-5Kb%Mab2=&0Fwc3>aI@tMr{QJVc^ZxVo zE1Z|V+Z6$6IIzkxQ9;vqqe%4o;VWxk^JY=qEPl?Rb?~QD$RPrCPatv9?L3-Vh$W*J z;nWLZqzYdXr>~5c*9bTf%SO@i`aZ?D$?Zbdyc#A-)cqDkqFj#}3#C#~i7}wH8uO4= zp}s66s*B)z{dDjC>zC2~7w|UOHP@+!MAfS5D2mN|72&_XV9nuEj)YO9i_Q0u z<5O!eok^X%zjv1l>pY&PwSxf)p%C^cT0TcaV^UsB)tYgQvlz(6=oDfw)>!cD}3uMGm}5=OkY#RYc4P9yhQ}F z+RFKh(8Ldpv(VkmOrS?eW?+nkEFGzUP+G zZ*sNYR5|n$WKC8TiLG6p6|kjFD+EzSr4KKV$5!o_Ubx!3bX`*sC_PyEiULz89qg9; zb4Xrs8Gs{&{WPVc(#rYiTlUk-moM|@+;@YQcWE>4J&uN>W2tO79HGwATqq)2cjC!W zG+NG<*8X(4INr7%v6>`fO}T9nO~eFo{2ZwzIg7?UwIoq!!BR#BIev-e*<=HAz;`tbe{YAB%v6w}Q`v>$Z9 zI@W?W$-2yRsPxT0mHA1TMfj-{@F3QbxKeLGs`RQXcqMzb7}MP*SWKDsMS|2^nCZ&7knjI%$RZLRE(zDcc zpm-)dOCc^(UO|OmOenI27d)eMPPm{`-e*oyr$p)(bXB)r3fyca!Z*+0GCH@k4Bn5s z3=Ae~E*kg=P4=pUp(Jss-6dB}4X4Z&o^e)x*>%HCg6&SosO!d(B39LdkJQ7?xX~R- zxmL5^PCkdvUoBp|dMnz_#i#~)?a+39^FIQu6D$H;nHJlfc6)8Y#d0MElMKSj2 zgmmw?7tM#0sr^x?01Jzu*>$-k*X^l$Bm}7mR660{Pif(km84$NWFv_cf?38L_5;4R zIz9CQ%4^+eOEO==0%~p>I*(|w4J7MXf8Q0|J%*fIQ#d-aHz(P6^f{{5wlHQ*&6kW2 zA)!Vdvaz*-SiKui+Y9nLOsY}TKar%C0CS%FvSj{cTsX~TT**6L)FeoF^E;8$$KZFX ztyA^|xew3dS}mlYY9Cb#%hz%(h;O_Mq$ahZWEnk}4OKa4r=yC~db_ZR8s(rjmeCJp z8OipCRU{5#GD4_NvomeqJAeCT+`0+N=%RPq7%mqzTyhMvY)<-yyH3Gk>nap@{Gz_2 z!_nkw=C*6}_vxQ#{F+{RnN8fhTDT44q6UV$4wcQg1Gwq1^82n^nKSMPO0I2S%q8P% zotkG{p<{1%LVZ@`nN*d-K6zZC7ZeY((45HGSiCS%B zl5#YLuTcRtBEF)WRZie>{%fM$zQG z)JpqF!7_#ve(763`PSRFfBPH8$1nUjw}fI0poaA{KC||hr>Cf_3+vbEaz-_~BQM}FEu;dL4o#J{cAclYmD|KSKG<60GIM`$xmR{kpG zM!h0k)uUnWE9udyq!KxawksL<4iSb8Vc8Fzj~Eg`4#}-@C_T{*eBU|P`Q=vsrT=8n zy$L<~vUj^`grU0MsC-1vr_h=wzNPO|QO~nW*oTFGQi8um1hK@h5$p!GX(uO83Wd^k zLf0j0+@@|Z+e8s6P8c@yR@vkdH}0P)NPqgDeho@10rN{48lF+2K@!<7=it-~7ahO|+$oasRpu8zRH7$uYO@Jc&9xg`%MDH)K-7hR! zo8)JqIsIQt&a)oF<8TJp6`}{0>SQrGw`MW=Q(s`P08ADZTGVI^`3I=iI2zI3pIY%J z&?m`=d1n1yC$JuX1W2VaW4KmXD;Uj+v}Jp@f|6FSn*v2t~kStw{oAK z`zulxL?hLFAOZ*qsk`C0Iyp)>0fyt~C_ZGB)np4@Ly4UqmU1Ap0}qm`wg=|P@7RK) zy!G4Z^xgQhe-k49vUj_xh|jk@CD~no66|(t5#Og|zYso?eP|8oJ{3Tcy=n%-OU5-I z+tgaJ0mH&ATp%rXJ4}h|S5mW+DtPU%N*pQ|ni5Q*HB*urOZtRx=&JJ(cD6gILuloE zl;H%XkXCzvqto8$dj)B{B4z5=Oo*v* zt(Z*n)ebl2wFz5f%^*5DKv@NL(9+F>Y9o&+4>5z-5twGbLoBuzMgdQ%*AC^6PLEH$ zneU?O#hwHuP3`nae0V4BACf%Pw>?E8qiLvI%Z>m9bzP|j9`%Hts_Vw`=}Rs0p53L@ z4W-s5pWGF)eLkhQx*7qk{Xrmbvz3~y1WP@xI2gFBGCoz9HlgWqR4COUtU4Gt{J