From 6eb75228e9754bcc3a05022f6750b67c32a1bd9a Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Fri, 9 Apr 2021 12:33:33 +0300 Subject: [PATCH 01/76] FlatDictionary performance test fix --- tests/performance/flat_dictionary.xml | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/tests/performance/flat_dictionary.xml b/tests/performance/flat_dictionary.xml index a571785a7f0..8cffd30c25e 100644 --- a/tests/performance/flat_dictionary.xml +++ b/tests/performance/flat_dictionary.xml @@ -25,7 +25,7 @@ ) PRIMARY KEY id SOURCE(CLICKHOUSE(DB 'default' TABLE 'simple_key_flat_dictionary_source_table')) - LAYOUT(FLAT(INITIAL_ARRAY_SIZE 50000 MAX_ARRAY_SIZE 5000000)) + LAYOUT(FLAT(INITIAL_ARRAY_SIZE 50000 MAX_ARRAY_SIZE 50000000)) LIFETIME(MIN 0 MAX 1000) @@ -33,7 +33,7 @@ INSERT INTO simple_key_flat_dictionary_source_table SELECT number, number, toString(number), toDecimal64(number, 8), toString(number) FROM system.numbers - LIMIT 5000000; + LIMIT 50000000; @@ -50,30 +50,22 @@ elements_count - 2500000 - 5000000 - 7500000 - 10000000 + 25000000 + 50000000 + 75000000 + 100000000 - SELECT dictGet('default.simple_key_flat_dictionary', {column_name}, number) - FROM system.numbers - LIMIT {elements_count} - FORMAT Null; - - - - SELECT dictHas('default.simple_key_flat_dictionary', number) + SELECT dictGet('default.simple_key_flat_dictionary', ('value_int', 'value_string', 'value_decimal', 'value_string_nullable'), number) FROM system.numbers LIMIT {elements_count} FORMAT Null; DROP TABLE IF EXISTS simple_key_flat_dictionary_source_table - DROP DICTIONARY IF EXISTS simple_key_flat_dictionary From f8c8b5e49dd03606dcf81a68a4030d17077fa013 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Fri, 9 Apr 2021 14:10:08 +0300 Subject: [PATCH 02/76] Updated test --- tests/performance/flat_dictionary.xml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/performance/flat_dictionary.xml b/tests/performance/flat_dictionary.xml index 8cffd30c25e..bea359ab4b1 100644 --- a/tests/performance/flat_dictionary.xml +++ b/tests/performance/flat_dictionary.xml @@ -25,7 +25,7 @@ ) PRIMARY KEY id SOURCE(CLICKHOUSE(DB 'default' TABLE 'simple_key_flat_dictionary_source_table')) - LAYOUT(FLAT(INITIAL_ARRAY_SIZE 50000 MAX_ARRAY_SIZE 50000000)) + LAYOUT(FLAT(INITIAL_ARRAY_SIZE 50000 MAX_ARRAY_SIZE 5000000)) LIFETIME(MIN 0 MAX 1000) @@ -33,7 +33,7 @@ INSERT INTO simple_key_flat_dictionary_source_table SELECT number, number, toString(number), toDecimal64(number, 8), toString(number) FROM system.numbers - LIMIT 50000000; + LIMIT 5000000; @@ -50,7 +50,6 @@ elements_count - 25000000 50000000 75000000 100000000 @@ -59,13 +58,21 @@ - SELECT dictGet('default.simple_key_flat_dictionary', ('value_int', 'value_string', 'value_decimal', 'value_string_nullable'), number) + SELECT dictGet('default.simple_key_flat_dictionary', {column_name}, rand64() % toUInt64(10000000)) + FROM system.numbers + LIMIT {elements_count} + FORMAT Null; + + + + SELECT dictHas('default.simple_key_flat_dictionary', rand64() % toUInt64(10000000)) FROM system.numbers LIMIT {elements_count} FORMAT Null; DROP TABLE IF EXISTS simple_key_flat_dictionary_source_table + DROP DICTIONARY IF EXISTS simple_key_flat_dictionary From 1557161d92f55f4f9796a25f875457431c81a046 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sat, 10 Apr 2021 00:55:54 +0300 Subject: [PATCH 03/76] Updated test --- tests/performance/flat_dictionary.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/performance/flat_dictionary.xml b/tests/performance/flat_dictionary.xml index bea359ab4b1..8111084586a 100644 --- a/tests/performance/flat_dictionary.xml +++ b/tests/performance/flat_dictionary.xml @@ -50,9 +50,9 @@ elements_count - 50000000 - 75000000 - 100000000 + 5000000 + 7500000 + 10000000 From 7c47832405d1750c6dcc26d17398a7af404385c8 Mon Sep 17 00:00:00 2001 From: Dmitry Krylov Date: Wed, 10 Mar 2021 21:24:01 +1000 Subject: [PATCH 04/76] Experiment: attempt to shrink arrays --- src/Functions/array/FunctionArrayMapped.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/Functions/array/FunctionArrayMapped.h b/src/Functions/array/FunctionArrayMapped.h index 4b6320824fa..f67435f57f9 100644 --- a/src/Functions/array/FunctionArrayMapped.h +++ b/src/Functions/array/FunctionArrayMapped.h @@ -42,6 +42,14 @@ namespace ErrorCodes template class FunctionArrayMapped : public IFunction { +private: + size_t + my_min(size_t a, size_t b) const + { + return (a < b) ? a : b; + } + + public: static constexpr auto name = Name::name; static FunctionPtr create(ContextPtr) { return std::make_shared(); } @@ -195,6 +203,14 @@ public: column_array = checkAndGetColumn(column_array_ptr.get()); } + ColumnPtr int_column = column_array->getDataPtr(); + ColumnPtr ca_ptr = ColumnArray::create( int_column->cut(0, my_min(1, int_column->size())), + column_array->getOffsetsPtr()); + const auto * ca = checkAndGetColumn(ca_ptr.get()); + + column_array_ptr = ca_ptr; + column_array = ca; + if (!array_type) throw Exception("Expected array type, found " + array_type_ptr->getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); From 2e8a296cc9faf0413c50004e4207e617e62e2b38 Mon Sep 17 00:00:00 2001 From: Dmitry Krylov Date: Fri, 12 Mar 2021 17:24:30 +1000 Subject: [PATCH 05/76] Draft: very simple variant --- src/Functions/array/FunctionArrayMapped.h | 131 +++++++++++++----- src/Functions/array/arrayAggregation.cpp | 1 + src/Functions/array/arrayAll.cpp | 1 + src/Functions/array/arrayCompact.cpp | 1 + src/Functions/array/arrayCount.cpp | 1 + src/Functions/array/arrayCumSum.cpp | 1 + .../array/arrayCumSumNonNegative.cpp | 1 + src/Functions/array/arrayDifference.cpp | 1 + src/Functions/array/arrayExists.cpp | 1 + src/Functions/array/arrayFill.cpp | 1 + src/Functions/array/arrayFilter.cpp | 1 + src/Functions/array/arrayFirst.cpp | 1 + src/Functions/array/arrayFirstIndex.cpp | 1 + src/Functions/array/arrayFold.cpp | 58 ++++++++ src/Functions/array/arrayMap.cpp | 2 + src/Functions/array/arraySort.cpp | 1 + src/Functions/array/arraySplit.cpp | 1 + .../registerFunctionsHigherOrder.cpp | 2 + src/Functions/ya.make | 1 + 19 files changed, 176 insertions(+), 32 deletions(-) create mode 100644 src/Functions/array/arrayFold.cpp diff --git a/src/Functions/array/FunctionArrayMapped.h b/src/Functions/array/FunctionArrayMapped.h index f67435f57f9..f9bde946e07 100644 --- a/src/Functions/array/FunctionArrayMapped.h +++ b/src/Functions/array/FunctionArrayMapped.h @@ -42,14 +42,6 @@ namespace ErrorCodes template class FunctionArrayMapped : public IFunction { -private: - size_t - my_min(size_t a, size_t b) const - { - return (a < b) ? a : b; - } - - public: static constexpr auto name = Name::name; static FunctionPtr create(ContextPtr) { return std::make_shared(); } @@ -75,8 +67,9 @@ public: throw Exception("Function " + getName() + " needs at least one array argument.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + size_t arguments_to_skip = Impl::isFolding() ? 1 : 0; DataTypes nested_types(arguments.size() - 1); - for (size_t i = 0; i < nested_types.size(); ++i) + for (size_t i = 0; i < nested_types.size() - arguments_to_skip; ++i) { const DataTypeArray * array_type = checkAndGetDataType(&*arguments[i + 1]); if (!array_type) @@ -84,6 +77,8 @@ public: + arguments[i + 1]->getName() + " instead.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); nested_types[i] = recursiveRemoveLowCardinality(array_type->getNestedType()); } + if (Impl::isFolding()) + nested_types[nested_types.size() - 1] = arguments[arguments.size() - 1]; const DataTypeFunction * function_type = checkAndGetDataType(arguments[0].get()); if (!function_type || function_type->getArgumentTypes().size() != nested_types.size()) @@ -138,14 +133,25 @@ public: throw Exception("Expression for function " + getName() + " must return UInt8, found " + return_type->getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - const auto * first_array_type = checkAndGetDataType(arguments[1].type.get()); - - return Impl::getReturnType(return_type, first_array_type->getNestedType()); + if (Impl::isFolding()) + { + const auto accum_type = arguments.back().type; + return Impl::getReturnType(return_type, accum_type); + } + else + { + const auto * first_array_type = checkAndGetDataType(arguments[1].type.get()); + return Impl::getReturnType(return_type, first_array_type->getNestedType()); + } } } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { + std::cerr << " *** FOLDING" << std::endl; + std::cerr << " isFolding(): " << (Impl::isFolding() ? "yes" : "no-") << std::endl; + std::cerr << " arguments.size() = " << arguments.size() << std::endl; + if (arguments.size() == 1) { ColumnPtr column_array_ptr = arguments[0].column; @@ -155,7 +161,7 @@ public: { const ColumnConst * column_const_array = checkAndGetColumnConst(column_array_ptr.get()); if (!column_const_array) - throw Exception("Expected array column, found " + column_array_ptr->getName(), ErrorCodes::ILLEGAL_COLUMN); + throw Exception("X1 Expected array column, found " + column_array_ptr->getName(), ErrorCodes::ILLEGAL_COLUMN); column_array_ptr = column_const_array->convertToFullColumn(); column_array = assert_cast(column_array_ptr.get()); } @@ -181,10 +187,12 @@ public: ColumnPtr column_first_array_ptr; const ColumnArray * column_first_array = nullptr; + size_t arguments_to_skip = Impl::isFolding() ? 1 : 0; + ColumnsWithTypeAndName arrays; arrays.reserve(arguments.size() - 1); - for (size_t i = 1; i < arguments.size(); ++i) + for (size_t i = 1; i < arguments.size() - arguments_to_skip; ++i) { const auto & array_with_type_and_name = arguments[i]; @@ -198,21 +206,13 @@ public: { const ColumnConst * column_const_array = checkAndGetColumnConst(column_array_ptr.get()); if (!column_const_array) - throw Exception("Expected array column, found " + column_array_ptr->getName(), ErrorCodes::ILLEGAL_COLUMN); + throw Exception("X2 Expected array column, found " + column_array_ptr->getName(), ErrorCodes::ILLEGAL_COLUMN); column_array_ptr = recursiveRemoveLowCardinality(column_const_array->convertToFullColumn()); column_array = checkAndGetColumn(column_array_ptr.get()); } - ColumnPtr int_column = column_array->getDataPtr(); - ColumnPtr ca_ptr = ColumnArray::create( int_column->cut(0, my_min(1, int_column->size())), - column_array->getOffsetsPtr()); - const auto * ca = checkAndGetColumn(ca_ptr.get()); - - column_array_ptr = ca_ptr; - column_array = ca; - if (!array_type) - throw Exception("Expected array type, found " + array_type_ptr->getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception("X3 Expected array type, found " + array_type_ptr->getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); if (!offsets_column) { @@ -236,17 +236,84 @@ public: recursiveRemoveLowCardinality(array_type->getNestedType()), array_with_type_and_name.name)); } + if (Impl::isFolding()) + arrays.emplace_back(arguments[arguments.size() - 1]); // TODO .last() + std::cerr << " arrays.size() = " << arrays.size() << std::endl; + std::cerr << " column_first_array->getData().size() = " << column_first_array->getData().size() << std::endl; - /// Put all the necessary columns multiplied by the sizes of arrays into the columns. - auto replicated_column_function_ptr = IColumn::mutate(column_function->replicate(column_first_array->getOffsets())); - auto * replicated_column_function = typeid_cast(replicated_column_function_ptr.get()); - replicated_column_function->appendArguments(arrays); + if (Impl::isFolding() && (column_first_array->getData().size() > 0)) // TODO .size() -> .empty() + { - auto lambda_result = replicated_column_function->reduce().column; - if (lambda_result->lowCardinality()) - lambda_result = lambda_result->convertToFullColumnIfLowCardinality(); - return Impl::execute(*column_first_array, lambda_result); + ColumnWithTypeAndName accumulator = arguments.back(); + ColumnPtr res; + + for(size_t i = 0; i < column_first_array->getData().size(); ++i) + { + std::cerr << " ----- iteration " << i << " ------" << std::endl; + // Make slice of input arrays and accumulator for lambda + ColumnsWithTypeAndName iter_arrays; + iter_arrays.reserve(arrays.size()); + for(size_t j = 0; j < arrays.size() - 1; ++j) + { + auto const & arr = arrays[j]; + std::cerr << " " << j << ") " << 1 << std::endl; + /* + const ColumnArray * arr_array = checkAndGetColumn(arr.column.get()); + std::cerr << " " << j << ") " << 1 << " " << arr_array << std::endl; + std::cerr << " " << j << ") " << 2 << std::endl; + const ColumnPtr & nested_column_x = arr_array->getData().cut(i, 1); + std::cerr << " " << j << ") " << 3 << std::endl; + const ColumnPtr & offsets_column_x = ColumnArray::ColumnOffsets::create(1, 1); + std::cerr << " " << j << ") " << 4 << std::endl; + auto new_arr_array = ColumnArray::create(nested_column_x, offsets_column_x); + std::cerr << " " << j << ") " << 5 << std::endl; + */ + iter_arrays.emplace_back(ColumnWithTypeAndName(arr.column->cut(i, 1), + arr.type, + arr.name)); + std::cerr << " " << j << ") " << 6 << std::endl; + } + iter_arrays.emplace_back(accumulator); + // ---- + std::cerr << " formed" << std::endl; + auto replicated_column_function_ptr = IColumn::mutate(column_function->replicate(IColumn::Offsets(1, 1))); + auto * replicated_column_function = typeid_cast(replicated_column_function_ptr.get()); + std::cerr << " pre append" << std::endl; + replicated_column_function->appendArguments(iter_arrays); + std::cerr << " post append" << std::endl; + auto lambda_result = replicated_column_function->reduce().column; + if (lambda_result->lowCardinality()) + lambda_result = lambda_result->convertToFullColumnIfLowCardinality(); + std::cerr << " pre execute" << std::endl; + res = Impl::execute(*column_first_array, lambda_result); // TODO column_first_array + std::cerr << " post execute : res " << res->dumpStructure() << std::endl; + std::cerr << " post execute : res[0] " << (*res)[0].dump() << std::endl; + // ~~~ + // ~~~ + // ~~~ + accumulator.column = res; + } + + return res; + + } + else + { + /// Put all the necessary columns multiplied by the sizes of arrays into the columns. + auto replicated_column_function_ptr = IColumn::mutate(column_function->replicate(column_first_array->getOffsets())); + auto * replicated_column_function = typeid_cast(replicated_column_function_ptr.get()); + replicated_column_function->appendArguments(arrays); + + auto lambda_result = replicated_column_function->reduce().column; + if (lambda_result->lowCardinality()) + lambda_result = lambda_result->convertToFullColumnIfLowCardinality(); + + ColumnPtr res = Impl::execute(*column_first_array, lambda_result); + std::cerr << " ^^^ FOLDING" << std::endl; + return res; + //return Impl::execute(*column_first_array, lambda_result); + } } } }; diff --git a/src/Functions/array/arrayAggregation.cpp b/src/Functions/array/arrayAggregation.cpp index e0e246b8af4..35b20740e90 100644 --- a/src/Functions/array/arrayAggregation.cpp +++ b/src/Functions/array/arrayAggregation.cpp @@ -76,6 +76,7 @@ struct ArrayAggregateImpl static bool needBoolean() { return false; } static bool needExpression() { return false; } static bool needOneArray() { return false; } + static bool isFolding() { return false; } static DataTypePtr getReturnType(const DataTypePtr & expression_return, const DataTypePtr & /*array_element*/) { diff --git a/src/Functions/array/arrayAll.cpp b/src/Functions/array/arrayAll.cpp index 34deafdffdf..1a6b007f35f 100644 --- a/src/Functions/array/arrayAll.cpp +++ b/src/Functions/array/arrayAll.cpp @@ -19,6 +19,7 @@ struct ArrayAllImpl static bool needBoolean() { return true; } static bool needExpression() { return false; } static bool needOneArray() { return false; } + static bool isFolding() { return false; } static DataTypePtr getReturnType(const DataTypePtr & /*expression_return*/, const DataTypePtr & /*array_element*/) { diff --git a/src/Functions/array/arrayCompact.cpp b/src/Functions/array/arrayCompact.cpp index e0f73207da8..d63037eec06 100644 --- a/src/Functions/array/arrayCompact.cpp +++ b/src/Functions/array/arrayCompact.cpp @@ -20,6 +20,7 @@ struct ArrayCompactImpl static bool needBoolean() { return false; } static bool needExpression() { return false; } static bool needOneArray() { return false; } + static bool isFolding() { return false; } static DataTypePtr getReturnType(const DataTypePtr & nested_type, const DataTypePtr &) { diff --git a/src/Functions/array/arrayCount.cpp b/src/Functions/array/arrayCount.cpp index 377a6eb8fb1..76d1cfbaba7 100644 --- a/src/Functions/array/arrayCount.cpp +++ b/src/Functions/array/arrayCount.cpp @@ -19,6 +19,7 @@ struct ArrayCountImpl static bool needBoolean() { return true; } static bool needExpression() { return false; } static bool needOneArray() { return false; } + static bool isFolding() { return false; } static DataTypePtr getReturnType(const DataTypePtr & /*expression_return*/, const DataTypePtr & /*array_element*/) { diff --git a/src/Functions/array/arrayCumSum.cpp b/src/Functions/array/arrayCumSum.cpp index 9a6eafb8822..8a7189d2ef9 100644 --- a/src/Functions/array/arrayCumSum.cpp +++ b/src/Functions/array/arrayCumSum.cpp @@ -20,6 +20,7 @@ struct ArrayCumSumImpl static bool needBoolean() { return false; } static bool needExpression() { return false; } static bool needOneArray() { return false; } + static bool isFolding() { return false; } static DataTypePtr getReturnType(const DataTypePtr & expression_return, const DataTypePtr & /*array_element*/) { diff --git a/src/Functions/array/arrayCumSumNonNegative.cpp b/src/Functions/array/arrayCumSumNonNegative.cpp index 2c7362a1605..b6f2703487e 100644 --- a/src/Functions/array/arrayCumSumNonNegative.cpp +++ b/src/Functions/array/arrayCumSumNonNegative.cpp @@ -23,6 +23,7 @@ struct ArrayCumSumNonNegativeImpl static bool needBoolean() { return false; } static bool needExpression() { return false; } static bool needOneArray() { return false; } + static bool isFolding() { return false; } static DataTypePtr getReturnType(const DataTypePtr & expression_return, const DataTypePtr & /*array_element*/) { diff --git a/src/Functions/array/arrayDifference.cpp b/src/Functions/array/arrayDifference.cpp index b4b30079a4e..8d405cac368 100644 --- a/src/Functions/array/arrayDifference.cpp +++ b/src/Functions/array/arrayDifference.cpp @@ -23,6 +23,7 @@ struct ArrayDifferenceImpl static bool needBoolean() { return false; } static bool needExpression() { return false; } static bool needOneArray() { return false; } + static bool isFolding() { return false; } static DataTypePtr getReturnType(const DataTypePtr & expression_return, const DataTypePtr & /*array_element*/) { diff --git a/src/Functions/array/arrayExists.cpp b/src/Functions/array/arrayExists.cpp index 34ea71af259..2b3521082ba 100644 --- a/src/Functions/array/arrayExists.cpp +++ b/src/Functions/array/arrayExists.cpp @@ -19,6 +19,7 @@ struct ArrayExistsImpl static bool needBoolean() { return true; } static bool needExpression() { return false; } static bool needOneArray() { return false; } + static bool isFolding() { return false; } static DataTypePtr getReturnType(const DataTypePtr & /*expression_return*/, const DataTypePtr & /*array_element*/) { diff --git a/src/Functions/array/arrayFill.cpp b/src/Functions/array/arrayFill.cpp index d4b36a89ba5..8e466ca3c38 100644 --- a/src/Functions/array/arrayFill.cpp +++ b/src/Functions/array/arrayFill.cpp @@ -22,6 +22,7 @@ struct ArrayFillImpl static bool needBoolean() { return true; } static bool needExpression() { return true; } static bool needOneArray() { return false; } + static bool isFolding() { return false; } static DataTypePtr getReturnType(const DataTypePtr & /*expression_return*/, const DataTypePtr & array_element) { diff --git a/src/Functions/array/arrayFilter.cpp b/src/Functions/array/arrayFilter.cpp index 1291989f9a2..9a4ec52a027 100644 --- a/src/Functions/array/arrayFilter.cpp +++ b/src/Functions/array/arrayFilter.cpp @@ -18,6 +18,7 @@ struct ArrayFilterImpl static bool needBoolean() { return true; } static bool needExpression() { return true; } static bool needOneArray() { return false; } + static bool isFolding() { return false; } static DataTypePtr getReturnType(const DataTypePtr & /*expression_return*/, const DataTypePtr & array_element) { diff --git a/src/Functions/array/arrayFirst.cpp b/src/Functions/array/arrayFirst.cpp index dbe545ea387..3d24ac4f620 100644 --- a/src/Functions/array/arrayFirst.cpp +++ b/src/Functions/array/arrayFirst.cpp @@ -16,6 +16,7 @@ struct ArrayFirstImpl static bool needBoolean() { return false; } static bool needExpression() { return true; } static bool needOneArray() { return false; } + static bool isFolding() { return false; } static DataTypePtr getReturnType(const DataTypePtr & /*expression_return*/, const DataTypePtr & array_element) { diff --git a/src/Functions/array/arrayFirstIndex.cpp b/src/Functions/array/arrayFirstIndex.cpp index d229687774e..51c75fe6127 100644 --- a/src/Functions/array/arrayFirstIndex.cpp +++ b/src/Functions/array/arrayFirstIndex.cpp @@ -16,6 +16,7 @@ struct ArrayFirstIndexImpl static bool needBoolean() { return false; } static bool needExpression() { return true; } static bool needOneArray() { return false; } + static bool isFolding() { return false; } static DataTypePtr getReturnType(const DataTypePtr & /*expression_return*/, const DataTypePtr & /*array_element*/) { diff --git a/src/Functions/array/arrayFold.cpp b/src/Functions/array/arrayFold.cpp new file mode 100644 index 00000000000..35ffb437732 --- /dev/null +++ b/src/Functions/array/arrayFold.cpp @@ -0,0 +1,58 @@ +#include "FunctionArrayMapped.h" +#include + + +namespace DB +{ + +/** arrayFold(x1,...,xn,accum -> expression, array1,...,arrayn, init_accum) - apply the expression to each element of the array (or set of parallel arrays). + */ +struct ArrayFoldImpl +{ + static bool needBoolean() { return false; } + static bool needExpression() { return true; } + static bool needOneArray() { return false; } + static bool isFolding() { return true; } + + static DataTypePtr getReturnType(const DataTypePtr & /*expression_return*/, const DataTypePtr & accum_type) + { + return accum_type; + } + + static ColumnPtr execute(const ColumnArray & array, ColumnPtr mapped) + { + std::cerr << " **** ArrayFoldImpl **** " << std::endl; + std::cerr << " array: " << array.dumpStructure() << std::endl; + std::cerr << " mapped: " << mapped->dumpStructure() << std::endl; + std::cerr << " mapped[0]: " << (*mapped)[0].dump() << std::endl; + // std::cerr << " mapped[1]: " << (*mapped)[1].dump() << std::endl; + // + + ColumnPtr res; + if (mapped->size() == 0) + { + res = mapped; + } + else + { + res = mapped->cut(0, 1); + } + std::cerr << " ^^^^ ArrayFoldImpl ^^^^" << std::endl; + return res; + + // return ColumnArray::create(mapped->convertToFullColumnIfConst(), array.getOffsetsPtr()); + // return ColumnArray::create(mapped->convertToFullColumnIfConst(), array.getOffsetsPtr()); + } +}; + +struct NameArrayFold { static constexpr auto name = "arrayFold"; }; +using FunctionArrayFold = FunctionArrayMapped; + +void registerFunctionArrayFold(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} + + diff --git a/src/Functions/array/arrayMap.cpp b/src/Functions/array/arrayMap.cpp index e3afaf7fb66..7337e1de20d 100644 --- a/src/Functions/array/arrayMap.cpp +++ b/src/Functions/array/arrayMap.cpp @@ -15,6 +15,8 @@ struct ArrayMapImpl static bool needExpression() { return true; } /// true if the array must be exactly one. static bool needOneArray() { return false; } + /// true if function do folding + static bool isFolding() { return false; } static DataTypePtr getReturnType(const DataTypePtr & expression_return, const DataTypePtr & /*array_element*/) { diff --git a/src/Functions/array/arraySort.cpp b/src/Functions/array/arraySort.cpp index 478c7e52614..93a37ebc645 100644 --- a/src/Functions/array/arraySort.cpp +++ b/src/Functions/array/arraySort.cpp @@ -13,6 +13,7 @@ struct ArraySortImpl static bool needBoolean() { return false; } static bool needExpression() { return false; } static bool needOneArray() { return false; } + static bool isFolding() { return false; } static DataTypePtr getReturnType(const DataTypePtr & /*expression_return*/, const DataTypePtr & array_element) { diff --git a/src/Functions/array/arraySplit.cpp b/src/Functions/array/arraySplit.cpp index 2e5f2d8432e..68045e8bd34 100644 --- a/src/Functions/array/arraySplit.cpp +++ b/src/Functions/array/arraySplit.cpp @@ -17,6 +17,7 @@ struct ArraySplitImpl static bool needBoolean() { return true; } static bool needExpression() { return true; } static bool needOneArray() { return false; } + static bool isFolding() { return false; } static DataTypePtr getReturnType(const DataTypePtr & /*expression_return*/, const DataTypePtr & array_element) { diff --git a/src/Functions/registerFunctionsHigherOrder.cpp b/src/Functions/registerFunctionsHigherOrder.cpp index d3621a03ecd..29416346f4e 100644 --- a/src/Functions/registerFunctionsHigherOrder.cpp +++ b/src/Functions/registerFunctionsHigherOrder.cpp @@ -4,6 +4,7 @@ namespace DB class FunctionFactory; void registerFunctionArrayMap(FunctionFactory & factory); +void registerFunctionArrayFold(FunctionFactory & factory); void registerFunctionArrayFilter(FunctionFactory & factory); void registerFunctionArrayCount(FunctionFactory & factory); void registerFunctionArrayExists(FunctionFactory & factory); @@ -22,6 +23,7 @@ void registerFunctionArrayDifference(FunctionFactory & factory); void registerFunctionsHigherOrder(FunctionFactory & factory) { registerFunctionArrayMap(factory); + registerFunctionArrayFold(factory); registerFunctionArrayFilter(factory); registerFunctionArrayCount(factory); registerFunctionArrayExists(factory); diff --git a/src/Functions/ya.make b/src/Functions/ya.make index 52ed54ec64f..8161243215a 100644 --- a/src/Functions/ya.make +++ b/src/Functions/ya.make @@ -144,6 +144,7 @@ SRCS( array/arrayFirst.cpp array/arrayFirstIndex.cpp array/arrayFlatten.cpp + array/arrayFold.cpp array/arrayIntersect.cpp array/arrayJoin.cpp array/arrayMap.cpp From fbd43aeea8869757ac01383683ed27f8f1d89601 Mon Sep 17 00:00:00 2001 From: Dmitry Krylov Date: Fri, 12 Mar 2021 19:55:32 +1000 Subject: [PATCH 06/76] Draft: taking into accout input arrays variable length --- src/Functions/array/FunctionArrayMapped.h | 136 ++++++++++++++-------- src/Functions/array/arrayFold.cpp | 5 +- 2 files changed, 90 insertions(+), 51 deletions(-) diff --git a/src/Functions/array/FunctionArrayMapped.h b/src/Functions/array/FunctionArrayMapped.h index f9bde946e07..b4f996789a5 100644 --- a/src/Functions/array/FunctionArrayMapped.h +++ b/src/Functions/array/FunctionArrayMapped.h @@ -239,63 +239,103 @@ public: if (Impl::isFolding()) arrays.emplace_back(arguments[arguments.size() - 1]); // TODO .last() std::cerr << " arrays.size() = " << arrays.size() << std::endl; + std::cerr << " column_first_array->size() = " << column_first_array->size() << std::endl; + std::cerr << " column_first_array->getOffsets().size() = " << column_first_array->getOffsets().size() << std::endl; std::cerr << " column_first_array->getData().size() = " << column_first_array->getData().size() << std::endl; if (Impl::isFolding() && (column_first_array->getData().size() > 0)) // TODO .size() -> .empty() { + size_t arr_cursor = 0; - ColumnWithTypeAndName accumulator = arguments.back(); - ColumnPtr res; + MutableColumnPtr result = arguments.back().column->convertToFullColumnIfConst()->cloneEmpty(); - for(size_t i = 0; i < column_first_array->getData().size(); ++i) + for(size_t irow = 0; irow < column_first_array->size(); ++irow) // for each row of result { - std::cerr << " ----- iteration " << i << " ------" << std::endl; - // Make slice of input arrays and accumulator for lambda - ColumnsWithTypeAndName iter_arrays; - iter_arrays.reserve(arrays.size()); - for(size_t j = 0; j < arrays.size() - 1; ++j) - { - auto const & arr = arrays[j]; - std::cerr << " " << j << ") " << 1 << std::endl; - /* - const ColumnArray * arr_array = checkAndGetColumn(arr.column.get()); - std::cerr << " " << j << ") " << 1 << " " << arr_array << std::endl; - std::cerr << " " << j << ") " << 2 << std::endl; - const ColumnPtr & nested_column_x = arr_array->getData().cut(i, 1); - std::cerr << " " << j << ") " << 3 << std::endl; - const ColumnPtr & offsets_column_x = ColumnArray::ColumnOffsets::create(1, 1); - std::cerr << " " << j << ") " << 4 << std::endl; - auto new_arr_array = ColumnArray::create(nested_column_x, offsets_column_x); - std::cerr << " " << j << ") " << 5 << std::endl; - */ - iter_arrays.emplace_back(ColumnWithTypeAndName(arr.column->cut(i, 1), - arr.type, - arr.name)); - std::cerr << " " << j << ") " << 6 << std::endl; - } - iter_arrays.emplace_back(accumulator); - // ---- - std::cerr << " formed" << std::endl; - auto replicated_column_function_ptr = IColumn::mutate(column_function->replicate(IColumn::Offsets(1, 1))); - auto * replicated_column_function = typeid_cast(replicated_column_function_ptr.get()); - std::cerr << " pre append" << std::endl; - replicated_column_function->appendArguments(iter_arrays); - std::cerr << " post append" << std::endl; - auto lambda_result = replicated_column_function->reduce().column; - if (lambda_result->lowCardinality()) - lambda_result = lambda_result->convertToFullColumnIfLowCardinality(); - std::cerr << " pre execute" << std::endl; - res = Impl::execute(*column_first_array, lambda_result); // TODO column_first_array - std::cerr << " post execute : res " << res->dumpStructure() << std::endl; - std::cerr << " post execute : res[0] " << (*res)[0].dump() << std::endl; - // ~~~ - // ~~~ - // ~~~ - accumulator.column = res; - } + std::cerr << " --- row " << irow << " ---" << std::endl; - return res; + // Make accumulator column for this row + // TODO проверить с константой + ColumnWithTypeAndName accumulator_column = arguments.back(); // TODO тут нужно ещё и позицию в аргументе извлекать + ColumnPtr acc(accumulator_column.column->cut(irow, 1)); + + std::cerr << " * accumulator.type " << accumulator_column.type->getName() << std::endl; + std::cerr << " * accumulator.column " << accumulator_column.column->dumpStructure() << std::endl; + std::cerr << " * acc " << acc->dumpStructure() << std::endl; + std::cerr << " * acc[0] " << (*acc)[0].dump() << std::endl; + + auto accumulator = ColumnWithTypeAndName(acc, + accumulator_column.type, + accumulator_column.name); + + ColumnPtr res; + size_t const arr_next = column_first_array->getOffsets()[irow]; // when we do folding + for(size_t iter = 0; arr_cursor < arr_next; ++iter, ++arr_cursor) + { + std::cerr << " ----- iteration " << iter << " ------" << std::endl; + // Make slice of input arrays and accumulator for lambda + ColumnsWithTypeAndName iter_arrays; + std::cerr << " arrays.size() = " << arrays.size() << std::endl; + iter_arrays.reserve(arrays.size() + 1); + //size_t arr_from = (iter == 0) ? 0 : column_first_array->getOffsets()[iter - 1]; + //size_t arr_len = column_first_array->getOffsets()[iter] - arr_from; + //std::cerr << " arr_from = " << arr_from << std::endl; + //std::cerr << " arr_len = " << arr_len << std::endl; + std::cerr << " arr_cursor = " << arr_cursor << std::endl; + std::cerr << " arr_next = " << arr_next << std::endl; + for(size_t icolumn = 0; icolumn < arrays.size() - 1; ++icolumn) + { + auto const & arr = arrays[icolumn]; + std::cerr << " @ " << icolumn << ") 1 :: " << arr_cursor << std::endl; + /* + const ColumnArray * arr_array = checkAndGetColumn(arr.column.get()); + std::cerr << " " << icolumn << ") " << 1 << " " << arr_array << std::endl; + std::cerr << " " << icolumn << ") " << 2 << std::endl; + const ColumnPtr & nested_column_x = arr_array->getData().cut(iter, 1); + std::cerr << " " << icolumn << ") " << 3 << std::endl; + const ColumnPtr & offsets_column_x = ColumnArray::ColumnOffsets::create(1, 1); + std::cerr << " " << icolumn << ") " << 4 << std::endl; + auto new_arr_array = ColumnArray::create(nested_column_x, offsets_column_x); + std::cerr << " " << icolumn << ") " << 5 << std::endl; + */ + iter_arrays.emplace_back(ColumnWithTypeAndName(arr.column->cut(arr_cursor, 1), + arr.type, + arr.name)); + std::cerr << " @ " << icolumn << ") 2 :: " << iter_arrays.back().column->dumpStructure() << std::endl; + } + iter_arrays.emplace_back(accumulator); + // ---- + std::cerr << " formed" << std::endl; + auto replicated_column_function_ptr = IColumn::mutate(column_function->replicate(column_first_array->getOffsets())); + auto * replicated_column_function = typeid_cast(replicated_column_function_ptr.get()); + std::cerr << " pre append" << std::endl; + replicated_column_function->appendArguments(iter_arrays); + std::cerr << " post append" << std::endl; + auto lambda_result = replicated_column_function->reduce().column; + if (lambda_result->lowCardinality()) + lambda_result = lambda_result->convertToFullColumnIfLowCardinality(); + std::cerr << " pre execute" << std::endl; + res = Impl::execute(*column_first_array, lambda_result); // TODO column_first_array + std::cerr << " post execute : res " << res->dumpStructure() << std::endl; + std::cerr << " post execute : res[0] " << (*res)[0].dump() << std::endl; + // ~~~ + // ~~~ + // ~~~ + accumulator.column = res; + + } + + std::cerr << " pre result " << result->dumpStructure() << std::endl; + //result->insertFrom(*res, 0); + result->insert((*res)[0]); + std::cerr << " post result " << result->dumpStructure() << std::endl; + std::cerr << " post res[0] " << (*res)[0].dump() << std::endl; + std::cerr << " post result[0] " << (*result)[0].dump() << std::endl; + + //return res; + + } + return result; } else diff --git a/src/Functions/array/arrayFold.cpp b/src/Functions/array/arrayFold.cpp index 35ffb437732..c6d4040fbef 100644 --- a/src/Functions/array/arrayFold.cpp +++ b/src/Functions/array/arrayFold.cpp @@ -24,9 +24,6 @@ struct ArrayFoldImpl std::cerr << " **** ArrayFoldImpl **** " << std::endl; std::cerr << " array: " << array.dumpStructure() << std::endl; std::cerr << " mapped: " << mapped->dumpStructure() << std::endl; - std::cerr << " mapped[0]: " << (*mapped)[0].dump() << std::endl; - // std::cerr << " mapped[1]: " << (*mapped)[1].dump() << std::endl; - // ColumnPtr res; if (mapped->size() == 0) @@ -35,6 +32,8 @@ struct ArrayFoldImpl } else { + std::cerr << " mapped[0]: " << (*mapped)[0].dump() << std::endl; + // std::cerr << " mapped[1]: " << (*mapped)[1].dump() << std::endl; res = mapped->cut(0, 1); } std::cerr << " ^^^^ ArrayFoldImpl ^^^^" << std::endl; From b19a06ba48e7c29195072bfb8653994b0f3098a9 Mon Sep 17 00:00:00 2001 From: Dmitry Krylov Date: Fri, 12 Mar 2021 19:59:22 +1000 Subject: [PATCH 07/76] Fix to work with empty arrays --- src/Functions/array/FunctionArrayMapped.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/array/FunctionArrayMapped.h b/src/Functions/array/FunctionArrayMapped.h index b4f996789a5..d7b633f4da6 100644 --- a/src/Functions/array/FunctionArrayMapped.h +++ b/src/Functions/array/FunctionArrayMapped.h @@ -268,7 +268,7 @@ public: accumulator_column.type, accumulator_column.name); - ColumnPtr res; + ColumnPtr res(acc); size_t const arr_next = column_first_array->getOffsets()[irow]; // when we do folding for(size_t iter = 0; arr_cursor < arr_next; ++iter, ++arr_cursor) { From 21941bffc8f3a48a3595cf02cf3f97cd927b1a37 Mon Sep 17 00:00:00 2001 From: Dmitry Krylov Date: Sun, 14 Mar 2021 19:49:46 +1000 Subject: [PATCH 08/76] Tests for `arrayFold` --- .../0_stateless/90001_array_fold.reference | 6 ++ .../queries/0_stateless/90001_array_fold.sql | 6 ++ .../0_stateless/90002_array_fold.reference | 70 +++++++++++++++++++ .../queries/0_stateless/90002_array_fold.sql | 7 ++ 4 files changed, 89 insertions(+) create mode 100644 tests/queries/0_stateless/90001_array_fold.reference create mode 100644 tests/queries/0_stateless/90001_array_fold.sql create mode 100644 tests/queries/0_stateless/90002_array_fold.reference create mode 100644 tests/queries/0_stateless/90002_array_fold.sql diff --git a/tests/queries/0_stateless/90001_array_fold.reference b/tests/queries/0_stateless/90001_array_fold.reference new file mode 100644 index 00000000000..fd735503608 --- /dev/null +++ b/tests/queries/0_stateless/90001_array_fold.reference @@ -0,0 +1,6 @@ +23 +3 +[1,2,3,4] +[4,3,2,1] +([4,3,2,1],[1,2,3,4]) +([1,3,5],[2,4,6]) diff --git a/tests/queries/0_stateless/90001_array_fold.sql b/tests/queries/0_stateless/90001_array_fold.sql new file mode 100644 index 00000000000..74ca70735f5 --- /dev/null +++ b/tests/queries/0_stateless/90001_array_fold.sql @@ -0,0 +1,6 @@ +SELECT arrayFold(x,acc -> acc + x * 2, [1,2,3,4], toInt64(3)); +SELECT arrayFold(x,acc -> acc + x * 2, emptyArrayInt64(), toInt64(3)); +SELECT arrayFold(x,acc -> arrayPushBack(acc,x), [1,2,3,4], emptyArrayInt64()); +SELECT arrayFold(x,acc -> arrayPushFront(acc,x), [1,2,3,4], emptyArrayInt64()); +SELECT arrayFold(x,acc -> (arrayPushFront(acc.1,x), arrayPushBack(acc.2,x)), [1,2,3,4], (emptyArrayInt64(), emptyArrayInt64())); +SELECT arrayFold(x,acc -> x % 2 ? (arrayPushBack(acc.1,x), acc.2): (acc.1, arrayPushBack(acc.2,x)), [1,2,3,4,5,6], (emptyArrayInt64(), emptyArrayInt64())); diff --git a/tests/queries/0_stateless/90002_array_fold.reference b/tests/queries/0_stateless/90002_array_fold.reference new file mode 100644 index 00000000000..2860dbba152 --- /dev/null +++ b/tests/queries/0_stateless/90002_array_fold.reference @@ -0,0 +1,70 @@ +0 +0 +1 +3 +6 +10 +15 +21 +28 +36 +0 +1 +3 +6 +10 +15 +21 +28 +36 +45 +[] +[0] +[1,0] +[2,1,0] +[3,2,1,0] +[4,3,2,1,0] +[5,4,3,2,1,0] +[6,5,4,3,2,1,0] +[7,6,5,4,3,2,1,0] +[8,7,6,5,4,3,2,1,0] +[] +[0] +[1,0] +[1,0,2] +[3,1,0,2] +[3,1,0,2,4] +[5,3,1,0,2,4] +[5,3,1,0,2,4,6] +[7,5,3,1,0,2,4,6] +[7,5,3,1,0,2,4,6,8] +(0,0) +(0,0) +(1,-1) +(3,-3) +(6,-6) +(10,-10) +(15,-15) +(21,-21) +(28,-28) +(36,-36) +(0,0) +(0,0) +(1,-1) +(3,-3) +(6,-6) +(10,-10) +(15,-15) +(21,-21) +(28,-28) +(36,-36) +[(0,0)] +[(0,1),(0,0)] +[(1,2),(0,1),(0,0)] +[(2,3),(1,2),(0,1),(0,0)] +[(3,4),(2,3),(1,2),(0,1),(0,0)] +[(4,5),(3,4),(2,3),(1,2),(0,1),(0,0)] +[(5,6),(4,5),(3,4),(2,3),(1,2),(0,1),(0,0)] +[(6,7),(5,6),(4,5),(3,4),(2,3),(1,2),(0,1),(0,0)] +[(7,8),(6,7),(5,6),(4,5),(3,4),(2,3),(1,2),(0,1),(0,0)] +[(8,9),(7,8),(6,7),(5,6),(4,5),(3,4),(2,3),(1,2),(0,1),(0,0)] diff --git a/tests/queries/0_stateless/90002_array_fold.sql b/tests/queries/0_stateless/90002_array_fold.sql new file mode 100644 index 00000000000..aac71e17563 --- /dev/null +++ b/tests/queries/0_stateless/90002_array_fold.sql @@ -0,0 +1,7 @@ +SELECT arrayFold(x,acc -> acc+x, range(number), toInt64(0)) FROM system.numbers LIMIT 10; +SELECT arrayFold(x,acc -> acc+x, range(number), number) FROM system.numbers LIMIT 10; +SELECT arrayFold(x,acc -> arrayPushFront(acc, x), range(number), emptyArrayUInt64()) FROM system.numbers LIMIT 10; +SELECT arrayFold(x,acc -> x % 2 ? arrayPushFront(acc, x) : arrayPushBack(acc, x), range(number), emptyArrayUInt64()) FROM system.numbers LIMIT 10; +SELECT arrayFold(x,acc -> (acc.1+x, acc.2-x), range(number), (toInt64(0), toInt64(0))) FROM system.numbers LIMIT 10; +SELECT arrayFold(x,acc -> (acc.1+x.1, acc.2-x.2), arrayZip(range(number), range(number)), (toInt64(0), toInt64(0))) FROM system.numbers LIMIT 10; +SELECT arrayFold(x,acc -> arrayPushFront(acc, (x, x+1)), range(number), [(toUInt64(0),toUInt64(0))]) FROM system.numbers LIMIT 10; From 9446df026bc8f355180572dedb43c5b03e975df9 Mon Sep 17 00:00:00 2001 From: Dmitry Krylov Date: Sun, 14 Mar 2021 20:07:19 +1000 Subject: [PATCH 09/76] Cleaning code --- src/Functions/array/FunctionArrayMapped.h | 74 ++--------------------- src/Functions/array/arrayFold.cpp | 23 +------ 2 files changed, 8 insertions(+), 89 deletions(-) diff --git a/src/Functions/array/FunctionArrayMapped.h b/src/Functions/array/FunctionArrayMapped.h index d7b633f4da6..5a34a865b9a 100644 --- a/src/Functions/array/FunctionArrayMapped.h +++ b/src/Functions/array/FunctionArrayMapped.h @@ -148,10 +148,6 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { - std::cerr << " *** FOLDING" << std::endl; - std::cerr << " isFolding(): " << (Impl::isFolding() ? "yes" : "no-") << std::endl; - std::cerr << " arguments.size() = " << arguments.size() << std::endl; - if (arguments.size() == 1) { ColumnPtr column_array_ptr = arguments[0].column; @@ -238,102 +234,47 @@ public: } if (Impl::isFolding()) arrays.emplace_back(arguments[arguments.size() - 1]); // TODO .last() - std::cerr << " arrays.size() = " << arrays.size() << std::endl; - std::cerr << " column_first_array->size() = " << column_first_array->size() << std::endl; - std::cerr << " column_first_array->getOffsets().size() = " << column_first_array->getOffsets().size() << std::endl; - std::cerr << " column_first_array->getData().size() = " << column_first_array->getData().size() << std::endl; if (Impl::isFolding() && (column_first_array->getData().size() > 0)) // TODO .size() -> .empty() { - size_t arr_cursor = 0; - MutableColumnPtr result = arguments.back().column->convertToFullColumnIfConst()->cloneEmpty(); - - for(size_t irow = 0; irow < column_first_array->size(); ++irow) // for each row of result + for (size_t irow = 0; irow < column_first_array->size(); ++irow) // for each row of result { - std::cerr << " --- row " << irow << " ---" << std::endl; - // Make accumulator column for this row // TODO проверить с константой ColumnWithTypeAndName accumulator_column = arguments.back(); // TODO тут нужно ещё и позицию в аргументе извлекать ColumnPtr acc(accumulator_column.column->cut(irow, 1)); - - std::cerr << " * accumulator.type " << accumulator_column.type->getName() << std::endl; - std::cerr << " * accumulator.column " << accumulator_column.column->dumpStructure() << std::endl; - std::cerr << " * acc " << acc->dumpStructure() << std::endl; - std::cerr << " * acc[0] " << (*acc)[0].dump() << std::endl; - auto accumulator = ColumnWithTypeAndName(acc, accumulator_column.type, accumulator_column.name); ColumnPtr res(acc); size_t const arr_next = column_first_array->getOffsets()[irow]; // when we do folding - for(size_t iter = 0; arr_cursor < arr_next; ++iter, ++arr_cursor) + for (size_t iter = 0; arr_cursor < arr_next; ++iter, ++arr_cursor) { - std::cerr << " ----- iteration " << iter << " ------" << std::endl; // Make slice of input arrays and accumulator for lambda ColumnsWithTypeAndName iter_arrays; - std::cerr << " arrays.size() = " << arrays.size() << std::endl; iter_arrays.reserve(arrays.size() + 1); - //size_t arr_from = (iter == 0) ? 0 : column_first_array->getOffsets()[iter - 1]; - //size_t arr_len = column_first_array->getOffsets()[iter] - arr_from; - //std::cerr << " arr_from = " << arr_from << std::endl; - //std::cerr << " arr_len = " << arr_len << std::endl; - std::cerr << " arr_cursor = " << arr_cursor << std::endl; - std::cerr << " arr_next = " << arr_next << std::endl; - for(size_t icolumn = 0; icolumn < arrays.size() - 1; ++icolumn) + for (size_t icolumn = 0; icolumn < arrays.size() - 1; ++icolumn) { auto const & arr = arrays[icolumn]; - std::cerr << " @ " << icolumn << ") 1 :: " << arr_cursor << std::endl; - /* - const ColumnArray * arr_array = checkAndGetColumn(arr.column.get()); - std::cerr << " " << icolumn << ") " << 1 << " " << arr_array << std::endl; - std::cerr << " " << icolumn << ") " << 2 << std::endl; - const ColumnPtr & nested_column_x = arr_array->getData().cut(iter, 1); - std::cerr << " " << icolumn << ") " << 3 << std::endl; - const ColumnPtr & offsets_column_x = ColumnArray::ColumnOffsets::create(1, 1); - std::cerr << " " << icolumn << ") " << 4 << std::endl; - auto new_arr_array = ColumnArray::create(nested_column_x, offsets_column_x); - std::cerr << " " << icolumn << ") " << 5 << std::endl; - */ iter_arrays.emplace_back(ColumnWithTypeAndName(arr.column->cut(arr_cursor, 1), arr.type, arr.name)); - std::cerr << " @ " << icolumn << ") 2 :: " << iter_arrays.back().column->dumpStructure() << std::endl; } iter_arrays.emplace_back(accumulator); - // ---- - std::cerr << " formed" << std::endl; + // Calculate function on arguments auto replicated_column_function_ptr = IColumn::mutate(column_function->replicate(column_first_array->getOffsets())); auto * replicated_column_function = typeid_cast(replicated_column_function_ptr.get()); - std::cerr << " pre append" << std::endl; replicated_column_function->appendArguments(iter_arrays); - std::cerr << " post append" << std::endl; auto lambda_result = replicated_column_function->reduce().column; if (lambda_result->lowCardinality()) lambda_result = lambda_result->convertToFullColumnIfLowCardinality(); - std::cerr << " pre execute" << std::endl; - res = Impl::execute(*column_first_array, lambda_result); // TODO column_first_array - std::cerr << " post execute : res " << res->dumpStructure() << std::endl; - std::cerr << " post execute : res[0] " << (*res)[0].dump() << std::endl; - // ~~~ - // ~~~ - // ~~~ + res = Impl::execute(*column_first_array, lambda_result); accumulator.column = res; - } - - std::cerr << " pre result " << result->dumpStructure() << std::endl; - //result->insertFrom(*res, 0); result->insert((*res)[0]); - std::cerr << " post result " << result->dumpStructure() << std::endl; - std::cerr << " post res[0] " << (*res)[0].dump() << std::endl; - std::cerr << " post result[0] " << (*result)[0].dump() << std::endl; - - //return res; - } return result; @@ -349,10 +290,7 @@ public: if (lambda_result->lowCardinality()) lambda_result = lambda_result->convertToFullColumnIfLowCardinality(); - ColumnPtr res = Impl::execute(*column_first_array, lambda_result); - std::cerr << " ^^^ FOLDING" << std::endl; - return res; - //return Impl::execute(*column_first_array, lambda_result); + return Impl::execute(*column_first_array, lambda_result); } } } diff --git a/src/Functions/array/arrayFold.cpp b/src/Functions/array/arrayFold.cpp index c6d4040fbef..0b5be625d61 100644 --- a/src/Functions/array/arrayFold.cpp +++ b/src/Functions/array/arrayFold.cpp @@ -19,28 +19,9 @@ struct ArrayFoldImpl return accum_type; } - static ColumnPtr execute(const ColumnArray & array, ColumnPtr mapped) + static ColumnPtr execute(const ColumnArray & /*array*/, ColumnPtr mapped) { - std::cerr << " **** ArrayFoldImpl **** " << std::endl; - std::cerr << " array: " << array.dumpStructure() << std::endl; - std::cerr << " mapped: " << mapped->dumpStructure() << std::endl; - - ColumnPtr res; - if (mapped->size() == 0) - { - res = mapped; - } - else - { - std::cerr << " mapped[0]: " << (*mapped)[0].dump() << std::endl; - // std::cerr << " mapped[1]: " << (*mapped)[1].dump() << std::endl; - res = mapped->cut(0, 1); - } - std::cerr << " ^^^^ ArrayFoldImpl ^^^^" << std::endl; - return res; - - // return ColumnArray::create(mapped->convertToFullColumnIfConst(), array.getOffsetsPtr()); - // return ColumnArray::create(mapped->convertToFullColumnIfConst(), array.getOffsetsPtr()); + return (mapped->size() == 0) ? mapped : mapped->cut(0, 1); } }; From 1861817bfcc736d69d3790df049caa51e2a8fe82 Mon Sep 17 00:00:00 2001 From: Dmitry Krylov Date: Tue, 16 Mar 2021 16:54:07 +1000 Subject: [PATCH 10/76] Remove debug messages --- src/Functions/array/FunctionArrayMapped.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Functions/array/FunctionArrayMapped.h b/src/Functions/array/FunctionArrayMapped.h index 5a34a865b9a..eacecd9da4c 100644 --- a/src/Functions/array/FunctionArrayMapped.h +++ b/src/Functions/array/FunctionArrayMapped.h @@ -202,13 +202,13 @@ public: { const ColumnConst * column_const_array = checkAndGetColumnConst(column_array_ptr.get()); if (!column_const_array) - throw Exception("X2 Expected array column, found " + column_array_ptr->getName(), ErrorCodes::ILLEGAL_COLUMN); + throw Exception("Expected array column, found " + column_array_ptr->getName(), ErrorCodes::ILLEGAL_COLUMN); column_array_ptr = recursiveRemoveLowCardinality(column_const_array->convertToFullColumn()); column_array = checkAndGetColumn(column_array_ptr.get()); } if (!array_type) - throw Exception("X3 Expected array type, found " + array_type_ptr->getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception("Expected array type, found " + array_type_ptr->getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); if (!offsets_column) { From e3502dabc79e40d169306a2d195ec8b85cce6c6f Mon Sep 17 00:00:00 2001 From: Dmitry Krylov Date: Tue, 16 Mar 2021 18:41:28 +1000 Subject: [PATCH 11/76] Remove debug messages --- src/Functions/array/FunctionArrayMapped.h | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/Functions/array/FunctionArrayMapped.h b/src/Functions/array/FunctionArrayMapped.h index eacecd9da4c..bd60ac9580f 100644 --- a/src/Functions/array/FunctionArrayMapped.h +++ b/src/Functions/array/FunctionArrayMapped.h @@ -157,7 +157,7 @@ public: { const ColumnConst * column_const_array = checkAndGetColumnConst(column_array_ptr.get()); if (!column_const_array) - throw Exception("X1 Expected array column, found " + column_array_ptr->getName(), ErrorCodes::ILLEGAL_COLUMN); + throw Exception("Expected array column, found " + column_array_ptr->getName(), ErrorCodes::ILLEGAL_COLUMN); column_array_ptr = column_const_array->convertToFullColumn(); column_array = assert_cast(column_array_ptr.get()); } @@ -233,17 +233,16 @@ public: array_with_type_and_name.name)); } if (Impl::isFolding()) - arrays.emplace_back(arguments[arguments.size() - 1]); // TODO .last() + arrays.emplace_back(arguments.back()); - if (Impl::isFolding() && (column_first_array->getData().size() > 0)) // TODO .size() -> .empty() + if (Impl::isFolding() && (! column_first_array->getData().empty())) { size_t arr_cursor = 0; MutableColumnPtr result = arguments.back().column->convertToFullColumnIfConst()->cloneEmpty(); for (size_t irow = 0; irow < column_first_array->size(); ++irow) // for each row of result { // Make accumulator column for this row - // TODO проверить с константой - ColumnWithTypeAndName accumulator_column = arguments.back(); // TODO тут нужно ещё и позицию в аргументе извлекать + ColumnWithTypeAndName accumulator_column = arguments.back(); ColumnPtr acc(accumulator_column.column->cut(irow, 1)); auto accumulator = ColumnWithTypeAndName(acc, accumulator_column.type, From 517ef8ebf42457e7411414836ffdeb0d333be964 Mon Sep 17 00:00:00 2001 From: Dmitry Krylov Date: Tue, 16 Mar 2021 18:45:07 +1000 Subject: [PATCH 12/76] Bug with string functions in `arrayFold` fixed --- src/Functions/array/FunctionArrayMapped.h | 2 +- tests/queries/0_stateless/90002_array_fold.reference | 10 ++++++++++ tests/queries/0_stateless/90002_array_fold.sql | 1 + 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/Functions/array/FunctionArrayMapped.h b/src/Functions/array/FunctionArrayMapped.h index bd60ac9580f..6de9751a6bc 100644 --- a/src/Functions/array/FunctionArrayMapped.h +++ b/src/Functions/array/FunctionArrayMapped.h @@ -264,7 +264,7 @@ public: } iter_arrays.emplace_back(accumulator); // Calculate function on arguments - auto replicated_column_function_ptr = IColumn::mutate(column_function->replicate(column_first_array->getOffsets())); + auto replicated_column_function_ptr = IColumn::mutate(column_function->replicate(ColumnArray::Offsets(column_first_array->getOffsets().size(), 1))); auto * replicated_column_function = typeid_cast(replicated_column_function_ptr.get()); replicated_column_function->appendArguments(iter_arrays); auto lambda_result = replicated_column_function->reduce().column; diff --git a/tests/queries/0_stateless/90002_array_fold.reference b/tests/queries/0_stateless/90002_array_fold.reference index 2860dbba152..ff0ec13e993 100644 --- a/tests/queries/0_stateless/90002_array_fold.reference +++ b/tests/queries/0_stateless/90002_array_fold.reference @@ -68,3 +68,13 @@ [(6,7),(5,6),(4,5),(3,4),(2,3),(1,2),(0,1),(0,0)] [(7,8),(6,7),(5,6),(4,5),(3,4),(2,3),(1,2),(0,1),(0,0)] [(8,9),(7,8),(6,7),(5,6),(4,5),(3,4),(2,3),(1,2),(0,1),(0,0)] +[] +['0'] +['0','1'] +['0','1','2'] +['0','1','2','3'] +['0','1','2','3','4'] +['0','1','2','3','4','5'] +['0','1','2','3','4','5','6'] +['0','1','2','3','4','5','6','7'] +['0','1','2','3','4','5','6','7','8'] diff --git a/tests/queries/0_stateless/90002_array_fold.sql b/tests/queries/0_stateless/90002_array_fold.sql index aac71e17563..23e85ead56f 100644 --- a/tests/queries/0_stateless/90002_array_fold.sql +++ b/tests/queries/0_stateless/90002_array_fold.sql @@ -5,3 +5,4 @@ SELECT arrayFold(x,acc -> x % 2 ? arrayPushFront(acc, x) : arrayPushBack(acc, x) SELECT arrayFold(x,acc -> (acc.1+x, acc.2-x), range(number), (toInt64(0), toInt64(0))) FROM system.numbers LIMIT 10; SELECT arrayFold(x,acc -> (acc.1+x.1, acc.2-x.2), arrayZip(range(number), range(number)), (toInt64(0), toInt64(0))) FROM system.numbers LIMIT 10; SELECT arrayFold(x,acc -> arrayPushFront(acc, (x, x+1)), range(number), [(toUInt64(0),toUInt64(0))]) FROM system.numbers LIMIT 10; +SELECT arrayFold(x, acc -> concat(acc, arrayMap(z -> toString(x), [number])) , range(number), CAST([] as Array(String))) FROM system.numbers LIMIT 10; From 15ea9b3f22bcda87627bf4c2f5178f4106986b64 Mon Sep 17 00:00:00 2001 From: Dmitry Krylov Date: Wed, 31 Mar 2021 19:42:13 +1000 Subject: [PATCH 13/76] Move `arrayFold` functionality to `arrayFold.cpp` --- src/Functions/array/FunctionArrayMapped.h | 86 +------ src/Functions/array/arrayAggregation.cpp | 1 - src/Functions/array/arrayAll.cpp | 1 - src/Functions/array/arrayCompact.cpp | 1 - src/Functions/array/arrayCount.cpp | 1 - src/Functions/array/arrayCumSum.cpp | 1 - .../array/arrayCumSumNonNegative.cpp | 1 - src/Functions/array/arrayDifference.cpp | 1 - src/Functions/array/arrayExists.cpp | 1 - src/Functions/array/arrayFill.cpp | 1 - src/Functions/array/arrayFilter.cpp | 1 - src/Functions/array/arrayFirst.cpp | 1 - src/Functions/array/arrayFirstIndex.cpp | 1 - src/Functions/array/arrayFold.cpp | 227 +++++++++++++++++- src/Functions/array/arrayMap.cpp | 2 - src/Functions/array/arraySort.cpp | 1 - src/Functions/array/arraySplit.cpp | 1 - 17 files changed, 228 insertions(+), 101 deletions(-) diff --git a/src/Functions/array/FunctionArrayMapped.h b/src/Functions/array/FunctionArrayMapped.h index 6de9751a6bc..4b6320824fa 100644 --- a/src/Functions/array/FunctionArrayMapped.h +++ b/src/Functions/array/FunctionArrayMapped.h @@ -67,9 +67,8 @@ public: throw Exception("Function " + getName() + " needs at least one array argument.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - size_t arguments_to_skip = Impl::isFolding() ? 1 : 0; DataTypes nested_types(arguments.size() - 1); - for (size_t i = 0; i < nested_types.size() - arguments_to_skip; ++i) + for (size_t i = 0; i < nested_types.size(); ++i) { const DataTypeArray * array_type = checkAndGetDataType(&*arguments[i + 1]); if (!array_type) @@ -77,8 +76,6 @@ public: + arguments[i + 1]->getName() + " instead.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); nested_types[i] = recursiveRemoveLowCardinality(array_type->getNestedType()); } - if (Impl::isFolding()) - nested_types[nested_types.size() - 1] = arguments[arguments.size() - 1]; const DataTypeFunction * function_type = checkAndGetDataType(arguments[0].get()); if (!function_type || function_type->getArgumentTypes().size() != nested_types.size()) @@ -133,16 +130,9 @@ public: throw Exception("Expression for function " + getName() + " must return UInt8, found " + return_type->getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - if (Impl::isFolding()) - { - const auto accum_type = arguments.back().type; - return Impl::getReturnType(return_type, accum_type); - } - else - { - const auto * first_array_type = checkAndGetDataType(arguments[1].type.get()); - return Impl::getReturnType(return_type, first_array_type->getNestedType()); - } + const auto * first_array_type = checkAndGetDataType(arguments[1].type.get()); + + return Impl::getReturnType(return_type, first_array_type->getNestedType()); } } @@ -183,12 +173,10 @@ public: ColumnPtr column_first_array_ptr; const ColumnArray * column_first_array = nullptr; - size_t arguments_to_skip = Impl::isFolding() ? 1 : 0; - ColumnsWithTypeAndName arrays; arrays.reserve(arguments.size() - 1); - for (size_t i = 1; i < arguments.size() - arguments_to_skip; ++i) + for (size_t i = 1; i < arguments.size(); ++i) { const auto & array_with_type_and_name = arguments[i]; @@ -232,65 +220,17 @@ public: recursiveRemoveLowCardinality(array_type->getNestedType()), array_with_type_and_name.name)); } - if (Impl::isFolding()) - arrays.emplace_back(arguments.back()); - if (Impl::isFolding() && (! column_first_array->getData().empty())) - { - size_t arr_cursor = 0; - MutableColumnPtr result = arguments.back().column->convertToFullColumnIfConst()->cloneEmpty(); - for (size_t irow = 0; irow < column_first_array->size(); ++irow) // for each row of result - { - // Make accumulator column for this row - ColumnWithTypeAndName accumulator_column = arguments.back(); - ColumnPtr acc(accumulator_column.column->cut(irow, 1)); - auto accumulator = ColumnWithTypeAndName(acc, - accumulator_column.type, - accumulator_column.name); + /// Put all the necessary columns multiplied by the sizes of arrays into the columns. + auto replicated_column_function_ptr = IColumn::mutate(column_function->replicate(column_first_array->getOffsets())); + auto * replicated_column_function = typeid_cast(replicated_column_function_ptr.get()); + replicated_column_function->appendArguments(arrays); - ColumnPtr res(acc); - size_t const arr_next = column_first_array->getOffsets()[irow]; // when we do folding - for (size_t iter = 0; arr_cursor < arr_next; ++iter, ++arr_cursor) - { - // Make slice of input arrays and accumulator for lambda - ColumnsWithTypeAndName iter_arrays; - iter_arrays.reserve(arrays.size() + 1); - for (size_t icolumn = 0; icolumn < arrays.size() - 1; ++icolumn) - { - auto const & arr = arrays[icolumn]; - iter_arrays.emplace_back(ColumnWithTypeAndName(arr.column->cut(arr_cursor, 1), - arr.type, - arr.name)); - } - iter_arrays.emplace_back(accumulator); - // Calculate function on arguments - auto replicated_column_function_ptr = IColumn::mutate(column_function->replicate(ColumnArray::Offsets(column_first_array->getOffsets().size(), 1))); - auto * replicated_column_function = typeid_cast(replicated_column_function_ptr.get()); - replicated_column_function->appendArguments(iter_arrays); - auto lambda_result = replicated_column_function->reduce().column; - if (lambda_result->lowCardinality()) - lambda_result = lambda_result->convertToFullColumnIfLowCardinality(); - res = Impl::execute(*column_first_array, lambda_result); - accumulator.column = res; - } - result->insert((*res)[0]); - } - return result; + auto lambda_result = replicated_column_function->reduce().column; + if (lambda_result->lowCardinality()) + lambda_result = lambda_result->convertToFullColumnIfLowCardinality(); - } - else - { - /// Put all the necessary columns multiplied by the sizes of arrays into the columns. - auto replicated_column_function_ptr = IColumn::mutate(column_function->replicate(column_first_array->getOffsets())); - auto * replicated_column_function = typeid_cast(replicated_column_function_ptr.get()); - replicated_column_function->appendArguments(arrays); - - auto lambda_result = replicated_column_function->reduce().column; - if (lambda_result->lowCardinality()) - lambda_result = lambda_result->convertToFullColumnIfLowCardinality(); - - return Impl::execute(*column_first_array, lambda_result); - } + return Impl::execute(*column_first_array, lambda_result); } } }; diff --git a/src/Functions/array/arrayAggregation.cpp b/src/Functions/array/arrayAggregation.cpp index 35b20740e90..e0e246b8af4 100644 --- a/src/Functions/array/arrayAggregation.cpp +++ b/src/Functions/array/arrayAggregation.cpp @@ -76,7 +76,6 @@ struct ArrayAggregateImpl static bool needBoolean() { return false; } static bool needExpression() { return false; } static bool needOneArray() { return false; } - static bool isFolding() { return false; } static DataTypePtr getReturnType(const DataTypePtr & expression_return, const DataTypePtr & /*array_element*/) { diff --git a/src/Functions/array/arrayAll.cpp b/src/Functions/array/arrayAll.cpp index 1a6b007f35f..34deafdffdf 100644 --- a/src/Functions/array/arrayAll.cpp +++ b/src/Functions/array/arrayAll.cpp @@ -19,7 +19,6 @@ struct ArrayAllImpl static bool needBoolean() { return true; } static bool needExpression() { return false; } static bool needOneArray() { return false; } - static bool isFolding() { return false; } static DataTypePtr getReturnType(const DataTypePtr & /*expression_return*/, const DataTypePtr & /*array_element*/) { diff --git a/src/Functions/array/arrayCompact.cpp b/src/Functions/array/arrayCompact.cpp index d63037eec06..e0f73207da8 100644 --- a/src/Functions/array/arrayCompact.cpp +++ b/src/Functions/array/arrayCompact.cpp @@ -20,7 +20,6 @@ struct ArrayCompactImpl static bool needBoolean() { return false; } static bool needExpression() { return false; } static bool needOneArray() { return false; } - static bool isFolding() { return false; } static DataTypePtr getReturnType(const DataTypePtr & nested_type, const DataTypePtr &) { diff --git a/src/Functions/array/arrayCount.cpp b/src/Functions/array/arrayCount.cpp index 76d1cfbaba7..377a6eb8fb1 100644 --- a/src/Functions/array/arrayCount.cpp +++ b/src/Functions/array/arrayCount.cpp @@ -19,7 +19,6 @@ struct ArrayCountImpl static bool needBoolean() { return true; } static bool needExpression() { return false; } static bool needOneArray() { return false; } - static bool isFolding() { return false; } static DataTypePtr getReturnType(const DataTypePtr & /*expression_return*/, const DataTypePtr & /*array_element*/) { diff --git a/src/Functions/array/arrayCumSum.cpp b/src/Functions/array/arrayCumSum.cpp index 8a7189d2ef9..9a6eafb8822 100644 --- a/src/Functions/array/arrayCumSum.cpp +++ b/src/Functions/array/arrayCumSum.cpp @@ -20,7 +20,6 @@ struct ArrayCumSumImpl static bool needBoolean() { return false; } static bool needExpression() { return false; } static bool needOneArray() { return false; } - static bool isFolding() { return false; } static DataTypePtr getReturnType(const DataTypePtr & expression_return, const DataTypePtr & /*array_element*/) { diff --git a/src/Functions/array/arrayCumSumNonNegative.cpp b/src/Functions/array/arrayCumSumNonNegative.cpp index b6f2703487e..2c7362a1605 100644 --- a/src/Functions/array/arrayCumSumNonNegative.cpp +++ b/src/Functions/array/arrayCumSumNonNegative.cpp @@ -23,7 +23,6 @@ struct ArrayCumSumNonNegativeImpl static bool needBoolean() { return false; } static bool needExpression() { return false; } static bool needOneArray() { return false; } - static bool isFolding() { return false; } static DataTypePtr getReturnType(const DataTypePtr & expression_return, const DataTypePtr & /*array_element*/) { diff --git a/src/Functions/array/arrayDifference.cpp b/src/Functions/array/arrayDifference.cpp index 8d405cac368..b4b30079a4e 100644 --- a/src/Functions/array/arrayDifference.cpp +++ b/src/Functions/array/arrayDifference.cpp @@ -23,7 +23,6 @@ struct ArrayDifferenceImpl static bool needBoolean() { return false; } static bool needExpression() { return false; } static bool needOneArray() { return false; } - static bool isFolding() { return false; } static DataTypePtr getReturnType(const DataTypePtr & expression_return, const DataTypePtr & /*array_element*/) { diff --git a/src/Functions/array/arrayExists.cpp b/src/Functions/array/arrayExists.cpp index 2b3521082ba..34ea71af259 100644 --- a/src/Functions/array/arrayExists.cpp +++ b/src/Functions/array/arrayExists.cpp @@ -19,7 +19,6 @@ struct ArrayExistsImpl static bool needBoolean() { return true; } static bool needExpression() { return false; } static bool needOneArray() { return false; } - static bool isFolding() { return false; } static DataTypePtr getReturnType(const DataTypePtr & /*expression_return*/, const DataTypePtr & /*array_element*/) { diff --git a/src/Functions/array/arrayFill.cpp b/src/Functions/array/arrayFill.cpp index 8e466ca3c38..d4b36a89ba5 100644 --- a/src/Functions/array/arrayFill.cpp +++ b/src/Functions/array/arrayFill.cpp @@ -22,7 +22,6 @@ struct ArrayFillImpl static bool needBoolean() { return true; } static bool needExpression() { return true; } static bool needOneArray() { return false; } - static bool isFolding() { return false; } static DataTypePtr getReturnType(const DataTypePtr & /*expression_return*/, const DataTypePtr & array_element) { diff --git a/src/Functions/array/arrayFilter.cpp b/src/Functions/array/arrayFilter.cpp index 9a4ec52a027..1291989f9a2 100644 --- a/src/Functions/array/arrayFilter.cpp +++ b/src/Functions/array/arrayFilter.cpp @@ -18,7 +18,6 @@ struct ArrayFilterImpl static bool needBoolean() { return true; } static bool needExpression() { return true; } static bool needOneArray() { return false; } - static bool isFolding() { return false; } static DataTypePtr getReturnType(const DataTypePtr & /*expression_return*/, const DataTypePtr & array_element) { diff --git a/src/Functions/array/arrayFirst.cpp b/src/Functions/array/arrayFirst.cpp index 3d24ac4f620..dbe545ea387 100644 --- a/src/Functions/array/arrayFirst.cpp +++ b/src/Functions/array/arrayFirst.cpp @@ -16,7 +16,6 @@ struct ArrayFirstImpl static bool needBoolean() { return false; } static bool needExpression() { return true; } static bool needOneArray() { return false; } - static bool isFolding() { return false; } static DataTypePtr getReturnType(const DataTypePtr & /*expression_return*/, const DataTypePtr & array_element) { diff --git a/src/Functions/array/arrayFirstIndex.cpp b/src/Functions/array/arrayFirstIndex.cpp index 51c75fe6127..d229687774e 100644 --- a/src/Functions/array/arrayFirstIndex.cpp +++ b/src/Functions/array/arrayFirstIndex.cpp @@ -16,7 +16,6 @@ struct ArrayFirstIndexImpl static bool needBoolean() { return false; } static bool needExpression() { return true; } static bool needOneArray() { return false; } - static bool isFolding() { return false; } static DataTypePtr getReturnType(const DataTypePtr & /*expression_return*/, const DataTypePtr & /*array_element*/) { diff --git a/src/Functions/array/arrayFold.cpp b/src/Functions/array/arrayFold.cpp index 0b5be625d61..87b86189e9e 100644 --- a/src/Functions/array/arrayFold.cpp +++ b/src/Functions/array/arrayFold.cpp @@ -7,32 +7,235 @@ namespace DB /** arrayFold(x1,...,xn,accum -> expression, array1,...,arrayn, init_accum) - apply the expression to each element of the array (or set of parallel arrays). */ -struct ArrayFoldImpl +class FunctionArrayFold : public IFunction { - static bool needBoolean() { return false; } - static bool needExpression() { return true; } - static bool needOneArray() { return false; } - static bool isFolding() { return true; } +public: + static constexpr auto name = "arrayFold"; + static FunctionPtr create(const Context &) { return std::make_shared(); } - static DataTypePtr getReturnType(const DataTypePtr & /*expression_return*/, const DataTypePtr & accum_type) + String getName() const override { - return accum_type; + return name; } - static ColumnPtr execute(const ColumnArray & /*array*/, ColumnPtr mapped) + bool isVariadic() const override { return true; } + size_t getNumberOfArguments() const override { return 0; } + + /// Called if at least one function argument is a lambda expression. + /// For argument-lambda expressions, it defines the types of arguments of these expressions. + void getLambdaArgumentTypes(DataTypes & arguments) const override { - return (mapped->size() == 0) ? mapped : mapped->cut(0, 1); + if (arguments.empty()) + throw Exception("Function " + getName() + " needs at least one argument; passed " + + toString(arguments.size()) + ".", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + if (arguments.size() == 1) + throw Exception("Function " + getName() + " needs at least one array argument.", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + DataTypes nested_types(arguments.size() - 1); + for (size_t i = 0; i < nested_types.size() - 1; ++i) + { + const DataTypeArray * array_type = checkAndGetDataType(&*arguments[i + 1]); + if (!array_type) + throw Exception("Argument " + toString(i + 2) + " of function " + getName() + " must be array. Found " + + arguments[i + 1]->getName() + " instead.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + nested_types[i] = recursiveRemoveLowCardinality(array_type->getNestedType()); + } + nested_types[nested_types.size() - 1] = arguments[arguments.size() - 1]; + + const DataTypeFunction * function_type = checkAndGetDataType(arguments[0].get()); + if (!function_type || function_type->getArgumentTypes().size() != nested_types.size()) + throw Exception("First argument for this overload of " + getName() + " must be a function with " + + toString(nested_types.size()) + " arguments. Found " + + arguments[0]->getName() + " instead.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + arguments[0] = std::make_shared(nested_types); + } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + size_t min_args = 2; + if (arguments.size() < min_args) + throw Exception("Function " + getName() + " needs at least " + + toString(min_args) + " argument; passed " + + toString(arguments.size()) + ".", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + if (arguments.size() == 1) + { + const auto * array_type = checkAndGetDataType(arguments[0].type.get()); + if (!array_type) + throw Exception("The only argument for function " + getName() + " must be array. Found " + + arguments[0].type->getName() + " instead.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + return array_type->getNestedType(); + } + else + { + const auto * data_type_function = checkAndGetDataType(arguments[0].type.get()); + + if (!data_type_function) + throw Exception("First argument for function " + getName() + " must be a function.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + /// The types of the remaining arguments are already checked in getLambdaArgumentTypes. + DataTypePtr return_type = removeLowCardinality(data_type_function->getReturnType()); + const auto accum_type = arguments.back().type; + return accum_type; + } + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override + { + if (arguments.size() == 1) + { + ColumnPtr column_array_ptr = arguments[0].column; + const auto * column_array = checkAndGetColumn(column_array_ptr.get()); + if (!column_array) + { + const ColumnConst * column_const_array = checkAndGetColumnConst(column_array_ptr.get()); + if (!column_const_array) + throw Exception("Expected array column, found " + column_array_ptr->getName(), ErrorCodes::ILLEGAL_COLUMN); + column_array_ptr = column_const_array->convertToFullColumn(); + column_array = assert_cast(column_array_ptr.get()); + } + auto res = column_array->getDataPtr(); + return res->empty() ? res : res->cut(0, 1); // TODO recheck + } + else + { + const auto & column_with_type_and_name = arguments[0]; + + if (!column_with_type_and_name.column) + throw Exception("First argument for function " + getName() + " must be a function.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + const auto * column_function = typeid_cast(column_with_type_and_name.column.get()); + + if (!column_function) + throw Exception("First argument for function " + getName() + " must be a function.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + ColumnPtr offsets_column; + + ColumnPtr column_first_array_ptr; + const ColumnArray * column_first_array = nullptr; + + ColumnsWithTypeAndName arrays; + arrays.reserve(arguments.size() - 1); + + for (size_t i = 1; i < arguments.size() - 1; ++i) + { + const auto & array_with_type_and_name = arguments[i]; + + ColumnPtr column_array_ptr = array_with_type_and_name.column; + const auto * column_array = checkAndGetColumn(column_array_ptr.get()); + + const DataTypePtr & array_type_ptr = array_with_type_and_name.type; + const auto * array_type = checkAndGetDataType(array_type_ptr.get()); + + if (!column_array) + { + const ColumnConst * column_const_array = checkAndGetColumnConst(column_array_ptr.get()); + if (!column_const_array) + throw Exception("Expected array column, found " + column_array_ptr->getName(), ErrorCodes::ILLEGAL_COLUMN); + column_array_ptr = recursiveRemoveLowCardinality(column_const_array->convertToFullColumn()); + column_array = checkAndGetColumn(column_array_ptr.get()); + } + + if (!array_type) + throw Exception("Expected array type, found " + array_type_ptr->getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + if (!offsets_column) + { + offsets_column = column_array->getOffsetsPtr(); + } + else + { + /// The first condition is optimization: do not compare data if the pointers are equal. + if (column_array->getOffsetsPtr() != offsets_column + && column_array->getOffsets() != typeid_cast(*offsets_column).getData()) + throw Exception("Arrays passed to " + getName() + " must have equal size", ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH); + } + + if (i == 1) + { + column_first_array_ptr = column_array_ptr; + column_first_array = column_array; + } + + arrays.emplace_back(ColumnWithTypeAndName(column_array->getDataPtr(), + recursiveRemoveLowCardinality(array_type->getNestedType()), + array_with_type_and_name.name)); + } + arrays.emplace_back(arguments.back()); + + if (! column_first_array->getData().empty()) + { + size_t arr_cursor = 0; + MutableColumnPtr result = arguments.back().column->convertToFullColumnIfConst()->cloneEmpty(); + for (size_t irow = 0; irow < column_first_array->size(); ++irow) // for each row of result + { + // Make accumulator column for this row + ColumnWithTypeAndName accumulator_column = arguments.back(); + ColumnPtr acc(accumulator_column.column->cut(irow, 1)); + auto accumulator = ColumnWithTypeAndName(acc, + accumulator_column.type, + accumulator_column.name); + + ColumnPtr res(acc); + size_t const arr_next = column_first_array->getOffsets()[irow]; // when we do folding + for (size_t iter = 0; arr_cursor < arr_next; ++iter, ++arr_cursor) + { + // Make slice of input arrays and accumulator for lambda + ColumnsWithTypeAndName iter_arrays; + iter_arrays.reserve(arrays.size() + 1); + for (size_t icolumn = 0; icolumn < arrays.size() - 1; ++icolumn) + { + auto const & arr = arrays[icolumn]; + iter_arrays.emplace_back(ColumnWithTypeAndName(arr.column->cut(arr_cursor, 1), + arr.type, + arr.name)); + } + iter_arrays.emplace_back(accumulator); + // Calculate function on arguments + auto replicated_column_function_ptr = IColumn::mutate(column_function->replicate(ColumnArray::Offsets(column_first_array->getOffsets().size(), 1))); + auto * replicated_column_function = typeid_cast(replicated_column_function_ptr.get()); + replicated_column_function->appendArguments(iter_arrays); + auto lambda_result = replicated_column_function->reduce().column; + if (lambda_result->lowCardinality()) + lambda_result = lambda_result->convertToFullColumnIfLowCardinality(); + res = lambda_result->empty() ? lambda_result : lambda_result->cut(0, 1); // TODO recheck this + accumulator.column = res; + } + result->insert((*res)[0]); + } + return result; + } + else + { + // TODO: for what this branch is needed? + // empty function + // + /// Put all the necessary columns multiplied by the sizes of arrays into the columns. + auto replicated_column_function_ptr = IColumn::mutate(column_function->replicate(column_first_array->getOffsets())); + auto * replicated_column_function = typeid_cast(replicated_column_function_ptr.get()); + replicated_column_function->appendArguments(arrays); + auto lambda_result = replicated_column_function->reduce().column; + if (lambda_result->lowCardinality()) + lambda_result = lambda_result->convertToFullColumnIfLowCardinality(); + return lambda_result->empty() ? lambda_result : lambda_result->cut(0, 1); // TODO recheck this + } + } } }; -struct NameArrayFold { static constexpr auto name = "arrayFold"; }; -using FunctionArrayFold = FunctionArrayMapped; void registerFunctionArrayFold(FunctionFactory & factory) { factory.registerFunction(); } + } - diff --git a/src/Functions/array/arrayMap.cpp b/src/Functions/array/arrayMap.cpp index 7337e1de20d..e3afaf7fb66 100644 --- a/src/Functions/array/arrayMap.cpp +++ b/src/Functions/array/arrayMap.cpp @@ -15,8 +15,6 @@ struct ArrayMapImpl static bool needExpression() { return true; } /// true if the array must be exactly one. static bool needOneArray() { return false; } - /// true if function do folding - static bool isFolding() { return false; } static DataTypePtr getReturnType(const DataTypePtr & expression_return, const DataTypePtr & /*array_element*/) { diff --git a/src/Functions/array/arraySort.cpp b/src/Functions/array/arraySort.cpp index 93a37ebc645..478c7e52614 100644 --- a/src/Functions/array/arraySort.cpp +++ b/src/Functions/array/arraySort.cpp @@ -13,7 +13,6 @@ struct ArraySortImpl static bool needBoolean() { return false; } static bool needExpression() { return false; } static bool needOneArray() { return false; } - static bool isFolding() { return false; } static DataTypePtr getReturnType(const DataTypePtr & /*expression_return*/, const DataTypePtr & array_element) { diff --git a/src/Functions/array/arraySplit.cpp b/src/Functions/array/arraySplit.cpp index 68045e8bd34..2e5f2d8432e 100644 --- a/src/Functions/array/arraySplit.cpp +++ b/src/Functions/array/arraySplit.cpp @@ -17,7 +17,6 @@ struct ArraySplitImpl static bool needBoolean() { return true; } static bool needExpression() { return true; } static bool needOneArray() { return false; } - static bool isFolding() { return false; } static DataTypePtr getReturnType(const DataTypePtr & /*expression_return*/, const DataTypePtr & array_element) { From ce1b9b20d5a689d8709b506a62cff88214032dc5 Mon Sep 17 00:00:00 2001 From: Dmitry Krylov Date: Thu, 1 Apr 2021 10:56:44 +1000 Subject: [PATCH 14/76] Remove unneeded branches of code --- src/Functions/array/arrayFold.cpp | 266 ++++++++++++------------------ 1 file changed, 110 insertions(+), 156 deletions(-) diff --git a/src/Functions/array/arrayFold.cpp b/src/Functions/array/arrayFold.cpp index 87b86189e9e..5df44b00574 100644 --- a/src/Functions/array/arrayFold.cpp +++ b/src/Functions/array/arrayFold.cpp @@ -13,11 +13,7 @@ public: static constexpr auto name = "arrayFold"; static FunctionPtr create(const Context &) { return std::make_shared(); } - String getName() const override - { - return name; - } - + String getName() const override { return name; } bool isVariadic() const override { return true; } size_t getNumberOfArguments() const override { return 0; } @@ -56,176 +52,134 @@ public: DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { - size_t min_args = 2; - if (arguments.size() < min_args) - throw Exception("Function " + getName() + " needs at least " - + toString(min_args) + " argument; passed " + if (arguments.size() < 2) + throw Exception("Function " + getName() + " needs at least 2 arguments; passed " + toString(arguments.size()) + ".", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - - if (arguments.size() == 1) - { - const auto * array_type = checkAndGetDataType(arguments[0].type.get()); - if (!array_type) - throw Exception("The only argument for function " + getName() + " must be array. Found " - + arguments[0].type->getName() + " instead.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - return array_type->getNestedType(); - } - else - { - const auto * data_type_function = checkAndGetDataType(arguments[0].type.get()); - - if (!data_type_function) - throw Exception("First argument for function " + getName() + " must be a function.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - /// The types of the remaining arguments are already checked in getLambdaArgumentTypes. - DataTypePtr return_type = removeLowCardinality(data_type_function->getReturnType()); - const auto accum_type = arguments.back().type; - return accum_type; - } + const auto * data_type_function = checkAndGetDataType(arguments[0].type.get()); + if (!data_type_function) + throw Exception("First argument for function " + getName() + " must be a function.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + /// The types of the remaining arguments are already checked in getLambdaArgumentTypes. + DataTypePtr return_type = removeLowCardinality(data_type_function->getReturnType()); + const auto accum_type = arguments.back().type; + return accum_type; } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { - if (arguments.size() == 1) + const auto & column_with_type_and_name = arguments[0]; + + if (!column_with_type_and_name.column) + throw Exception("First argument for function " + getName() + " must be a function.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + const auto * column_function = typeid_cast(column_with_type_and_name.column.get()); + + if (!column_function) + throw Exception("First argument for function " + getName() + " must be a function.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + ColumnPtr offsets_column; + ColumnPtr column_first_array_ptr; + const ColumnArray * column_first_array = nullptr; + ColumnsWithTypeAndName arrays; + arrays.reserve(arguments.size() - 1); + + for (size_t i = 1; i < arguments.size() - 1; ++i) { - ColumnPtr column_array_ptr = arguments[0].column; + const auto & array_with_type_and_name = arguments[i]; + ColumnPtr column_array_ptr = array_with_type_and_name.column; const auto * column_array = checkAndGetColumn(column_array_ptr.get()); + const DataTypePtr & array_type_ptr = array_with_type_and_name.type; + const auto * array_type = checkAndGetDataType(array_type_ptr.get()); if (!column_array) { const ColumnConst * column_const_array = checkAndGetColumnConst(column_array_ptr.get()); if (!column_const_array) throw Exception("Expected array column, found " + column_array_ptr->getName(), ErrorCodes::ILLEGAL_COLUMN); - column_array_ptr = column_const_array->convertToFullColumn(); - column_array = assert_cast(column_array_ptr.get()); + column_array_ptr = recursiveRemoveLowCardinality(column_const_array->convertToFullColumn()); + column_array = checkAndGetColumn(column_array_ptr.get()); } - auto res = column_array->getDataPtr(); - return res->empty() ? res : res->cut(0, 1); // TODO recheck - } - else - { - const auto & column_with_type_and_name = arguments[0]; - - if (!column_with_type_and_name.column) - throw Exception("First argument for function " + getName() + " must be a function.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - const auto * column_function = typeid_cast(column_with_type_and_name.column.get()); - - if (!column_function) - throw Exception("First argument for function " + getName() + " must be a function.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - ColumnPtr offsets_column; - - ColumnPtr column_first_array_ptr; - const ColumnArray * column_first_array = nullptr; - - ColumnsWithTypeAndName arrays; - arrays.reserve(arguments.size() - 1); - - for (size_t i = 1; i < arguments.size() - 1; ++i) + if (!array_type) + throw Exception("Expected array type, found " + array_type_ptr->getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + if (!offsets_column) { - const auto & array_with_type_and_name = arguments[i]; - - ColumnPtr column_array_ptr = array_with_type_and_name.column; - const auto * column_array = checkAndGetColumn(column_array_ptr.get()); - - const DataTypePtr & array_type_ptr = array_with_type_and_name.type; - const auto * array_type = checkAndGetDataType(array_type_ptr.get()); - - if (!column_array) - { - const ColumnConst * column_const_array = checkAndGetColumnConst(column_array_ptr.get()); - if (!column_const_array) - throw Exception("Expected array column, found " + column_array_ptr->getName(), ErrorCodes::ILLEGAL_COLUMN); - column_array_ptr = recursiveRemoveLowCardinality(column_const_array->convertToFullColumn()); - column_array = checkAndGetColumn(column_array_ptr.get()); - } - - if (!array_type) - throw Exception("Expected array type, found " + array_type_ptr->getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - if (!offsets_column) - { - offsets_column = column_array->getOffsetsPtr(); - } - else - { - /// The first condition is optimization: do not compare data if the pointers are equal. - if (column_array->getOffsetsPtr() != offsets_column - && column_array->getOffsets() != typeid_cast(*offsets_column).getData()) - throw Exception("Arrays passed to " + getName() + " must have equal size", ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH); - } - - if (i == 1) - { - column_first_array_ptr = column_array_ptr; - column_first_array = column_array; - } - - arrays.emplace_back(ColumnWithTypeAndName(column_array->getDataPtr(), - recursiveRemoveLowCardinality(array_type->getNestedType()), - array_with_type_and_name.name)); - } - arrays.emplace_back(arguments.back()); - - if (! column_first_array->getData().empty()) - { - size_t arr_cursor = 0; - MutableColumnPtr result = arguments.back().column->convertToFullColumnIfConst()->cloneEmpty(); - for (size_t irow = 0; irow < column_first_array->size(); ++irow) // for each row of result - { - // Make accumulator column for this row - ColumnWithTypeAndName accumulator_column = arguments.back(); - ColumnPtr acc(accumulator_column.column->cut(irow, 1)); - auto accumulator = ColumnWithTypeAndName(acc, - accumulator_column.type, - accumulator_column.name); - - ColumnPtr res(acc); - size_t const arr_next = column_first_array->getOffsets()[irow]; // when we do folding - for (size_t iter = 0; arr_cursor < arr_next; ++iter, ++arr_cursor) - { - // Make slice of input arrays and accumulator for lambda - ColumnsWithTypeAndName iter_arrays; - iter_arrays.reserve(arrays.size() + 1); - for (size_t icolumn = 0; icolumn < arrays.size() - 1; ++icolumn) - { - auto const & arr = arrays[icolumn]; - iter_arrays.emplace_back(ColumnWithTypeAndName(arr.column->cut(arr_cursor, 1), - arr.type, - arr.name)); - } - iter_arrays.emplace_back(accumulator); - // Calculate function on arguments - auto replicated_column_function_ptr = IColumn::mutate(column_function->replicate(ColumnArray::Offsets(column_first_array->getOffsets().size(), 1))); - auto * replicated_column_function = typeid_cast(replicated_column_function_ptr.get()); - replicated_column_function->appendArguments(iter_arrays); - auto lambda_result = replicated_column_function->reduce().column; - if (lambda_result->lowCardinality()) - lambda_result = lambda_result->convertToFullColumnIfLowCardinality(); - res = lambda_result->empty() ? lambda_result : lambda_result->cut(0, 1); // TODO recheck this - accumulator.column = res; - } - result->insert((*res)[0]); - } - return result; + offsets_column = column_array->getOffsetsPtr(); } else { - // TODO: for what this branch is needed? - // empty function - // - /// Put all the necessary columns multiplied by the sizes of arrays into the columns. - auto replicated_column_function_ptr = IColumn::mutate(column_function->replicate(column_first_array->getOffsets())); - auto * replicated_column_function = typeid_cast(replicated_column_function_ptr.get()); - replicated_column_function->appendArguments(arrays); - auto lambda_result = replicated_column_function->reduce().column; - if (lambda_result->lowCardinality()) - lambda_result = lambda_result->convertToFullColumnIfLowCardinality(); - return lambda_result->empty() ? lambda_result : lambda_result->cut(0, 1); // TODO recheck this + /// The first condition is optimization: do not compare data if the pointers are equal. + if (column_array->getOffsetsPtr() != offsets_column + && column_array->getOffsets() != typeid_cast(*offsets_column).getData()) + throw Exception("Arrays passed to " + getName() + " must have equal size", ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH); } + if (i == 1) + { + column_first_array_ptr = column_array_ptr; + column_first_array = column_array; + } + arrays.emplace_back(ColumnWithTypeAndName(column_array->getDataPtr(), + recursiveRemoveLowCardinality(array_type->getNestedType()), + array_with_type_and_name.name)); + } + arrays.emplace_back(arguments.back()); + + if (! column_first_array->getData().empty()) + { + size_t arr_cursor = 0; + MutableColumnPtr result = arguments.back().column->convertToFullColumnIfConst()->cloneEmpty(); + for (size_t irow = 0; irow < column_first_array->size(); ++irow) // for each row of result + { + // Make accumulator column for this row + ColumnWithTypeAndName accumulator_column = arguments.back(); + ColumnPtr acc(accumulator_column.column->cut(irow, 1)); + auto accumulator = ColumnWithTypeAndName(acc, + accumulator_column.type, + accumulator_column.name); + ColumnPtr res(acc); + size_t const arr_next = column_first_array->getOffsets()[irow]; // when we do folding + for (size_t iter = 0; arr_cursor < arr_next; ++iter, ++arr_cursor) + { + // Make slice of input arrays and accumulator for lambda + ColumnsWithTypeAndName iter_arrays; + iter_arrays.reserve(arrays.size() + 1); + for (size_t icolumn = 0; icolumn < arrays.size() - 1; ++icolumn) + { + auto const & arr = arrays[icolumn]; + iter_arrays.emplace_back(ColumnWithTypeAndName(arr.column->cut(arr_cursor, 1), + arr.type, + arr.name)); + } + iter_arrays.emplace_back(accumulator); + // Calculate function on arguments + auto replicated_column_function_ptr = IColumn::mutate(column_function->replicate(ColumnArray::Offsets(column_first_array->getOffsets().size(), 1))); + auto * replicated_column_function = typeid_cast(replicated_column_function_ptr.get()); + replicated_column_function->appendArguments(iter_arrays); + auto lambda_result = replicated_column_function->reduce().column; + if (lambda_result->lowCardinality()) + lambda_result = lambda_result->convertToFullColumnIfLowCardinality(); + res = lambda_result->empty() ? lambda_result : lambda_result->cut(0, 1); // TODO recheck this + accumulator.column = res; + } + result->insert((*res)[0]); + } + return result; + } + else + { + // TODO: for what this branch is needed? + // empty function + // + /// Put all the necessary columns multiplied by the sizes of arrays into the columns. + auto replicated_column_function_ptr = IColumn::mutate(column_function->replicate(column_first_array->getOffsets())); + auto * replicated_column_function = typeid_cast(replicated_column_function_ptr.get()); + replicated_column_function->appendArguments(arrays); + auto lambda_result = replicated_column_function->reduce().column; + if (lambda_result->lowCardinality()) + lambda_result = lambda_result->convertToFullColumnIfLowCardinality(); + // return lambda_result->empty() ? lambda_result : lambda_result->cut(0, 1); // TODO recheck this + return lambda_result; // TODO recheck this } } }; From c2f326e2ae007ed42fe5c1a1268883b8019e0488 Mon Sep 17 00:00:00 2001 From: Dmitry Krylov Date: Thu, 1 Apr 2021 11:07:11 +1000 Subject: [PATCH 15/76] Remove unneeded branches of code --- src/Functions/array/arrayFold.cpp | 82 ++++++++++++------------------- 1 file changed, 32 insertions(+), 50 deletions(-) diff --git a/src/Functions/array/arrayFold.cpp b/src/Functions/array/arrayFold.cpp index 5df44b00574..bf6f97bac7c 100644 --- a/src/Functions/array/arrayFold.cpp +++ b/src/Functions/array/arrayFold.cpp @@ -125,62 +125,44 @@ public: } arrays.emplace_back(arguments.back()); - if (! column_first_array->getData().empty()) + MutableColumnPtr result = arguments.back().column->convertToFullColumnIfConst()->cloneEmpty(); + size_t arr_cursor = 0; + for (size_t irow = 0; irow < column_first_array->size(); ++irow) // for each row of result { - size_t arr_cursor = 0; - MutableColumnPtr result = arguments.back().column->convertToFullColumnIfConst()->cloneEmpty(); - for (size_t irow = 0; irow < column_first_array->size(); ++irow) // for each row of result + // Make accumulator column for this row + ColumnWithTypeAndName accumulator_column = arguments.back(); + ColumnPtr acc(accumulator_column.column->cut(irow, 1)); + auto accumulator = ColumnWithTypeAndName(acc, + accumulator_column.type, + accumulator_column.name); + ColumnPtr res(acc); + size_t const arr_next = column_first_array->getOffsets()[irow]; // when we do folding + for (size_t iter = 0; arr_cursor < arr_next; ++iter, ++arr_cursor) { - // Make accumulator column for this row - ColumnWithTypeAndName accumulator_column = arguments.back(); - ColumnPtr acc(accumulator_column.column->cut(irow, 1)); - auto accumulator = ColumnWithTypeAndName(acc, - accumulator_column.type, - accumulator_column.name); - ColumnPtr res(acc); - size_t const arr_next = column_first_array->getOffsets()[irow]; // when we do folding - for (size_t iter = 0; arr_cursor < arr_next; ++iter, ++arr_cursor) + // Make slice of input arrays and accumulator for lambda + ColumnsWithTypeAndName iter_arrays; + iter_arrays.reserve(arrays.size() + 1); + for (size_t icolumn = 0; icolumn < arrays.size() - 1; ++icolumn) { - // Make slice of input arrays and accumulator for lambda - ColumnsWithTypeAndName iter_arrays; - iter_arrays.reserve(arrays.size() + 1); - for (size_t icolumn = 0; icolumn < arrays.size() - 1; ++icolumn) - { - auto const & arr = arrays[icolumn]; - iter_arrays.emplace_back(ColumnWithTypeAndName(arr.column->cut(arr_cursor, 1), - arr.type, - arr.name)); - } - iter_arrays.emplace_back(accumulator); - // Calculate function on arguments - auto replicated_column_function_ptr = IColumn::mutate(column_function->replicate(ColumnArray::Offsets(column_first_array->getOffsets().size(), 1))); - auto * replicated_column_function = typeid_cast(replicated_column_function_ptr.get()); - replicated_column_function->appendArguments(iter_arrays); - auto lambda_result = replicated_column_function->reduce().column; - if (lambda_result->lowCardinality()) - lambda_result = lambda_result->convertToFullColumnIfLowCardinality(); - res = lambda_result->empty() ? lambda_result : lambda_result->cut(0, 1); // TODO recheck this - accumulator.column = res; + auto const & arr = arrays[icolumn]; + iter_arrays.emplace_back(ColumnWithTypeAndName(arr.column->cut(arr_cursor, 1), + arr.type, + arr.name)); } - result->insert((*res)[0]); + iter_arrays.emplace_back(accumulator); + // Calculate function on arguments + auto replicated_column_function_ptr = IColumn::mutate(column_function->replicate(ColumnArray::Offsets(column_first_array->getOffsets().size(), 1))); + auto * replicated_column_function = typeid_cast(replicated_column_function_ptr.get()); + replicated_column_function->appendArguments(iter_arrays); + auto lambda_result = replicated_column_function->reduce().column; + if (lambda_result->lowCardinality()) + lambda_result = lambda_result->convertToFullColumnIfLowCardinality(); + res = lambda_result->empty() ? lambda_result : lambda_result->cut(0, 1); // TODO recheck this + accumulator.column = res; } - return result; - } - else - { - // TODO: for what this branch is needed? - // empty function - // - /// Put all the necessary columns multiplied by the sizes of arrays into the columns. - auto replicated_column_function_ptr = IColumn::mutate(column_function->replicate(column_first_array->getOffsets())); - auto * replicated_column_function = typeid_cast(replicated_column_function_ptr.get()); - replicated_column_function->appendArguments(arrays); - auto lambda_result = replicated_column_function->reduce().column; - if (lambda_result->lowCardinality()) - lambda_result = lambda_result->convertToFullColumnIfLowCardinality(); - // return lambda_result->empty() ? lambda_result : lambda_result->cut(0, 1); // TODO recheck this - return lambda_result; // TODO recheck this + result->insert((*res)[0]); } + return result; } }; From 582f6d3a72a0796ff344a8c65e05af6eecc10679 Mon Sep 17 00:00:00 2001 From: Dmitry Krylov Date: Thu, 1 Apr 2021 11:13:35 +1000 Subject: [PATCH 16/76] Remove unneeded branches of code --- src/Functions/array/arrayFold.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/array/arrayFold.cpp b/src/Functions/array/arrayFold.cpp index bf6f97bac7c..b82d84a77ba 100644 --- a/src/Functions/array/arrayFold.cpp +++ b/src/Functions/array/arrayFold.cpp @@ -157,7 +157,7 @@ public: auto lambda_result = replicated_column_function->reduce().column; if (lambda_result->lowCardinality()) lambda_result = lambda_result->convertToFullColumnIfLowCardinality(); - res = lambda_result->empty() ? lambda_result : lambda_result->cut(0, 1); // TODO recheck this + res = lambda_result->cut(0, 1); accumulator.column = res; } result->insert((*res)[0]); From 160d4a756e896a680ab485612eea57dabf72757e Mon Sep 17 00:00:00 2001 From: Dmitry Krylov Date: Thu, 1 Apr 2021 11:30:38 +1000 Subject: [PATCH 17/76] Tests for errors and tests for multi arrays --- tests/queries/0_stateless/90001_array_fold.reference | 2 ++ tests/queries/0_stateless/90001_array_fold.sql | 2 ++ ...ce => 90002_array_fold_data_from_tables.reference} | 0 ...fold.sql => 90002_array_fold_data_from_tables.sql} | 0 .../0_stateless/90003_array_fold_errors.reference | 0 tests/queries/0_stateless/90003_array_fold_errors.sql | 11 +++++++++++ 6 files changed, 15 insertions(+) rename tests/queries/0_stateless/{90002_array_fold.reference => 90002_array_fold_data_from_tables.reference} (100%) rename tests/queries/0_stateless/{90002_array_fold.sql => 90002_array_fold_data_from_tables.sql} (100%) create mode 100644 tests/queries/0_stateless/90003_array_fold_errors.reference create mode 100644 tests/queries/0_stateless/90003_array_fold_errors.sql diff --git a/tests/queries/0_stateless/90001_array_fold.reference b/tests/queries/0_stateless/90001_array_fold.reference index fd735503608..d0c64c8a31f 100644 --- a/tests/queries/0_stateless/90001_array_fold.reference +++ b/tests/queries/0_stateless/90001_array_fold.reference @@ -1,5 +1,7 @@ 23 3 +101 +269 [1,2,3,4] [4,3,2,1] ([4,3,2,1],[1,2,3,4]) diff --git a/tests/queries/0_stateless/90001_array_fold.sql b/tests/queries/0_stateless/90001_array_fold.sql index 74ca70735f5..545b7542ce6 100644 --- a/tests/queries/0_stateless/90001_array_fold.sql +++ b/tests/queries/0_stateless/90001_array_fold.sql @@ -1,5 +1,7 @@ SELECT arrayFold(x,acc -> acc + x * 2, [1,2,3,4], toInt64(3)); SELECT arrayFold(x,acc -> acc + x * 2, emptyArrayInt64(), toInt64(3)); +SELECT arrayFold(x,y,acc -> acc + x * 2 + y * 3, [1,2,3,4], [5,6,7,8], toInt64(3)); +SELECT arrayFold(x,y,z,acc -> acc + x * 2 + y * 3 + z * 4, [1,2,3,4], [5,6,7,8], [9,10,11,12], toInt64(3)); SELECT arrayFold(x,acc -> arrayPushBack(acc,x), [1,2,3,4], emptyArrayInt64()); SELECT arrayFold(x,acc -> arrayPushFront(acc,x), [1,2,3,4], emptyArrayInt64()); SELECT arrayFold(x,acc -> (arrayPushFront(acc.1,x), arrayPushBack(acc.2,x)), [1,2,3,4], (emptyArrayInt64(), emptyArrayInt64())); diff --git a/tests/queries/0_stateless/90002_array_fold.reference b/tests/queries/0_stateless/90002_array_fold_data_from_tables.reference similarity index 100% rename from tests/queries/0_stateless/90002_array_fold.reference rename to tests/queries/0_stateless/90002_array_fold_data_from_tables.reference diff --git a/tests/queries/0_stateless/90002_array_fold.sql b/tests/queries/0_stateless/90002_array_fold_data_from_tables.sql similarity index 100% rename from tests/queries/0_stateless/90002_array_fold.sql rename to tests/queries/0_stateless/90002_array_fold_data_from_tables.sql diff --git a/tests/queries/0_stateless/90003_array_fold_errors.reference b/tests/queries/0_stateless/90003_array_fold_errors.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/90003_array_fold_errors.sql b/tests/queries/0_stateless/90003_array_fold_errors.sql new file mode 100644 index 00000000000..49fd085dfe2 --- /dev/null +++ b/tests/queries/0_stateless/90003_array_fold_errors.sql @@ -0,0 +1,11 @@ +SELECT arrayFold([]); -- { serverError 42 } +SELECT arrayFold([1,2,3]); -- { serverError 42 } +SELECT arrayFold([1,2,3], [4,5,6]); -- { serverError 43 } +SELECT arrayFold(1234); -- { serverError 42 } +SELECT arrayFold(x, acc -> acc + x, 10, 20); -- { serverError 43 } +SELECT arrayFold(x, acc -> acc + x, 10, [20, 30, 40]); -- { serverError 43 } +SELECT arrayFold(x -> x * 2, [1,2,3,4], toInt64(3)); -- { serverError 43 } +SELECT arrayFold(x,acc -> acc+x, number, toInt64(0)) FROM system.numbers LIMIT 10; -- { serverError 43 } +SELECT arrayFold(x,y,acc -> acc + x * 2 + y * 3, [1,2,3,4], [5,6,7], toInt64(3)); -- { serverError 190 } +SELECT arrayFold(x,acc -> acc + x * 2 + y * 3, [1,2,3,4], [5,6,7,8], toInt64(3)); -- { serverError 47 } +SELECT arrayFold(x,acc -> acc + x * 2, [1,2,3,4], [5,6,7,8], toInt64(3)); -- { serverError 43 } From 923e9c644e6d3c758737273f85821710ced28e7b Mon Sep 17 00:00:00 2001 From: Dmitry Krylov Date: Thu, 1 Apr 2021 12:55:36 +1000 Subject: [PATCH 18/76] Adding documentation --- .../functions/array-functions.md | 56 ++++++++++++++++++ .../functions/array-functions.md | 57 +++++++++++++++++++ 2 files changed, 113 insertions(+) diff --git a/docs/en/sql-reference/functions/array-functions.md b/docs/en/sql-reference/functions/array-functions.md index 499376a70d4..b4d6143dddf 100644 --- a/docs/en/sql-reference/functions/array-functions.md +++ b/docs/en/sql-reference/functions/array-functions.md @@ -1213,6 +1213,62 @@ SELECT arrayFill(x -> not isNull(x), [1, null, 3, 11, 12, null, null, 5, 6, 14, Note that the `arrayFill` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You must pass a lambda function to it as the first argument, and it can’t be omitted. +## arrayFold(func, arr1, …, init) {#array-fold} + +Returns an result of [folding](https://en.wikipedia.org/wiki/Fold_(higher-order_function)) arrays and value `init` using function `func`. +I.e. result of calculation `func(arr1[n], …, func(arr1[n - 1], …, func(…, func(arr1[2], …, func(arr1[1], …, init)))))`. + +Note that the `arrayMap` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You must pass a lambda function to it as the first argument, and it can’t be omitted. + +**Arguments** + +- `func` — The lambda function with `n+1` arguments (where `n` is number of input arrays), first `n` arguments are for + current elements of input arrays, and last argument is for current value of accumulator. +- `arr` — Any number of [arrays](../../sql-reference/data-types/array.md). +- `init` - Initial value of accumulator. + +**Returned value** + +Final value of accumulator. + +**Examples** + +The following example shows how to acquire product and sum of elements of array: + +``` sql +SELECT arrayMap(x, accum -> (accum.1 * x, accum.2 + x), [1, 2, 3], (0, 1)) as res; +``` + +``` text +┌─res───────┐ +│ (120, 15) │ +└───────────┘ +``` + +The following example shows how to reverse elements of array: + +``` sql +SELECT arrayFold(x, acc -> arrayPushFront(acc, x), [1,2,3,4,5], emptyArrayUInt64()) as res; +``` + +``` text +┌─res─────────┐ +│ [5,4,3,2,1] │ +└─────────────┘ +``` + +Folding may be used to access of already passed elements due to function calculation, for example: + +``` sql +SELECT arrayFold(x, acc -> (x, concat(acc.2, toString(acc.1), ',')), [1,2], (0,'')) +``` + +``` text +┌─res────────┐ +│ (2,'0,1,') │ +└────────────┘ +``` + ## arrayReverseFill(func, arr1, …) {#array-reverse-fill} Scan through `arr1` from the last element to the first element and replace `arr1[i]` by `arr1[i + 1]` if `func` returns 0. The last element of `arr1` will not be replaced. diff --git a/docs/ru/sql-reference/functions/array-functions.md b/docs/ru/sql-reference/functions/array-functions.md index 560795506a0..4d6a37feef5 100644 --- a/docs/ru/sql-reference/functions/array-functions.md +++ b/docs/ru/sql-reference/functions/array-functions.md @@ -1147,6 +1147,62 @@ SELECT arrayReverseFill(x -> not isNull(x), [1, null, 3, 11, 12, null, null, 5, Функция `arrayReverseFill` является [функцией высшего порядка](../../sql-reference/functions/index.md#higher-order-functions) — в качестве первого аргумента ей нужно передать лямбда-функцию, и этот аргумент не может быть опущен. +## arrayFold(func, arr1, …, init) {#array-fold} + +Возвращает результат [сворачивания](https://ru.wikipedia.org/wiki/%D0%A1%D0%B2%D1%91%D1%80%D1%82%D0%BA%D0%B0_%D1%81%D0%BF%D0%B8%D1%81%D0%BA%D0%B0) массивов и начального значения `init` с помощью функции `func`. +Т.е. результат вычисления `func(arr1[n], …, func(arr1[n - 1], …, func(…, func(arr1[2], …, func(arr1[1], …, init)))))`. + +Функция `arrayFold` является [функцией высшего порядка](../../sql-reference/functions/index.md#higher-order-functions) — в качестве первого аргумента ей нужно передать лямбда-функцию, и этот аргумент не может быть опущен. + +**Аргументы** + +- `func` — лямбда-функция с `n+1` параметром (где `n` это количество входных массивов), причём первые `n` параметров + используются для текущих элементов входных массивов, а последний элемент для текущего значения аккумулятора. +- `arr` — произвольное количество [массивов](../../sql-reference/data-types/array.md). +- `init` - начальное значение аккумулятора. + +**Возвращаемое значение** + +Итоговое значение аккумулятора. + +**Примеры** + +Следующий пример показывает, как вычислить произведение и сумму элементов массива: + +``` sql +SELECT arrayMap(x, accum -> (accum.1 * x, accum.2 + x), [1, 2, 3], (0, 1)) as res; +``` + +``` text +┌─res───────┐ +│ (120, 15) │ +└───────────┘ +``` + +В этом примере показано, как обратить массив: + +``` sql +SELECT arrayFold(x, acc -> arrayPushFront(acc, x), [1,2,3,4,5], emptyArrayUInt64()) as res; +``` + +``` text +┌─res─────────┐ +│ [5,4,3,2,1] │ +└─────────────┘ +``` + +Свёртка может быть использована для доступа к уже пройденным в процессе вычисления элементам. Например: + +``` sql +SELECT arrayFold(x, acc -> (x, concat(acc.2, toString(acc.1), ',')), [1,2], (0,'')) +``` + +``` text +┌─res────────┐ +│ (2,'0,1,') │ +└────────────┘ +``` + ## arraySplit(func, arr1, …) {#array-split} Разделяет массив `arr1` на несколько. Если `func` возвращает не 0, то массив разделяется, а элемент помещается в левую часть. Массив не разбивается по первому элементу. @@ -1183,6 +1239,7 @@ SELECT arrayReverseSplit((x, y) -> y, [1, 2, 3, 4, 5], [1, 0, 0, 1, 0]) AS res Функция `arrayReverseSplit` является [функцией высшего порядка](../../sql-reference/functions/index.md#higher-order-functions) — в качестве первого аргумента ей нужно передать лямбда-функцию, и этот аргумент не может быть опущен. + ## arrayExists(\[func,\] arr1, …) {#arrayexistsfunc-arr1} Возвращает 1, если существует хотя бы один элемент массива `arr`, для которого функция func возвращает не 0. Иначе возвращает 0. From 602fb190b09850bb6a19befe6fe392d9aa7c9696 Mon Sep 17 00:00:00 2001 From: Dmitry Krylov Date: Thu, 1 Apr 2021 13:48:26 +1000 Subject: [PATCH 19/76] Benchmarks for `arrayFold` --- tests/performance/array_reduce_small.xml | 37 ++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 tests/performance/array_reduce_small.xml diff --git a/tests/performance/array_reduce_small.xml b/tests/performance/array_reduce_small.xml new file mode 100644 index 00000000000..e449559cf81 --- /dev/null +++ b/tests/performance/array_reduce_small.xml @@ -0,0 +1,37 @@ + + + + + SELECT arrayReduce('count', range(1000000)) + SELECT arrayReduce('sum', range(1000000)) + SELECT arrayReduceInRanges('count', [(1, 1000000)], range(1000000)) + SELECT arrayReduceInRanges('sum', [(1, 1000000)], range(1000000)) + SELECT arrayReduceInRanges('count', arrayZip(range(1000000), range(1000000)), range(1000000))[123456] + SELECT arrayReduceInRanges('sum', arrayZip(range(1000000), range(1000000)), range(1000000))[123456] + + SELECT arrayFold(x, acc -> acc + 1, range(1000000), toUInt64(0)) + SELECT arrayFold(x, acc -> acc + x, range(1000000), toUInt64(0)) + + + + + + From b5f6de3cf9432ed3b8e248acaa43b0a9ffa4572d Mon Sep 17 00:00:00 2001 From: Dmitry Krylov Date: Sun, 11 Apr 2021 12:06:29 +1000 Subject: [PATCH 20/76] Renaming tests --- .../{90001_array_fold.reference => 01810_array_fold.reference} | 0 .../0_stateless/{90001_array_fold.sql => 01811_array_fold.sql} | 0 ...bles.reference => 01812_array_fold_data_from_tables.reference} | 0 ...data_from_tables.sql => 01813_array_fold_data_from_tables.sql} | 0 ...ay_fold_errors.reference => 01814_array_fold_errors.reference} | 0 .../{90003_array_fold_errors.sql => 01815_array_fold_errors.sql} | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename tests/queries/0_stateless/{90001_array_fold.reference => 01810_array_fold.reference} (100%) rename tests/queries/0_stateless/{90001_array_fold.sql => 01811_array_fold.sql} (100%) rename tests/queries/0_stateless/{90002_array_fold_data_from_tables.reference => 01812_array_fold_data_from_tables.reference} (100%) rename tests/queries/0_stateless/{90002_array_fold_data_from_tables.sql => 01813_array_fold_data_from_tables.sql} (100%) rename tests/queries/0_stateless/{90003_array_fold_errors.reference => 01814_array_fold_errors.reference} (100%) rename tests/queries/0_stateless/{90003_array_fold_errors.sql => 01815_array_fold_errors.sql} (100%) diff --git a/tests/queries/0_stateless/90001_array_fold.reference b/tests/queries/0_stateless/01810_array_fold.reference similarity index 100% rename from tests/queries/0_stateless/90001_array_fold.reference rename to tests/queries/0_stateless/01810_array_fold.reference diff --git a/tests/queries/0_stateless/90001_array_fold.sql b/tests/queries/0_stateless/01811_array_fold.sql similarity index 100% rename from tests/queries/0_stateless/90001_array_fold.sql rename to tests/queries/0_stateless/01811_array_fold.sql diff --git a/tests/queries/0_stateless/90002_array_fold_data_from_tables.reference b/tests/queries/0_stateless/01812_array_fold_data_from_tables.reference similarity index 100% rename from tests/queries/0_stateless/90002_array_fold_data_from_tables.reference rename to tests/queries/0_stateless/01812_array_fold_data_from_tables.reference diff --git a/tests/queries/0_stateless/90002_array_fold_data_from_tables.sql b/tests/queries/0_stateless/01813_array_fold_data_from_tables.sql similarity index 100% rename from tests/queries/0_stateless/90002_array_fold_data_from_tables.sql rename to tests/queries/0_stateless/01813_array_fold_data_from_tables.sql diff --git a/tests/queries/0_stateless/90003_array_fold_errors.reference b/tests/queries/0_stateless/01814_array_fold_errors.reference similarity index 100% rename from tests/queries/0_stateless/90003_array_fold_errors.reference rename to tests/queries/0_stateless/01814_array_fold_errors.reference diff --git a/tests/queries/0_stateless/90003_array_fold_errors.sql b/tests/queries/0_stateless/01815_array_fold_errors.sql similarity index 100% rename from tests/queries/0_stateless/90003_array_fold_errors.sql rename to tests/queries/0_stateless/01815_array_fold_errors.sql From 5c24225d86e7126f9dc67a0726ff98d3e569a1e9 Mon Sep 17 00:00:00 2001 From: Dmitry Krylov Date: Sun, 11 Apr 2021 12:09:22 +1000 Subject: [PATCH 21/76] Fix performance tests --- tests/performance/array_fold_small.xml | 4 +++ tests/performance/array_reduce_small.xml | 37 ------------------------ 2 files changed, 4 insertions(+), 37 deletions(-) create mode 100644 tests/performance/array_fold_small.xml delete mode 100644 tests/performance/array_reduce_small.xml diff --git a/tests/performance/array_fold_small.xml b/tests/performance/array_fold_small.xml new file mode 100644 index 00000000000..96b30ae8ace --- /dev/null +++ b/tests/performance/array_fold_small.xml @@ -0,0 +1,4 @@ + + SELECT arrayFold(x, acc -> acc + 1, range(100000), toUInt64(0)) + SELECT arrayFold(x, acc -> acc + x, range(100000), toUInt64(0)) + diff --git a/tests/performance/array_reduce_small.xml b/tests/performance/array_reduce_small.xml deleted file mode 100644 index e449559cf81..00000000000 --- a/tests/performance/array_reduce_small.xml +++ /dev/null @@ -1,37 +0,0 @@ - - - - - SELECT arrayReduce('count', range(1000000)) - SELECT arrayReduce('sum', range(1000000)) - SELECT arrayReduceInRanges('count', [(1, 1000000)], range(1000000)) - SELECT arrayReduceInRanges('sum', [(1, 1000000)], range(1000000)) - SELECT arrayReduceInRanges('count', arrayZip(range(1000000), range(1000000)), range(1000000))[123456] - SELECT arrayReduceInRanges('sum', arrayZip(range(1000000), range(1000000)), range(1000000))[123456] - - SELECT arrayFold(x, acc -> acc + 1, range(1000000), toUInt64(0)) - SELECT arrayFold(x, acc -> acc + x, range(1000000), toUInt64(0)) - - - - - - From 35472bcc2ff09dea516cbea7fd348ba95833d2bf Mon Sep 17 00:00:00 2001 From: Dmitry Krylov Date: Mon, 12 Apr 2021 11:37:44 +1000 Subject: [PATCH 22/76] Update to new IFunction::create() --- src/Functions/array/arrayFold.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/array/arrayFold.cpp b/src/Functions/array/arrayFold.cpp index b82d84a77ba..78dabfff025 100644 --- a/src/Functions/array/arrayFold.cpp +++ b/src/Functions/array/arrayFold.cpp @@ -11,7 +11,7 @@ class FunctionArrayFold : public IFunction { public: static constexpr auto name = "arrayFold"; - static FunctionPtr create(const Context &) { return std::make_shared(); } + static FunctionPtr create(ContextPtr) { return std::make_shared(); } String getName() const override { return name; } bool isVariadic() const override { return true; } From 68bd27f3c5f5edffd3f2fc57fe02c953a2bfa420 Mon Sep 17 00:00:00 2001 From: Dmitry Krylov Date: Mon, 12 Apr 2021 12:12:40 +1000 Subject: [PATCH 23/76] Fix code style --- src/Functions/array/arrayFold.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/Functions/array/arrayFold.cpp b/src/Functions/array/arrayFold.cpp index 78dabfff025..546c5a627bb 100644 --- a/src/Functions/array/arrayFold.cpp +++ b/src/Functions/array/arrayFold.cpp @@ -5,6 +5,15 @@ namespace DB { +namespace ErrorCodes +{ + extern const int ILLEGAL_COLUMN; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int SIZES_OF_ARRAYS_DOESNT_MATCH; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + + /** arrayFold(x1,...,xn,accum -> expression, array1,...,arrayn, init_accum) - apply the expression to each element of the array (or set of parallel arrays). */ class FunctionArrayFold : public IFunction From 18f58b0c49d8ac913f00fc37b9c33199952b6caa Mon Sep 17 00:00:00 2001 From: Dmitry Krylov Date: Mon, 12 Apr 2021 12:16:20 +1000 Subject: [PATCH 24/76] Fix test filenames --- tests/queries/0_stateless/01810_array_fold.reference | 8 -------- .../0_stateless/01813_array_fold_data_from_tables.sql | 8 -------- .../0_stateless/01814_array_fold_errors.reference | 0 tests/queries/0_stateless/01815_array_fold_errors.sql | 11 ----------- 4 files changed, 27 deletions(-) delete mode 100644 tests/queries/0_stateless/01810_array_fold.reference delete mode 100644 tests/queries/0_stateless/01813_array_fold_data_from_tables.sql delete mode 100644 tests/queries/0_stateless/01814_array_fold_errors.reference delete mode 100644 tests/queries/0_stateless/01815_array_fold_errors.sql diff --git a/tests/queries/0_stateless/01810_array_fold.reference b/tests/queries/0_stateless/01810_array_fold.reference deleted file mode 100644 index d0c64c8a31f..00000000000 --- a/tests/queries/0_stateless/01810_array_fold.reference +++ /dev/null @@ -1,8 +0,0 @@ -23 -3 -101 -269 -[1,2,3,4] -[4,3,2,1] -([4,3,2,1],[1,2,3,4]) -([1,3,5],[2,4,6]) diff --git a/tests/queries/0_stateless/01813_array_fold_data_from_tables.sql b/tests/queries/0_stateless/01813_array_fold_data_from_tables.sql deleted file mode 100644 index 23e85ead56f..00000000000 --- a/tests/queries/0_stateless/01813_array_fold_data_from_tables.sql +++ /dev/null @@ -1,8 +0,0 @@ -SELECT arrayFold(x,acc -> acc+x, range(number), toInt64(0)) FROM system.numbers LIMIT 10; -SELECT arrayFold(x,acc -> acc+x, range(number), number) FROM system.numbers LIMIT 10; -SELECT arrayFold(x,acc -> arrayPushFront(acc, x), range(number), emptyArrayUInt64()) FROM system.numbers LIMIT 10; -SELECT arrayFold(x,acc -> x % 2 ? arrayPushFront(acc, x) : arrayPushBack(acc, x), range(number), emptyArrayUInt64()) FROM system.numbers LIMIT 10; -SELECT arrayFold(x,acc -> (acc.1+x, acc.2-x), range(number), (toInt64(0), toInt64(0))) FROM system.numbers LIMIT 10; -SELECT arrayFold(x,acc -> (acc.1+x.1, acc.2-x.2), arrayZip(range(number), range(number)), (toInt64(0), toInt64(0))) FROM system.numbers LIMIT 10; -SELECT arrayFold(x,acc -> arrayPushFront(acc, (x, x+1)), range(number), [(toUInt64(0),toUInt64(0))]) FROM system.numbers LIMIT 10; -SELECT arrayFold(x, acc -> concat(acc, arrayMap(z -> toString(x), [number])) , range(number), CAST([] as Array(String))) FROM system.numbers LIMIT 10; diff --git a/tests/queries/0_stateless/01814_array_fold_errors.reference b/tests/queries/0_stateless/01814_array_fold_errors.reference deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/queries/0_stateless/01815_array_fold_errors.sql b/tests/queries/0_stateless/01815_array_fold_errors.sql deleted file mode 100644 index 49fd085dfe2..00000000000 --- a/tests/queries/0_stateless/01815_array_fold_errors.sql +++ /dev/null @@ -1,11 +0,0 @@ -SELECT arrayFold([]); -- { serverError 42 } -SELECT arrayFold([1,2,3]); -- { serverError 42 } -SELECT arrayFold([1,2,3], [4,5,6]); -- { serverError 43 } -SELECT arrayFold(1234); -- { serverError 42 } -SELECT arrayFold(x, acc -> acc + x, 10, 20); -- { serverError 43 } -SELECT arrayFold(x, acc -> acc + x, 10, [20, 30, 40]); -- { serverError 43 } -SELECT arrayFold(x -> x * 2, [1,2,3,4], toInt64(3)); -- { serverError 43 } -SELECT arrayFold(x,acc -> acc+x, number, toInt64(0)) FROM system.numbers LIMIT 10; -- { serverError 43 } -SELECT arrayFold(x,y,acc -> acc + x * 2 + y * 3, [1,2,3,4], [5,6,7], toInt64(3)); -- { serverError 190 } -SELECT arrayFold(x,acc -> acc + x * 2 + y * 3, [1,2,3,4], [5,6,7,8], toInt64(3)); -- { serverError 47 } -SELECT arrayFold(x,acc -> acc + x * 2, [1,2,3,4], [5,6,7,8], toInt64(3)); -- { serverError 43 } From a5a4fbdb0007ecd0af1a111eb6309fe33a038e0a Mon Sep 17 00:00:00 2001 From: Dmitry Krylov Date: Mon, 12 Apr 2021 13:09:36 +1000 Subject: [PATCH 25/76] Fix test filenames --- tests/queries/0_stateless/01811_array_fold.reference | 8 ++++++++ .../0_stateless/01812_array_fold_data_from_tables.sql | 8 ++++++++ .../0_stateless/01813_array_fold_errors.reference | 0 tests/queries/0_stateless/01813_array_fold_errors.sql | 11 +++++++++++ 4 files changed, 27 insertions(+) create mode 100644 tests/queries/0_stateless/01811_array_fold.reference create mode 100644 tests/queries/0_stateless/01812_array_fold_data_from_tables.sql create mode 100644 tests/queries/0_stateless/01813_array_fold_errors.reference create mode 100644 tests/queries/0_stateless/01813_array_fold_errors.sql diff --git a/tests/queries/0_stateless/01811_array_fold.reference b/tests/queries/0_stateless/01811_array_fold.reference new file mode 100644 index 00000000000..d0c64c8a31f --- /dev/null +++ b/tests/queries/0_stateless/01811_array_fold.reference @@ -0,0 +1,8 @@ +23 +3 +101 +269 +[1,2,3,4] +[4,3,2,1] +([4,3,2,1],[1,2,3,4]) +([1,3,5],[2,4,6]) diff --git a/tests/queries/0_stateless/01812_array_fold_data_from_tables.sql b/tests/queries/0_stateless/01812_array_fold_data_from_tables.sql new file mode 100644 index 00000000000..23e85ead56f --- /dev/null +++ b/tests/queries/0_stateless/01812_array_fold_data_from_tables.sql @@ -0,0 +1,8 @@ +SELECT arrayFold(x,acc -> acc+x, range(number), toInt64(0)) FROM system.numbers LIMIT 10; +SELECT arrayFold(x,acc -> acc+x, range(number), number) FROM system.numbers LIMIT 10; +SELECT arrayFold(x,acc -> arrayPushFront(acc, x), range(number), emptyArrayUInt64()) FROM system.numbers LIMIT 10; +SELECT arrayFold(x,acc -> x % 2 ? arrayPushFront(acc, x) : arrayPushBack(acc, x), range(number), emptyArrayUInt64()) FROM system.numbers LIMIT 10; +SELECT arrayFold(x,acc -> (acc.1+x, acc.2-x), range(number), (toInt64(0), toInt64(0))) FROM system.numbers LIMIT 10; +SELECT arrayFold(x,acc -> (acc.1+x.1, acc.2-x.2), arrayZip(range(number), range(number)), (toInt64(0), toInt64(0))) FROM system.numbers LIMIT 10; +SELECT arrayFold(x,acc -> arrayPushFront(acc, (x, x+1)), range(number), [(toUInt64(0),toUInt64(0))]) FROM system.numbers LIMIT 10; +SELECT arrayFold(x, acc -> concat(acc, arrayMap(z -> toString(x), [number])) , range(number), CAST([] as Array(String))) FROM system.numbers LIMIT 10; diff --git a/tests/queries/0_stateless/01813_array_fold_errors.reference b/tests/queries/0_stateless/01813_array_fold_errors.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01813_array_fold_errors.sql b/tests/queries/0_stateless/01813_array_fold_errors.sql new file mode 100644 index 00000000000..49fd085dfe2 --- /dev/null +++ b/tests/queries/0_stateless/01813_array_fold_errors.sql @@ -0,0 +1,11 @@ +SELECT arrayFold([]); -- { serverError 42 } +SELECT arrayFold([1,2,3]); -- { serverError 42 } +SELECT arrayFold([1,2,3], [4,5,6]); -- { serverError 43 } +SELECT arrayFold(1234); -- { serverError 42 } +SELECT arrayFold(x, acc -> acc + x, 10, 20); -- { serverError 43 } +SELECT arrayFold(x, acc -> acc + x, 10, [20, 30, 40]); -- { serverError 43 } +SELECT arrayFold(x -> x * 2, [1,2,3,4], toInt64(3)); -- { serverError 43 } +SELECT arrayFold(x,acc -> acc+x, number, toInt64(0)) FROM system.numbers LIMIT 10; -- { serverError 43 } +SELECT arrayFold(x,y,acc -> acc + x * 2 + y * 3, [1,2,3,4], [5,6,7], toInt64(3)); -- { serverError 190 } +SELECT arrayFold(x,acc -> acc + x * 2 + y * 3, [1,2,3,4], [5,6,7,8], toInt64(3)); -- { serverError 47 } +SELECT arrayFold(x,acc -> acc + x * 2, [1,2,3,4], [5,6,7,8], toInt64(3)); -- { serverError 43 } From a3d57bd5aff3f6f57cc0e3ec7fe5e9eefbebe7b2 Mon Sep 17 00:00:00 2001 From: Dmitry Krylov Date: Mon, 12 Apr 2021 20:29:35 +1000 Subject: [PATCH 26/76] Check number of arguments --- src/Functions/array/arrayFold.cpp | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/Functions/array/arrayFold.cpp b/src/Functions/array/arrayFold.cpp index 546c5a627bb..bd38a25d77a 100644 --- a/src/Functions/array/arrayFold.cpp +++ b/src/Functions/array/arrayFold.cpp @@ -30,13 +30,8 @@ public: /// For argument-lambda expressions, it defines the types of arguments of these expressions. void getLambdaArgumentTypes(DataTypes & arguments) const override { - if (arguments.empty()) - throw Exception("Function " + getName() + " needs at least one argument; passed " - + toString(arguments.size()) + ".", - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - - if (arguments.size() == 1) - throw Exception("Function " + getName() + " needs at least one array argument.", + if (arguments.size() < 3) + throw Exception("Function " + getName() + " needs lambda function, at least one array argument and one accumulator argument.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); DataTypes nested_types(arguments.size() - 1); From df6072484718589413d0fab7c5a6d7ddd41897d8 Mon Sep 17 00:00:00 2001 From: Dmitry Krylov Date: Mon, 12 Apr 2021 20:29:58 +1000 Subject: [PATCH 27/76] Fix constness --- src/Functions/array/arrayFold.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Functions/array/arrayFold.cpp b/src/Functions/array/arrayFold.cpp index bd38a25d77a..5c80b01c5c9 100644 --- a/src/Functions/array/arrayFold.cpp +++ b/src/Functions/array/arrayFold.cpp @@ -65,9 +65,7 @@ public: throw Exception("First argument for function " + getName() + " must be a function.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); /// The types of the remaining arguments are already checked in getLambdaArgumentTypes. - DataTypePtr return_type = removeLowCardinality(data_type_function->getReturnType()); - const auto accum_type = arguments.back().type; - return accum_type; + return DataTypePtr(arguments.back().type); } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override From d5580a8e71d736da4267bf05a4a2310070c31f24 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 12 Apr 2021 23:02:42 +0300 Subject: [PATCH 28/76] Dynamic dispatch for intDiv --- src/Functions/intDiv.cpp | 126 ++++++++++++++++++++++++++++++++------- 1 file changed, 105 insertions(+), 21 deletions(-) diff --git a/src/Functions/intDiv.cpp b/src/Functions/intDiv.cpp index 804696f2776..42b0299ce01 100644 --- a/src/Functions/intDiv.cpp +++ b/src/Functions/intDiv.cpp @@ -1,12 +1,29 @@ #include #include +#include -#if defined(__SSE2__) -# define LIBDIVIDE_SSE2 1 +#if defined(__x86_64__) + #define LIBDIVIDE_SSE2 1 + #define LIBDIVIDE_AVX2 1 + + #if defined(__clang__) + #pragma clang attribute push(__attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2"))), apply_to=function) + #else + #pragma GCC push_options + #pragma GCC target("sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,tune=native") + #endif #endif #include +#if defined(__x86_64__) + #if defined(__clang__) + #pragma clang attribute pop + #else + #pragma GCC pop_options + #endif +#endif + namespace DB { @@ -20,6 +37,83 @@ namespace /// Optimizations for integer division by a constant. +#if defined(__x86_64__) + +DECLARE_DEFAULT_CODE ( + template + void divideImpl(const A * __restrict a_pos, B b, ResultType * __restrict c_pos, size_t size) + { + libdivide::divider divider(b); + const A * a_end = a_pos + size; + + static constexpr size_t values_per_simd_register = 16 / sizeof(A); + const A * a_end_simd = a_pos + size / values_per_simd_register * values_per_simd_register; + + while (a_pos < a_end_simd) + { + _mm_storeu_si128(reinterpret_cast<__m128i *>(c_pos), + _mm_loadu_si128(reinterpret_cast(a_pos)) / divider); + + a_pos += values_per_simd_register; + c_pos += values_per_simd_register; + } + + while (a_pos < a_end) + { + *c_pos = *a_pos / divider; + ++a_pos; + ++c_pos; + } + } +) + +DECLARE_AVX2_SPECIFIC_CODE ( + template + void divideImpl(const A * __restrict a_pos, B b, ResultType * __restrict c_pos, size_t size) + { + libdivide::divider divider(b); + const A * a_end = a_pos + size; + + static constexpr size_t values_per_simd_register = 32 / sizeof(A); + const A * a_end_simd = a_pos + size / values_per_simd_register * values_per_simd_register; + + while (a_pos < a_end_simd) + { + _mm256_storeu_si256(reinterpret_cast<__m256i *>(c_pos), + _mm256_loadu_si256(reinterpret_cast(a_pos)) / divider); + + a_pos += values_per_simd_register; + c_pos += values_per_simd_register; + } + + while (a_pos < a_end) + { + *c_pos = *a_pos / divider; + ++a_pos; + ++c_pos; + } + } +) + +#else + +template +void divideImpl(const A * __restrict a_pos, B b, ResultType * __restrict c_pos, size_t size) +{ + libdivide::divider divider(b); + const A * a_end = a_pos + size; + + while (a_pos < a_end) + { + *c_pos = *a_pos / divider; + ++a_pos; + ++c_pos; + } +} + +#endif + + template struct DivideIntegralByConstantImpl : BinaryOperation> @@ -70,29 +164,19 @@ struct DivideIntegralByConstantImpl if (unlikely(static_cast(b) == 0)) throw Exception("Division by zero", ErrorCodes::ILLEGAL_DIVISION); - libdivide::divider divider(b); - - const A * a_end = a_pos + size; - -#if defined(__SSE2__) - static constexpr size_t values_per_sse_register = 16 / sizeof(A); - const A * a_end_sse = a_pos + size / values_per_sse_register * values_per_sse_register; - - while (a_pos < a_end_sse) +#if USE_MULTITARGET_CODE + if (isArchSupported(TargetArch::AVX2)) { - _mm_storeu_si128(reinterpret_cast<__m128i *>(c_pos), - _mm_loadu_si128(reinterpret_cast(a_pos)) / divider); - - a_pos += values_per_sse_register; - c_pos += values_per_sse_register; + TargetSpecific::AVX2::divideImpl(a_pos, b, c_pos, size); } + else #endif - - while (a_pos < a_end) { - *c_pos = *a_pos / divider; - ++a_pos; - ++c_pos; +#if __x86_64__ + TargetSpecific::Default::divideImpl(a_pos, b, c_pos, size); +#else + divideImpl(a_pos, b, c_pos, size); +#endif } } }; From 3effb74d317ac88809de1e9dd6fb3574d5984c1e Mon Sep 17 00:00:00 2001 From: songenjie Date: Thu, 8 Apr 2021 11:29:36 +0800 Subject: [PATCH 29/76] [clickhouse][server][ddl] add fetch part docs --- .../sql-reference/statements/alter/partition.md | 16 +++++++++++----- docs/en/sql-reference/statements/grant.md | 2 +- .../en/2016/how-to-update-data-in-clickhouse.md | 2 +- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/docs/en/sql-reference/statements/alter/partition.md b/docs/en/sql-reference/statements/alter/partition.md index f7183ba525c..23f10684ae7 100644 --- a/docs/en/sql-reference/statements/alter/partition.md +++ b/docs/en/sql-reference/statements/alter/partition.md @@ -16,7 +16,7 @@ The following operations with [partitions](../../../engines/table-engines/merget - [CLEAR COLUMN IN PARTITION](#alter_clear-column-partition) — Resets the value of a specified column in a partition. - [CLEAR INDEX IN PARTITION](#alter_clear-index-partition) — Resets the specified secondary index in a partition. - [FREEZE PARTITION](#alter_freeze-partition) — Creates a backup of a partition. -- [FETCH PARTITION](#alter_fetch-partition) — Downloads a partition from another server. +- [FETCH PART\|PARTITION](#alter_fetch-partition) — Downloads a part or partition from another server. - [MOVE PARTITION\|PART](#alter_move-partition) — Move partition/data part to another disk or volume. @@ -198,29 +198,35 @@ ALTER TABLE table_name CLEAR INDEX index_name IN PARTITION partition_expr The query works similar to `CLEAR COLUMN`, but it resets an index instead of a column data. -## FETCH PARTITION {#alter_fetch-partition} +## FETCH PART|PARTITION {#alter_fetch-partition} ``` sql -ALTER TABLE table_name FETCH PARTITION partition_expr FROM 'path-in-zookeeper' +ALTER TABLE table_name FETCH PART|PARTITION partition_expr FROM 'path-in-zookeeper' ``` Downloads a partition from another server. This query only works for the replicated tables. The query does the following: -1. Downloads the partition from the specified shard. In ‘path-in-zookeeper’ you must specify a path to the shard in ZooKeeper. +1. Downloads the partition|part from the specified shard. In ‘path-in-zookeeper’ you must specify a path to the shard in ZooKeeper. 2. Then the query puts the downloaded data to the `detached` directory of the `table_name` table. Use the [ATTACH PARTITION\|PART](#alter_attach-partition) query to add the data to the table. For example: +1. FETCH PARTITION ``` sql ALTER TABLE users FETCH PARTITION 201902 FROM '/clickhouse/tables/01-01/visits'; ALTER TABLE users ATTACH PARTITION 201902; ``` +2. FETCH PART +``` sql +ALTER TABLE users FETCH PART 201901_2_2_0 FROM '/clickhouse/tables/01-01/visits'; +ALTER TABLE users ATTACH PART 201901_2_2_0; +``` Note that: -- The `ALTER ... FETCH PARTITION` query isn’t replicated. It places the partition to the `detached` directory only on the local server. +- The `ALTER ... FETCH PART|PARTITION` query isn’t replicated. It places the part or partition to the `detached` directory only on the local server. - The `ALTER TABLE ... ATTACH` query is replicated. It adds the data to all replicas. The data is added to one of the replicas from the `detached` directory, and to the others - from neighboring replicas. Before downloading, the system checks if the partition exists and the table structure matches. The most appropriate replica is selected automatically from the healthy replicas. diff --git a/docs/en/sql-reference/statements/grant.md b/docs/en/sql-reference/statements/grant.md index 0afc9b5b95f..daa020f9469 100644 --- a/docs/en/sql-reference/statements/grant.md +++ b/docs/en/sql-reference/statements/grant.md @@ -279,7 +279,7 @@ Allows executing [ALTER](../../sql-reference/statements/alter/index.md) queries - `ALTER MATERIALIZE TTL`. Level: `TABLE`. Aliases: `MATERIALIZE TTL` - `ALTER SETTINGS`. Level: `TABLE`. Aliases: `ALTER SETTING`, `ALTER MODIFY SETTING`, `MODIFY SETTING` - `ALTER MOVE PARTITION`. Level: `TABLE`. Aliases: `ALTER MOVE PART`, `MOVE PARTITION`, `MOVE PART` - - `ALTER FETCH PARTITION`. Level: `TABLE`. Aliases: `FETCH PARTITION` + - `ALTER FETCH PARTITION`. Level: `TABLE`. Aliases: `FETCH PARTITION`, `FETCH PART` - `ALTER FREEZE PARTITION`. Level: `TABLE`. Aliases: `FREEZE PARTITION` - `ALTER VIEW` Level: `GROUP` - `ALTER VIEW REFRESH`. Level: `VIEW`. Aliases: `ALTER LIVE VIEW REFRESH`, `REFRESH VIEW` diff --git a/website/blog/en/2016/how-to-update-data-in-clickhouse.md b/website/blog/en/2016/how-to-update-data-in-clickhouse.md index 22c2fa3ccc1..7e9b938203f 100644 --- a/website/blog/en/2016/how-to-update-data-in-clickhouse.md +++ b/website/blog/en/2016/how-to-update-data-in-clickhouse.md @@ -67,7 +67,7 @@ There is a nice set of operations to work with partitions: - `DROP PARTITION` - Delete a partition. - `ATTACH PART|PARTITION` -- Add a new part or partition from the 'detached' directory to the table. - `FREEZE PARTITION` - Create a backup of a partition. -- `FETCH PARTITION` - Download a partition from another server. +- `FETCH PART|PARTITION` - Download a part or partition from another server. We can do any data management operations on partitions level: move, copy and delete. Also, special DETACH and ATTACH operations are created to simplify data manipulation. DETACH detaches partition from table, moving all data to detached directory. Data is still there and you can copy it anywhere but detached data is not visible on request level. ATTACH is the opposite: attaches data from detached directory so it become visible. From 564136ec46ed7847ff296869c8b6156ee202f34b Mon Sep 17 00:00:00 2001 From: songenjie Date: Tue, 13 Apr 2021 12:40:33 +0800 Subject: [PATCH 30/76] [clickhouse][server][dll][alter]support fetch part --- src/Access/AccessType.h | 2 +- src/Common/ErrorCodes.cpp | 2 + src/Parsers/ASTAlterQuery.cpp | 2 +- src/Parsers/ParserAlterQuery.cpp | 16 +++ src/Storages/MergeTree/MergeTreeData.cpp | 9 +- src/Storages/MergeTree/MergeTreeData.h | 7 +- src/Storages/PartitionCommands.cpp | 6 +- src/Storages/StorageReplicatedMergeTree.cpp | 102 ++++++++++++++---- src/Storages/StorageReplicatedMergeTree.h | 10 +- tests/fuzz/ast.dict | 1 + .../__init__.py | 0 .../configs/zookeeper_config.xml | 28 +++++ .../test.py | 40 +++++++ .../test.py | 2 +- .../rbac/tests/privileges/grant_option.py | 2 +- 15 files changed, 198 insertions(+), 31 deletions(-) create mode 100644 tests/integration/test_fetch_part_from_auxiliary_zookeeper/__init__.py create mode 100644 tests/integration/test_fetch_part_from_auxiliary_zookeeper/configs/zookeeper_config.xml create mode 100644 tests/integration/test_fetch_part_from_auxiliary_zookeeper/test.py diff --git a/src/Access/AccessType.h b/src/Access/AccessType.h index 40740b3164e..c7311997ba2 100644 --- a/src/Access/AccessType.h +++ b/src/Access/AccessType.h @@ -62,7 +62,7 @@ enum class AccessType enabled implicitly by the grant ALTER_TABLE */\ M(ALTER_SETTINGS, "ALTER SETTING, ALTER MODIFY SETTING, MODIFY SETTING", TABLE, ALTER_TABLE) /* allows to execute ALTER MODIFY SETTING */\ M(ALTER_MOVE_PARTITION, "ALTER MOVE PART, MOVE PARTITION, MOVE PART", TABLE, ALTER_TABLE) \ - M(ALTER_FETCH_PARTITION, "FETCH PARTITION", TABLE, ALTER_TABLE) \ + M(ALTER_FETCH_PARTITION, "ALTER FETCH PART, FETCH PARTITION, FETCH PART", TABLE, ALTER_TABLE) \ M(ALTER_FREEZE_PARTITION, "FREEZE PARTITION, UNFREEZE", TABLE, ALTER_TABLE) \ \ M(ALTER_TABLE, "", GROUP, ALTER) \ diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index ad0463db889..7658448976b 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -549,6 +549,8 @@ M(579, INCORRECT_PART_TYPE) \ M(580, CANNOT_SET_ROUNDING_MODE) \ M(581, TOO_LARGE_DISTRIBUTED_DEPTH) \ + M(582, PART_DOESNT_EXIST) \ + M(583, PART_ALREADY_EXISTS) \ \ M(998, POSTGRESQL_CONNECTION_FAILURE) \ M(999, KEEPER_EXCEPTION) \ diff --git a/src/Parsers/ASTAlterQuery.cpp b/src/Parsers/ASTAlterQuery.cpp index df4a9a5f99a..5b052bae856 100644 --- a/src/Parsers/ASTAlterQuery.cpp +++ b/src/Parsers/ASTAlterQuery.cpp @@ -245,7 +245,7 @@ void ASTAlterCommand::formatImpl( else if (type == ASTAlterCommand::FETCH_PARTITION) { settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "FETCH " - << "PARTITION " << (settings.hilite ? hilite_none : ""); + << (part ? "PART " : "PARTITION ") << (settings.hilite ? hilite_none : ""); partition->formatImpl(settings, state, frame); settings.ostr << (settings.hilite ? hilite_keyword : "") << " FROM " << (settings.hilite ? hilite_none : "") << DB::quote << from; diff --git a/src/Parsers/ParserAlterQuery.cpp b/src/Parsers/ParserAlterQuery.cpp index e5cc4b1b95e..de524342fb4 100644 --- a/src/Parsers/ParserAlterQuery.cpp +++ b/src/Parsers/ParserAlterQuery.cpp @@ -61,6 +61,7 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected ParserKeyword s_drop_detached_partition("DROP DETACHED PARTITION"); ParserKeyword s_drop_detached_part("DROP DETACHED PART"); ParserKeyword s_fetch_partition("FETCH PARTITION"); + ParserKeyword s_fetch_part("FETCH PART"); ParserKeyword s_replace_partition("REPLACE PARTITION"); ParserKeyword s_freeze("FREEZE"); ParserKeyword s_unfreeze("UNFREEZE"); @@ -428,6 +429,21 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected command->from = ast_from->as().value.get(); command->type = ASTAlterCommand::FETCH_PARTITION; } + else if (s_fetch_part.ignore(pos, expected)) + { + if (!parser_string_literal.parse(pos, command->partition, expected)) + return false; + + if (!s_from.ignore(pos, expected)) + return false; + + ASTPtr ast_from; + if (!parser_string_literal.parse(pos, ast_from, expected)) + return false; + command->from = ast_from->as().value.get(); + command->part = true; + command->type = ASTAlterCommand::FETCH_PARTITION; + } else if (s_freeze.ignore(pos, expected)) { if (s_partition.ignore(pos, expected)) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index ee8e15008cb..b5da72c517f 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -2909,7 +2909,12 @@ void MergeTreeData::movePartitionToVolume(const ASTPtr & partition, const String throw Exception("Cannot move parts because moves are manually disabled", ErrorCodes::ABORTED); } -void MergeTreeData::fetchPartition(const ASTPtr & /*partition*/, const StorageMetadataPtr & /*metadata_snapshot*/, const String & /*from*/, ContextPtr /*query_context*/) +void MergeTreeData::fetchPartition( + const ASTPtr & /*partition*/, + const StorageMetadataPtr & /*metadata_snapshot*/, + const String & /*from*/, + bool /*fetch_part*/, + ContextPtr /*query_context*/) { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "FETCH PARTITION is not supported by storage {}", getName()); } @@ -2972,7 +2977,7 @@ Pipe MergeTreeData::alterPartition( break; case PartitionCommand::FETCH_PARTITION: - fetchPartition(command.partition, metadata_snapshot, command.from_zookeeper_path, query_context); + fetchPartition(command.partition, metadata_snapshot, command.from_zookeeper_path, command.part, query_context); break; case PartitionCommand::FREEZE_PARTITION: diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 6c0bda07bb1..46c0014d9f7 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -970,7 +970,12 @@ protected: virtual void movePartitionToTable(const StoragePtr & dest_table, const ASTPtr & partition, ContextPtr context) = 0; /// Makes sense only for replicated tables - virtual void fetchPartition(const ASTPtr & partition, const StorageMetadataPtr & metadata_snapshot, const String & from, ContextPtr query_context); + virtual void fetchPartition( + const ASTPtr & partition, + const StorageMetadataPtr & metadata_snapshot, + const String & from, + bool fetch_part, + ContextPtr query_context); void writePartLog( PartLogElement::Type type, diff --git a/src/Storages/PartitionCommands.cpp b/src/Storages/PartitionCommands.cpp index e51a64d5d81..f09f60887e8 100644 --- a/src/Storages/PartitionCommands.cpp +++ b/src/Storages/PartitionCommands.cpp @@ -82,6 +82,7 @@ std::optional PartitionCommand::parse(const ASTAlterCommand * res.type = FETCH_PARTITION; res.partition = command_ast->partition; res.from_zookeeper_path = command_ast->from; + res.part = command_ast->part; return res; } else if (command_ast->type == ASTAlterCommand::FREEZE_PARTITION) @@ -140,7 +141,10 @@ std::string PartitionCommand::typeToString() const else return "DROP DETACHED PARTITION"; case PartitionCommand::Type::FETCH_PARTITION: - return "FETCH PARTITION"; + if (part) + return "FETCH PART"; + else + return "FETCH PARTITION"; case PartitionCommand::Type::FREEZE_ALL_PARTITIONS: return "FREEZE ALL"; case PartitionCommand::Type::FREEZE_PARTITION: diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index a2b1f1737a2..2d836e00a08 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -112,9 +112,11 @@ namespace ErrorCodes extern const int NOT_A_LEADER; extern const int TABLE_WAS_NOT_DROPPED; extern const int PARTITION_ALREADY_EXISTS; + extern const int PART_ALREADY_EXISTS; extern const int TOO_MANY_RETRIES_TO_FETCH_PARTS; extern const int RECEIVED_ERROR_FROM_REMOTE_IO_SERVER; extern const int PARTITION_DOESNT_EXIST; + extern const int PART_DOESNT_EXIST; extern const int UNFINISHED; extern const int RECEIVED_ERROR_TOO_MANY_REQUESTS; extern const int TOO_MANY_FETCHES; @@ -5356,11 +5358,11 @@ void StorageReplicatedMergeTree::getReplicaDelays(time_t & out_absolute_delay, t } } - void StorageReplicatedMergeTree::fetchPartition( const ASTPtr & partition, const StorageMetadataPtr & metadata_snapshot, const String & from_, + bool fetch_part, ContextPtr query_context) { Macros::MacroExpansionInfo info; @@ -5373,40 +5375,54 @@ void StorageReplicatedMergeTree::fetchPartition( if (from.empty()) throw Exception("ZooKeeper path should not be empty", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - String partition_id = getPartitionIDFromQuery(partition, query_context); zkutil::ZooKeeperPtr zookeeper; if (auxiliary_zookeeper_name != default_zookeeper_name) - { zookeeper = getContext()->getAuxiliaryZooKeeper(auxiliary_zookeeper_name); - - LOG_INFO(log, "Will fetch partition {} from shard {} (auxiliary zookeeper '{}')", partition_id, from_, auxiliary_zookeeper_name); - } else - { zookeeper = getZooKeeper(); - LOG_INFO(log, "Will fetch partition {} from shard {}", partition_id, from_); - } - if (from.back() == '/') from.resize(from.size() - 1); + if (fetch_part) + { + String part_name = partition->as().value.safeGet(); + auto replica_path_ = findReplicaHavingPart(part_name, from, zookeeper); + + if (replica_path_.empty()) + throw Exception("fetch part " + part_name + " not exists !", ErrorCodes::PART_DOESNT_EXIST); + /** Let's check that there is no such part in the `detached` directory (where we will write the downloaded parts). + * Unreliable (there is a race condition) - such a part may appear a little later. + */ + if (checkDetachPartIfExists(part_name)) + throw Exception("Detached part " + part_name + " already exists.", ErrorCodes::PART_ALREADY_EXISTS); + LOG_INFO(log, "Will fetch part {} from shard {} (zookeeper '{}')", part_name, from_, auxiliary_zookeeper_name); + + try + { + /// part name , metadata, replica path , true, 0, zookeeper + if (!fetchPart(part_name, metadata_snapshot, replica_path_, true, 0, zookeeper)) + throw Exception("fetch part " + part_name + " failed! ", ErrorCodes::UNFINISHED); + } + catch (const DB::Exception & e) + { + if (e.code() != ErrorCodes::RECEIVED_ERROR_FROM_REMOTE_IO_SERVER && e.code() != ErrorCodes::RECEIVED_ERROR_TOO_MANY_REQUESTS + && e.code() != ErrorCodes::CANNOT_READ_ALL_DATA) + throw; + + LOG_INFO(log, e.displayText()); + } + return; + } + + String partition_id = getPartitionIDFromQuery(partition, query_context); + LOG_INFO(log, "Will fetch partition {} from shard {} (zookeeper '{}')", partition_id, from_, auxiliary_zookeeper_name); /** Let's check that there is no such partition in the `detached` directory (where we will write the downloaded parts). * Unreliable (there is a race condition) - such a partition may appear a little later. */ - Poco::DirectoryIterator dir_end; - for (const std::string & path : getDataPaths()) - { - for (Poco::DirectoryIterator dir_it{path + "detached/"}; dir_it != dir_end; ++dir_it) - { - MergeTreePartInfo part_info; - if (MergeTreePartInfo::tryParsePartName(dir_it.name(), &part_info, format_version) - && part_info.partition_id == partition_id) - throw Exception("Detached partition " + partition_id + " already exists.", ErrorCodes::PARTITION_ALREADY_EXISTS); - } - - } + if (checkDetachPartitionIfExists(partition_id)) + throw Exception("Detached partition " + partition_id + " already exists.", ErrorCodes::PARTITION_ALREADY_EXISTS); zkutil::Strings replicas; zkutil::Strings active_replicas; @@ -6913,4 +6929,46 @@ String StorageReplicatedMergeTree::getSharedDataReplica( return best_replica; } +String StorageReplicatedMergeTree::findReplicaHavingPart( + const String & part_name, const String & zookeeper_path_, zkutil::ZooKeeper::Ptr zookeeper_) +{ + Strings replicas = zookeeper_->getChildren(zookeeper_path_ + "/replicas"); + + /// Select replicas in uniformly random order. + std::shuffle(replicas.begin(), replicas.end(), thread_local_rng); + + for (const String & replica : replicas) + { + if (zookeeper_->exists(zookeeper_path_ + "/replicas/" + replica + "/parts/" + part_name) + && zookeeper_->exists(zookeeper_path_ + "/replicas/" + replica + "/is_active")) + return zookeeper_path_ + "/replicas/" + replica; + } + + return {}; +} + +bool StorageReplicatedMergeTree::checkDetachPartIfExists(const String & part_name) +{ + Poco::DirectoryIterator dir_end; + for (const std::string & path : getDataPaths()) + for (Poco::DirectoryIterator dir_it{path + "detached/"}; dir_it != dir_end; ++dir_it) + if (dir_it.name() == part_name) + return true; + return false; +} + +bool StorageReplicatedMergeTree::checkDetachPartitionIfExists(const String & partition_name) +{ + Poco::DirectoryIterator dir_end; + for (const std::string & path : getDataPaths()) + { + for (Poco::DirectoryIterator dir_it{path + "detached/"}; dir_it != dir_end; ++dir_it) + { + MergeTreePartInfo part_info; + if (MergeTreePartInfo::tryParsePartName(dir_it.name(), &part_info, format_version) && part_info.partition_id == partition_name) + return true; + } + } + return false; +} } diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 0180fa8bc1a..673c713fabc 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -522,8 +522,11 @@ private: /** Returns an empty string if no one has a part. */ String findReplicaHavingPart(const String & part_name, bool active); + String findReplicaHavingPart(const String & part_name, const String & zookeeper_path_, zkutil::ZooKeeper::Ptr zookeeper_); bool checkReplicaHavePart(const String & replica, const String & part_name); + bool checkDetachPartIfExists(const String & part_name); + bool checkDetachPartitionIfExists(const String & partition_name); /** Find replica having specified part or any part that covers it. * If active = true, consider only active replicas. @@ -626,7 +629,12 @@ private: PartitionCommandsResultInfo attachPartition(const ASTPtr & partition, const StorageMetadataPtr & metadata_snapshot, bool part, ContextPtr query_context) override; void replacePartitionFrom(const StoragePtr & source_table, const ASTPtr & partition, bool replace, ContextPtr query_context) override; void movePartitionToTable(const StoragePtr & dest_table, const ASTPtr & partition, ContextPtr query_context) override; - void fetchPartition(const ASTPtr & partition, const StorageMetadataPtr & metadata_snapshot, const String & from, ContextPtr query_context) override; + void fetchPartition( + const ASTPtr & partition, + const StorageMetadataPtr & metadata_snapshot, + const String & from, + bool fetch_part, + ContextPtr query_context) override; /// Check granularity of already existing replicated table in zookeeper if it exists /// return true if it's fixed diff --git a/tests/fuzz/ast.dict b/tests/fuzz/ast.dict index 8327f276b31..7befb36c840 100644 --- a/tests/fuzz/ast.dict +++ b/tests/fuzz/ast.dict @@ -156,6 +156,7 @@ "extractURLParameterNames" "extractURLParameters" "FETCH PARTITION" +"FETCH PART" "FINAL" "FIRST" "firstSignificantSubdomain" diff --git a/tests/integration/test_fetch_part_from_auxiliary_zookeeper/__init__.py b/tests/integration/test_fetch_part_from_auxiliary_zookeeper/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_fetch_part_from_auxiliary_zookeeper/configs/zookeeper_config.xml b/tests/integration/test_fetch_part_from_auxiliary_zookeeper/configs/zookeeper_config.xml new file mode 100644 index 00000000000..b2b0667ebbf --- /dev/null +++ b/tests/integration/test_fetch_part_from_auxiliary_zookeeper/configs/zookeeper_config.xml @@ -0,0 +1,28 @@ + + + + zoo1 + 2181 + + + zoo2 + 2181 + + + zoo3 + 2181 + + + + + + zoo1 + 2181 + + + zoo2 + 2181 + + + + diff --git a/tests/integration/test_fetch_part_from_auxiliary_zookeeper/test.py b/tests/integration/test_fetch_part_from_auxiliary_zookeeper/test.py new file mode 100644 index 00000000000..3d13da49a63 --- /dev/null +++ b/tests/integration/test_fetch_part_from_auxiliary_zookeeper/test.py @@ -0,0 +1,40 @@ + + +import pytest +from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance("node", main_configs=["configs/zookeeper_config.xml"], with_zookeeper=True) + + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + + yield cluster + finally: + cluster.shutdown() + + +def test_fetch_part_from_allowed_zookeeper(start_cluster): + node.query( + "CREATE TABLE simple (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', 'node') ORDER BY tuple() PARTITION BY date;" + ) + node.query("INSERT INTO simple VALUES ('2020-08-27', 1)") + + node.query( + "CREATE TABLE simple2 (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/1/simple', 'node') ORDER BY tuple() PARTITION BY date;" + ) + node.query( + "ALTER TABLE simple2 FETCH PART '20200827_1_1_0' FROM 'zookeeper2:/clickhouse/tables/0/simple';" + ) + node.query("ALTER TABLE simple2 ATTACH PART '20200827_1_1_0';") + + with pytest.raises(QueryRuntimeException): + node.query( + "ALTER TABLE simple2 FETCH PART '20200827_1_1_0' FROM 'zookeeper:/clickhouse/tables/0/simple';" + ) + + assert node.query("SELECT id FROM simple2").strip() == "1" diff --git a/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py b/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py index f9c10d68fe3..0c94dfd3c48 100644 --- a/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py +++ b/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py @@ -18,7 +18,7 @@ def start_cluster(): cluster.shutdown() -def test_fetch_part_from_allowed_zookeeper(start_cluster): +def test_fetch_partition_from_allowed_zookeeper(start_cluster): node.query( "CREATE TABLE simple (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', 'node') ORDER BY tuple() PARTITION BY date;" ) diff --git a/tests/testflows/rbac/tests/privileges/grant_option.py b/tests/testflows/rbac/tests/privileges/grant_option.py index f337aec2619..a28350a78b9 100644 --- a/tests/testflows/rbac/tests/privileges/grant_option.py +++ b/tests/testflows/rbac/tests/privileges/grant_option.py @@ -89,7 +89,7 @@ def grant_option_check(grant_option_target, grant_target, user_name, table_type, @Examples("privilege", [ ("ALTER MOVE PARTITION",), ("ALTER MOVE PART",), ("MOVE PARTITION",), ("MOVE PART",), ("ALTER DELETE",), ("DELETE",), - ("ALTER FETCH PARTITION",), ("FETCH PARTITION",), + ("ALTER FETCH PARTITION",), ("ALTER FETCH PART",), ("FETCH PARTITION",), ("FETCH PART",), ("ALTER FREEZE PARTITION",), ("FREEZE PARTITION",), ("ALTER UPDATE",), ("UPDATE",), ("ALTER ADD COLUMN",), ("ADD COLUMN",), From fbd26789afe5b2aac3c9a808a24049dcd465755f Mon Sep 17 00:00:00 2001 From: songenjie Date: Tue, 13 Apr 2021 14:08:26 +0800 Subject: [PATCH 31/76] [test] show privileges --- tests/queries/0_stateless/01271_show_privileges.reference | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01271_show_privileges.reference b/tests/queries/0_stateless/01271_show_privileges.reference index 7928f531a7d..e2784f1dfd5 100644 --- a/tests/queries/0_stateless/01271_show_privileges.reference +++ b/tests/queries/0_stateless/01271_show_privileges.reference @@ -28,7 +28,7 @@ ALTER TTL ['ALTER MODIFY TTL','MODIFY TTL'] TABLE ALTER TABLE ALTER MATERIALIZE TTL ['MATERIALIZE TTL'] TABLE ALTER TABLE ALTER SETTINGS ['ALTER SETTING','ALTER MODIFY SETTING','MODIFY SETTING'] TABLE ALTER TABLE ALTER MOVE PARTITION ['ALTER MOVE PART','MOVE PARTITION','MOVE PART'] TABLE ALTER TABLE -ALTER FETCH PARTITION ['FETCH PARTITION'] TABLE ALTER TABLE +ALTER FETCH PARTITION ['ALTER FETCH PART','FETCH PARTITION','FETCH PART'] TABLE ALTER TABLE ALTER FREEZE PARTITION ['FREEZE PARTITION','UNFREEZE'] TABLE ALTER TABLE ALTER TABLE [] \N ALTER ALTER VIEW REFRESH ['ALTER LIVE VIEW REFRESH','REFRESH VIEW'] VIEW ALTER VIEW From 520f4f39eccfcc16491fbc58d9cb44856deb5f53 Mon Sep 17 00:00:00 2001 From: songenjie Date: Tue, 13 Apr 2021 15:17:03 +0800 Subject: [PATCH 32/76] [test][integration] fetch part --- .../test_fetch_part_from_auxiliary_zookeeper/test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_fetch_part_from_auxiliary_zookeeper/test.py b/tests/integration/test_fetch_part_from_auxiliary_zookeeper/test.py index 3d13da49a63..17617f1c45c 100644 --- a/tests/integration/test_fetch_part_from_auxiliary_zookeeper/test.py +++ b/tests/integration/test_fetch_part_from_auxiliary_zookeeper/test.py @@ -28,13 +28,13 @@ def test_fetch_part_from_allowed_zookeeper(start_cluster): "CREATE TABLE simple2 (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/1/simple', 'node') ORDER BY tuple() PARTITION BY date;" ) node.query( - "ALTER TABLE simple2 FETCH PART '20200827_1_1_0' FROM 'zookeeper2:/clickhouse/tables/0/simple';" + "ALTER TABLE simple2 FETCH PART '20200827_0_0_0' FROM 'zookeeper2:/clickhouse/tables/0/simple';" ) - node.query("ALTER TABLE simple2 ATTACH PART '20200827_1_1_0';") + node.query("ALTER TABLE simple2 ATTACH PART '20200827_0_0_0';") with pytest.raises(QueryRuntimeException): node.query( - "ALTER TABLE simple2 FETCH PART '20200827_1_1_0' FROM 'zookeeper:/clickhouse/tables/0/simple';" + "ALTER TABLE simple2 FETCH PART '20200827_0_0_0' FROM 'zookeeper:/clickhouse/tables/0/simple';" ) assert node.query("SELECT id FROM simple2").strip() == "1" From 7c32cc1e18e364553ed31a76ca24c280df45e025 Mon Sep 17 00:00:00 2001 From: songenjie Date: Tue, 13 Apr 2021 17:34:04 +0800 Subject: [PATCH 33/76] fix case style for local variable --- src/Storages/StorageReplicatedMergeTree.cpp | 8 ++++---- src/Storages/StorageReplicatedMergeTree.h | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 2d836e00a08..98f1ee5560e 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -5387,9 +5387,9 @@ void StorageReplicatedMergeTree::fetchPartition( if (fetch_part) { String part_name = partition->as().value.safeGet(); - auto replica_path_ = findReplicaHavingPart(part_name, from, zookeeper); + auto part_path = findReplicaHavingPart(part_name, from, zookeeper); - if (replica_path_.empty()) + if (part_path.empty()) throw Exception("fetch part " + part_name + " not exists !", ErrorCodes::PART_DOESNT_EXIST); /** Let's check that there is no such part in the `detached` directory (where we will write the downloaded parts). * Unreliable (there is a race condition) - such a part may appear a little later. @@ -5400,8 +5400,8 @@ void StorageReplicatedMergeTree::fetchPartition( try { - /// part name , metadata, replica path , true, 0, zookeeper - if (!fetchPart(part_name, metadata_snapshot, replica_path_, true, 0, zookeeper)) + /// part name , metadata, part_path , true, 0, zookeeper + if (!fetchPart(part_name, metadata_snapshot, part_path, true, 0, zookeeper)) throw Exception("fetch part " + part_name + " failed! ", ErrorCodes::UNFINISHED); } catch (const DB::Exception & e) diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 673c713fabc..bc59a1bdf5f 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -522,7 +522,7 @@ private: /** Returns an empty string if no one has a part. */ String findReplicaHavingPart(const String & part_name, bool active); - String findReplicaHavingPart(const String & part_name, const String & zookeeper_path_, zkutil::ZooKeeper::Ptr zookeeper_); + static String findReplicaHavingPart(const String & part_name, const String & zookeeper_path_, zkutil::ZooKeeper::Ptr zookeeper_); bool checkReplicaHavePart(const String & replica, const String & part_name); bool checkDetachPartIfExists(const String & part_name); From 8abaf01a5d3ec5f41a29e3cf4915a34efc219b9a Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 13 Apr 2021 15:57:11 +0300 Subject: [PATCH 34/76] Updated dictionaries tests --- tests/performance/direct_dictionary.xml | 24 ++++++++++++++---------- tests/performance/flat_dictionary.xml | 15 +++++++-------- tests/performance/hashed_dictionary.xml | 20 ++++++++++++++------ 3 files changed, 35 insertions(+), 24 deletions(-) diff --git a/tests/performance/direct_dictionary.xml b/tests/performance/direct_dictionary.xml index e827ea0a76f..3f01449ed99 100644 --- a/tests/performance/direct_dictionary.xml +++ b/tests/performance/direct_dictionary.xml @@ -55,14 +55,14 @@ INSERT INTO simple_key_direct_dictionary_source_table SELECT number, number, toString(number), toDecimal64(number, 8), toString(number) FROM system.numbers - LIMIT 100000; + LIMIT 50000; INSERT INTO complex_key_direct_dictionary_source_table SELECT number, toString(number), number, toString(number), toDecimal64(number, 8), toString(number) FROM system.numbers - LIMIT 100000; + LIMIT 50000; @@ -79,47 +79,51 @@ elements_count - 25000 50000 75000 - 100000 - SELECT dictGet('default.simple_key_direct_dictionary', {column_name}, number) + WITH rand64() % toUInt64({elements_count}) as key + SELECT dictGet('default.simple_key_direct_dictionary', {column_name}, key) FROM system.numbers LIMIT {elements_count} FORMAT Null; - SELECT dictGet('default.simple_key_direct_dictionary', ('value_int', 'value_string', 'value_decimal', 'value_string_nullable'), number) + WITH rand64() % toUInt64({elements_count}) as key + SELECT dictGet('default.simple_key_direct_dictionary', ('value_int', 'value_string', 'value_decimal', 'value_string_nullable'), key) FROM system.numbers LIMIT {elements_count} FORMAT Null; - SELECT dictHas('default.simple_key_direct_dictionary', number) + WITH rand64() % toUInt64({elements_count}) as key + SELECT dictHas('default.simple_key_direct_dictionary', key) FROM system.numbers LIMIT {elements_count} FORMAT Null; - SELECT dictGet('default.complex_key_direct_dictionary', {column_name}, (number, toString(number))) + WITH (number, toString(number)) as key + SELECT dictGet('default.complex_key_direct_dictionary', {column_name}, key) FROM system.numbers LIMIT {elements_count} FORMAT Null; - SELECT dictGet('default.complex_key_direct_dictionary', ('value_int', 'value_string', 'value_decimal', 'value_string_nullable'), (number, toString(number))) + WITH (number, toString(number)) as key + SELECT dictGet('default.complex_key_direct_dictionary', ('value_int', 'value_string', 'value_decimal', 'value_string_nullable'), key) FROM system.numbers LIMIT {elements_count} FORMAT Null; - SELECT dictHas('default.complex_key_direct_dictionary', (number, toString(number))) + WITH (number, toString(number)) as key + SELECT dictHas('default.complex_key_direct_dictionary', key) FROM system.numbers LIMIT {elements_count} FORMAT Null; diff --git a/tests/performance/flat_dictionary.xml b/tests/performance/flat_dictionary.xml index 8111084586a..92ed975a671 100644 --- a/tests/performance/flat_dictionary.xml +++ b/tests/performance/flat_dictionary.xml @@ -1,8 +1,4 @@ - - please_fix_me - - CREATE TABLE simple_key_flat_dictionary_source_table ( @@ -52,22 +48,25 @@ 5000000 7500000 - 10000000 - SELECT dictGet('default.simple_key_flat_dictionary', {column_name}, rand64() % toUInt64(10000000)) + SELECT dictGet('default.simple_key_flat_dictionary', {column_name}, rand64() % toUInt64({elements_count})) FROM system.numbers LIMIT {elements_count} FORMAT Null; - SELECT dictHas('default.simple_key_flat_dictionary', rand64() % toUInt64(10000000)) + SELECT * FROM simple_key_flat_dictionary FORMAT Null; + + + + SELECT dictHas('default.simple_key_flat_dictionary', rand64() % toUInt64(75000000)) FROM system.numbers - LIMIT {elements_count} + LIMIT 75000000 FORMAT Null; diff --git a/tests/performance/hashed_dictionary.xml b/tests/performance/hashed_dictionary.xml index a38d2f30c23..b83018c67df 100644 --- a/tests/performance/hashed_dictionary.xml +++ b/tests/performance/hashed_dictionary.xml @@ -81,35 +81,43 @@ elements_count - 2500000 5000000 7500000 - 10000000 - SELECT dictGet('default.simple_key_hashed_dictionary', {column_name}, number) + WITH rand64() % toUInt64({elements_count}) as key + SELECT dictGet('default.simple_key_hashed_dictionary', {column_name}, key) FROM system.numbers LIMIT {elements_count} FORMAT Null; - SELECT dictHas('default.simple_key_hashed_dictionary', number) + SELECT * FROM default.simple_key_hashed_dictionary; + + + WITH rand64() % toUInt64({elements_count}) as key + SELECT dictHas('default.simple_key_hashed_dictionary', key) FROM system.numbers LIMIT {elements_count} FORMAT Null; - SELECT dictGet('default.complex_key_hashed_dictionary', {column_name}, (number, toString(number))) + WITH (rand64() % toUInt64({elements_count}), toString(rand64() % toUInt64({elements_count}))) as key + SELECT dictGet('default.complex_key_hashed_dictionary', {column_name}, key) FROM system.numbers LIMIT {elements_count} FORMAT Null; - SELECT dictHas('default.complex_key_hashed_dictionary', (number, toString(number))) + SELECT * FROM default.complex_key_hashed_dictionary; + + + WITH (rand64() % toUInt64({elements_count}), toString(rand64() % toUInt64({elements_count}))) as key + SELECT dictHas('default.complex_key_hashed_dictionary', key) FROM system.numbers LIMIT {elements_count} FORMAT Null; From da3d3e906a202b7cb4e718f0380805f07bb3cd06 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 13 Apr 2021 21:13:04 +0300 Subject: [PATCH 35/76] Updated tests --- tests/performance/flat_dictionary.xml | 3 ++- tests/performance/hashed_dictionary.xml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/performance/flat_dictionary.xml b/tests/performance/flat_dictionary.xml index 92ed975a671..56a94358eb9 100644 --- a/tests/performance/flat_dictionary.xml +++ b/tests/performance/flat_dictionary.xml @@ -60,7 +60,8 @@ - SELECT * FROM simple_key_flat_dictionary FORMAT Null; + SELECT * FROM simple_key_flat_dictionary + FORMAT Null; diff --git a/tests/performance/hashed_dictionary.xml b/tests/performance/hashed_dictionary.xml index b83018c67df..cd19ba035e5 100644 --- a/tests/performance/hashed_dictionary.xml +++ b/tests/performance/hashed_dictionary.xml @@ -113,7 +113,8 @@ FORMAT Null; - SELECT * FROM default.complex_key_hashed_dictionary; + SELECT * FROM default.complex_key_hashed_dictionary + FORMAT Null; WITH (rand64() % toUInt64({elements_count}), toString(rand64() % toUInt64({elements_count}))) as key From b00c66cb36ea611e750197f669c0cc702ba2d615 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 13 Apr 2021 21:53:55 +0300 Subject: [PATCH 36/76] More safe CPU dispatching --- src/Functions/CMakeLists.txt | 4 +- src/Functions/intDiv.cpp | 115 +---------------------------------- 2 files changed, 5 insertions(+), 114 deletions(-) diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index 1c3beb2e47d..7cbca175c0d 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -1,5 +1,7 @@ configure_file(config_functions.h.in ${ConfigIncludePath}/config_functions.h) +add_subdirectory(divide) + include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake) add_headers_and_sources(clickhouse_functions .) @@ -25,7 +27,7 @@ target_link_libraries(clickhouse_functions PRIVATE ${ZLIB_LIBRARIES} boost::filesystem - libdivide + divide_impl ) if (OPENSSL_CRYPTO_LIBRARY) diff --git a/src/Functions/intDiv.cpp b/src/Functions/intDiv.cpp index 42b0299ce01..98ce4fe30de 100644 --- a/src/Functions/intDiv.cpp +++ b/src/Functions/intDiv.cpp @@ -1,28 +1,7 @@ #include #include -#include -#if defined(__x86_64__) - #define LIBDIVIDE_SSE2 1 - #define LIBDIVIDE_AVX2 1 - - #if defined(__clang__) - #pragma clang attribute push(__attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2"))), apply_to=function) - #else - #pragma GCC push_options - #pragma GCC target("sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,tune=native") - #endif -#endif - -#include - -#if defined(__x86_64__) - #if defined(__clang__) - #pragma clang attribute pop - #else - #pragma GCC pop_options - #endif -#endif +#include "divide/divide.h" namespace DB @@ -37,83 +16,6 @@ namespace /// Optimizations for integer division by a constant. -#if defined(__x86_64__) - -DECLARE_DEFAULT_CODE ( - template - void divideImpl(const A * __restrict a_pos, B b, ResultType * __restrict c_pos, size_t size) - { - libdivide::divider divider(b); - const A * a_end = a_pos + size; - - static constexpr size_t values_per_simd_register = 16 / sizeof(A); - const A * a_end_simd = a_pos + size / values_per_simd_register * values_per_simd_register; - - while (a_pos < a_end_simd) - { - _mm_storeu_si128(reinterpret_cast<__m128i *>(c_pos), - _mm_loadu_si128(reinterpret_cast(a_pos)) / divider); - - a_pos += values_per_simd_register; - c_pos += values_per_simd_register; - } - - while (a_pos < a_end) - { - *c_pos = *a_pos / divider; - ++a_pos; - ++c_pos; - } - } -) - -DECLARE_AVX2_SPECIFIC_CODE ( - template - void divideImpl(const A * __restrict a_pos, B b, ResultType * __restrict c_pos, size_t size) - { - libdivide::divider divider(b); - const A * a_end = a_pos + size; - - static constexpr size_t values_per_simd_register = 32 / sizeof(A); - const A * a_end_simd = a_pos + size / values_per_simd_register * values_per_simd_register; - - while (a_pos < a_end_simd) - { - _mm256_storeu_si256(reinterpret_cast<__m256i *>(c_pos), - _mm256_loadu_si256(reinterpret_cast(a_pos)) / divider); - - a_pos += values_per_simd_register; - c_pos += values_per_simd_register; - } - - while (a_pos < a_end) - { - *c_pos = *a_pos / divider; - ++a_pos; - ++c_pos; - } - } -) - -#else - -template -void divideImpl(const A * __restrict a_pos, B b, ResultType * __restrict c_pos, size_t size) -{ - libdivide::divider divider(b); - const A * a_end = a_pos + size; - - while (a_pos < a_end) - { - *c_pos = *a_pos / divider; - ++a_pos; - ++c_pos; - } -} - -#endif - - template struct DivideIntegralByConstantImpl : BinaryOperation> @@ -164,20 +66,7 @@ struct DivideIntegralByConstantImpl if (unlikely(static_cast(b) == 0)) throw Exception("Division by zero", ErrorCodes::ILLEGAL_DIVISION); -#if USE_MULTITARGET_CODE - if (isArchSupported(TargetArch::AVX2)) - { - TargetSpecific::AVX2::divideImpl(a_pos, b, c_pos, size); - } - else -#endif - { -#if __x86_64__ - TargetSpecific::Default::divideImpl(a_pos, b, c_pos, size); -#else - divideImpl(a_pos, b, c_pos, size); -#endif - } + divideImpl(a_pos, b, c_pos, size); } }; From fb98915435e5ce9981958b5ab874320f0f15c989 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 13 Apr 2021 21:54:46 +0300 Subject: [PATCH 37/76] More safe CPU dispatching --- src/Functions/divide/CMakeLists.txt | 10 +++ src/Functions/divide/divide.cpp | 66 ++++++++++++++++++++ src/Functions/divide/divide.h | 4 ++ src/Functions/divide/divideImpl.cpp | 95 +++++++++++++++++++++++++++++ 4 files changed, 175 insertions(+) create mode 100644 src/Functions/divide/CMakeLists.txt create mode 100644 src/Functions/divide/divide.cpp create mode 100644 src/Functions/divide/divide.h create mode 100644 src/Functions/divide/divideImpl.cpp diff --git a/src/Functions/divide/CMakeLists.txt b/src/Functions/divide/CMakeLists.txt new file mode 100644 index 00000000000..2bdd7e4c5ef --- /dev/null +++ b/src/Functions/divide/CMakeLists.txt @@ -0,0 +1,10 @@ +add_library(divide_impl_sse2 divideImpl.cpp) +target_compile_options(divide_impl_sse2 PRIVATE -msse2 -DNAMESPACE=SSE2) +target_link_libraries(divide_impl_sse2 libdivide) + +add_library(divide_impl_avx2 divideImpl.cpp) +target_compile_options(divide_impl_avx2 PRIVATE -mavx2 -DNAMESPACE=AVX2) +target_link_libraries(divide_impl_avx2 libdivide) + +add_library(divide_impl divide.cpp) +target_link_libraries(divide_impl divide_impl_sse2 divide_impl_avx2 clickhouse_common_io) diff --git a/src/Functions/divide/divide.cpp b/src/Functions/divide/divide.cpp new file mode 100644 index 00000000000..0c275dff6f6 --- /dev/null +++ b/src/Functions/divide/divide.cpp @@ -0,0 +1,66 @@ +#include "divide.h" +#include + + +namespace SSE2 +{ + template + void divideImpl(const A * __restrict a_pos, B b, ResultType * __restrict c_pos, size_t size); +} + +namespace AVX2 +{ + template + void divideImpl(const A * __restrict a_pos, B b, ResultType * __restrict c_pos, size_t size); +} + + +template +void divideImpl(const A * __restrict a_pos, B b, ResultType * __restrict c_pos, size_t size) +{ + if (DB::Cpu::CpuFlagsCache::have_AVX2) + AVX2::divideImpl(a_pos, b, c_pos, size); + else if (DB::Cpu::CpuFlagsCache::have_SSE2) + SSE2::divideImpl(a_pos, b, c_pos, size); +} + + +template void divideImpl(const uint64_t * __restrict, uint64_t, uint64_t * __restrict, size_t); +template void divideImpl(const uint64_t * __restrict, uint32_t, uint64_t * __restrict, size_t); +template void divideImpl(const uint64_t * __restrict, uint16_t, uint64_t * __restrict, size_t); +template void divideImpl(const uint64_t * __restrict, char8_t, uint64_t * __restrict, size_t); + +template void divideImpl(const uint32_t * __restrict, uint64_t, uint32_t * __restrict, size_t); +template void divideImpl(const uint32_t * __restrict, uint32_t, uint32_t * __restrict, size_t); +template void divideImpl(const uint32_t * __restrict, uint16_t, uint32_t * __restrict, size_t); +template void divideImpl(const uint32_t * __restrict, char8_t, uint32_t * __restrict, size_t); + +template void divideImpl(const int64_t * __restrict, uint64_t, int64_t * __restrict, size_t); +template void divideImpl(const int64_t * __restrict, uint32_t, int64_t * __restrict, size_t); +template void divideImpl(const int64_t * __restrict, uint16_t, int64_t * __restrict, size_t); +template void divideImpl(const int64_t * __restrict, char8_t, int64_t * __restrict, size_t); + +template void divideImpl(const int32_t * __restrict, uint64_t, int32_t * __restrict, size_t); +template void divideImpl(const int32_t * __restrict, uint32_t, int32_t * __restrict, size_t); +template void divideImpl(const int32_t * __restrict, uint16_t, int32_t * __restrict, size_t); +template void divideImpl(const int32_t * __restrict, char8_t, int32_t * __restrict, size_t); + +template void divideImpl(const uint64_t * __restrict, int64_t, uint64_t * __restrict, size_t); +template void divideImpl(const uint64_t * __restrict, int32_t, uint64_t * __restrict, size_t); +template void divideImpl(const uint64_t * __restrict, int16_t, uint64_t * __restrict, size_t); +template void divideImpl(const uint64_t * __restrict, int8_t, uint64_t * __restrict, size_t); + +template void divideImpl(const uint32_t * __restrict, int64_t, uint32_t * __restrict, size_t); +template void divideImpl(const uint32_t * __restrict, int32_t, uint32_t * __restrict, size_t); +template void divideImpl(const uint32_t * __restrict, int16_t, uint32_t * __restrict, size_t); +template void divideImpl(const uint32_t * __restrict, int8_t, uint32_t * __restrict, size_t); + +template void divideImpl(const int64_t * __restrict, int64_t, int64_t * __restrict, size_t); +template void divideImpl(const int64_t * __restrict, int32_t, int64_t * __restrict, size_t); +template void divideImpl(const int64_t * __restrict, int16_t, int64_t * __restrict, size_t); +template void divideImpl(const int64_t * __restrict, int8_t, int64_t * __restrict, size_t); + +template void divideImpl(const int32_t * __restrict, int64_t, int32_t * __restrict, size_t); +template void divideImpl(const int32_t * __restrict, int32_t, int32_t * __restrict, size_t); +template void divideImpl(const int32_t * __restrict, int16_t, int32_t * __restrict, size_t); +template void divideImpl(const int32_t * __restrict, int8_t, int32_t * __restrict, size_t); diff --git a/src/Functions/divide/divide.h b/src/Functions/divide/divide.h new file mode 100644 index 00000000000..11a5371bc31 --- /dev/null +++ b/src/Functions/divide/divide.h @@ -0,0 +1,4 @@ +#include + +template +void divideImpl(const A * __restrict a_pos, B b, ResultType * __restrict c_pos, size_t size); diff --git a/src/Functions/divide/divideImpl.cpp b/src/Functions/divide/divideImpl.cpp new file mode 100644 index 00000000000..a5c1755ab1f --- /dev/null +++ b/src/Functions/divide/divideImpl.cpp @@ -0,0 +1,95 @@ +/// This translation unit should be compiled multiple times +/// with different values of NAMESPACE and machine flags (sse2, avx2). + +#if !defined(NAMESPACE) +#error "NAMESPACE macro must be defined" +#endif + +#if defined(__AVX2__) + #define REG_SIZE 32 + #define LIBDIVIDE_AVX2 +#elif defined(__SSE2__) + #define REG_SIZE 16 + #define LIBDIVIDE_SSE2 +#endif + +#include + + +namespace NAMESPACE +{ + +template +void divideImpl(const A * __restrict a_pos, B b, ResultType * __restrict c_pos, size_t size) +{ + libdivide::divider divider(b); + const A * a_end = a_pos + size; + +#if defined(__SSE2__) + static constexpr size_t values_per_simd_register = REG_SIZE / sizeof(A); + const A * a_end_simd = a_pos + size / values_per_simd_register * values_per_simd_register; + + while (a_pos < a_end_simd) + { +#if defined(__AVX2__) + _mm256_storeu_si256(reinterpret_cast<__m256i *>(c_pos), + _mm256_loadu_si256(reinterpret_cast(a_pos)) / divider); +#else + _mm_storeu_si128(reinterpret_cast<__m128i *>(c_pos), + _mm_loadu_si128(reinterpret_cast(a_pos)) / divider); +#endif + + a_pos += values_per_simd_register; + c_pos += values_per_simd_register; + } +#endif + + while (a_pos < a_end) + { + *c_pos = *a_pos / divider; + ++a_pos; + ++c_pos; + } +} + +template void divideImpl(const uint64_t * __restrict, uint64_t, uint64_t * __restrict, size_t); +template void divideImpl(const uint64_t * __restrict, uint32_t, uint64_t * __restrict, size_t); +template void divideImpl(const uint64_t * __restrict, uint16_t, uint64_t * __restrict, size_t); +template void divideImpl(const uint64_t * __restrict, char8_t, uint64_t * __restrict, size_t); + +template void divideImpl(const uint32_t * __restrict, uint64_t, uint32_t * __restrict, size_t); +template void divideImpl(const uint32_t * __restrict, uint32_t, uint32_t * __restrict, size_t); +template void divideImpl(const uint32_t * __restrict, uint16_t, uint32_t * __restrict, size_t); +template void divideImpl(const uint32_t * __restrict, char8_t, uint32_t * __restrict, size_t); + +template void divideImpl(const int64_t * __restrict, uint64_t, int64_t * __restrict, size_t); +template void divideImpl(const int64_t * __restrict, uint32_t, int64_t * __restrict, size_t); +template void divideImpl(const int64_t * __restrict, uint16_t, int64_t * __restrict, size_t); +template void divideImpl(const int64_t * __restrict, char8_t, int64_t * __restrict, size_t); + +template void divideImpl(const int32_t * __restrict, uint64_t, int32_t * __restrict, size_t); +template void divideImpl(const int32_t * __restrict, uint32_t, int32_t * __restrict, size_t); +template void divideImpl(const int32_t * __restrict, uint16_t, int32_t * __restrict, size_t); +template void divideImpl(const int32_t * __restrict, char8_t, int32_t * __restrict, size_t); + +template void divideImpl(const uint64_t * __restrict, int64_t, uint64_t * __restrict, size_t); +template void divideImpl(const uint64_t * __restrict, int32_t, uint64_t * __restrict, size_t); +template void divideImpl(const uint64_t * __restrict, int16_t, uint64_t * __restrict, size_t); +template void divideImpl(const uint64_t * __restrict, int8_t, uint64_t * __restrict, size_t); + +template void divideImpl(const uint32_t * __restrict, int64_t, uint32_t * __restrict, size_t); +template void divideImpl(const uint32_t * __restrict, int32_t, uint32_t * __restrict, size_t); +template void divideImpl(const uint32_t * __restrict, int16_t, uint32_t * __restrict, size_t); +template void divideImpl(const uint32_t * __restrict, int8_t, uint32_t * __restrict, size_t); + +template void divideImpl(const int64_t * __restrict, int64_t, int64_t * __restrict, size_t); +template void divideImpl(const int64_t * __restrict, int32_t, int64_t * __restrict, size_t); +template void divideImpl(const int64_t * __restrict, int16_t, int64_t * __restrict, size_t); +template void divideImpl(const int64_t * __restrict, int8_t, int64_t * __restrict, size_t); + +template void divideImpl(const int32_t * __restrict, int64_t, int32_t * __restrict, size_t); +template void divideImpl(const int32_t * __restrict, int32_t, int32_t * __restrict, size_t); +template void divideImpl(const int32_t * __restrict, int16_t, int32_t * __restrict, size_t); +template void divideImpl(const int32_t * __restrict, int8_t, int32_t * __restrict, size_t); + +} From 42412f9a085834f732772a9cc706bd7265c2b46a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 13 Apr 2021 21:59:55 +0300 Subject: [PATCH 38/76] extern template --- src/Functions/divide/divide.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/divide/divide.h b/src/Functions/divide/divide.h index 11a5371bc31..daf406038f2 100644 --- a/src/Functions/divide/divide.h +++ b/src/Functions/divide/divide.h @@ -1,4 +1,4 @@ #include template -void divideImpl(const A * __restrict a_pos, B b, ResultType * __restrict c_pos, size_t size); +extern void divideImpl(const A * __restrict a_pos, B b, ResultType * __restrict c_pos, size_t size); From 8bd5578c92d3bd1f293405a7a16844d3b8223603 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 13 Apr 2021 22:02:07 +0300 Subject: [PATCH 39/76] Less amount of template instantiations --- src/Functions/divide/divide.cpp | 20 -------------------- src/Functions/divide/divideImpl.cpp | 20 -------------------- 2 files changed, 40 deletions(-) diff --git a/src/Functions/divide/divide.cpp b/src/Functions/divide/divide.cpp index 0c275dff6f6..1c3c11af312 100644 --- a/src/Functions/divide/divide.cpp +++ b/src/Functions/divide/divide.cpp @@ -35,26 +35,6 @@ template void divideImpl(const uint32_t * __restri template void divideImpl(const uint32_t * __restrict, uint16_t, uint32_t * __restrict, size_t); template void divideImpl(const uint32_t * __restrict, char8_t, uint32_t * __restrict, size_t); -template void divideImpl(const int64_t * __restrict, uint64_t, int64_t * __restrict, size_t); -template void divideImpl(const int64_t * __restrict, uint32_t, int64_t * __restrict, size_t); -template void divideImpl(const int64_t * __restrict, uint16_t, int64_t * __restrict, size_t); -template void divideImpl(const int64_t * __restrict, char8_t, int64_t * __restrict, size_t); - -template void divideImpl(const int32_t * __restrict, uint64_t, int32_t * __restrict, size_t); -template void divideImpl(const int32_t * __restrict, uint32_t, int32_t * __restrict, size_t); -template void divideImpl(const int32_t * __restrict, uint16_t, int32_t * __restrict, size_t); -template void divideImpl(const int32_t * __restrict, char8_t, int32_t * __restrict, size_t); - -template void divideImpl(const uint64_t * __restrict, int64_t, uint64_t * __restrict, size_t); -template void divideImpl(const uint64_t * __restrict, int32_t, uint64_t * __restrict, size_t); -template void divideImpl(const uint64_t * __restrict, int16_t, uint64_t * __restrict, size_t); -template void divideImpl(const uint64_t * __restrict, int8_t, uint64_t * __restrict, size_t); - -template void divideImpl(const uint32_t * __restrict, int64_t, uint32_t * __restrict, size_t); -template void divideImpl(const uint32_t * __restrict, int32_t, uint32_t * __restrict, size_t); -template void divideImpl(const uint32_t * __restrict, int16_t, uint32_t * __restrict, size_t); -template void divideImpl(const uint32_t * __restrict, int8_t, uint32_t * __restrict, size_t); - template void divideImpl(const int64_t * __restrict, int64_t, int64_t * __restrict, size_t); template void divideImpl(const int64_t * __restrict, int32_t, int64_t * __restrict, size_t); template void divideImpl(const int64_t * __restrict, int16_t, int64_t * __restrict, size_t); diff --git a/src/Functions/divide/divideImpl.cpp b/src/Functions/divide/divideImpl.cpp index a5c1755ab1f..a62ce8126e2 100644 --- a/src/Functions/divide/divideImpl.cpp +++ b/src/Functions/divide/divideImpl.cpp @@ -62,26 +62,6 @@ template void divideImpl(const uint32_t * __restri template void divideImpl(const uint32_t * __restrict, uint16_t, uint32_t * __restrict, size_t); template void divideImpl(const uint32_t * __restrict, char8_t, uint32_t * __restrict, size_t); -template void divideImpl(const int64_t * __restrict, uint64_t, int64_t * __restrict, size_t); -template void divideImpl(const int64_t * __restrict, uint32_t, int64_t * __restrict, size_t); -template void divideImpl(const int64_t * __restrict, uint16_t, int64_t * __restrict, size_t); -template void divideImpl(const int64_t * __restrict, char8_t, int64_t * __restrict, size_t); - -template void divideImpl(const int32_t * __restrict, uint64_t, int32_t * __restrict, size_t); -template void divideImpl(const int32_t * __restrict, uint32_t, int32_t * __restrict, size_t); -template void divideImpl(const int32_t * __restrict, uint16_t, int32_t * __restrict, size_t); -template void divideImpl(const int32_t * __restrict, char8_t, int32_t * __restrict, size_t); - -template void divideImpl(const uint64_t * __restrict, int64_t, uint64_t * __restrict, size_t); -template void divideImpl(const uint64_t * __restrict, int32_t, uint64_t * __restrict, size_t); -template void divideImpl(const uint64_t * __restrict, int16_t, uint64_t * __restrict, size_t); -template void divideImpl(const uint64_t * __restrict, int8_t, uint64_t * __restrict, size_t); - -template void divideImpl(const uint32_t * __restrict, int64_t, uint32_t * __restrict, size_t); -template void divideImpl(const uint32_t * __restrict, int32_t, uint32_t * __restrict, size_t); -template void divideImpl(const uint32_t * __restrict, int16_t, uint32_t * __restrict, size_t); -template void divideImpl(const uint32_t * __restrict, int8_t, uint32_t * __restrict, size_t); - template void divideImpl(const int64_t * __restrict, int64_t, int64_t * __restrict, size_t); template void divideImpl(const int64_t * __restrict, int32_t, int64_t * __restrict, size_t); template void divideImpl(const int64_t * __restrict, int16_t, int64_t * __restrict, size_t); From 03662165f35fb6f45980d410e95239433189ffbe Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 13 Apr 2021 22:03:52 +0300 Subject: [PATCH 40/76] Comment --- src/Functions/intDiv.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/intDiv.cpp b/src/Functions/intDiv.cpp index 98ce4fe30de..79e35a19283 100644 --- a/src/Functions/intDiv.cpp +++ b/src/Functions/intDiv.cpp @@ -70,7 +70,7 @@ struct DivideIntegralByConstantImpl } }; -/** Specializations are specified for dividing numbers of the type UInt64 and UInt32 by the numbers of the same sign. +/** Specializations are specified for dividing numbers of the type UInt64, UInt32, Int64, Int32 by the numbers of the same sign. * Can be expanded to all possible combinations, but more code is needed. */ From f731739fab70c31bf02bc41af35db9d0a3115808 Mon Sep 17 00:00:00 2001 From: songenjie Date: Wed, 14 Apr 2021 10:05:41 +0800 Subject: [PATCH 41/76] fix suggests --- .../sql-reference/statements/alter/partition.md | 8 ++++---- src/Access/AccessType.h | 2 +- src/Common/ErrorCodes.cpp | 2 -- src/Storages/StorageReplicatedMergeTree.cpp | 15 +++++++-------- src/Storages/StorageReplicatedMergeTree.h | 4 ++-- .../0_stateless/01271_show_privileges.reference | 2 +- 6 files changed, 15 insertions(+), 18 deletions(-) diff --git a/docs/en/sql-reference/statements/alter/partition.md b/docs/en/sql-reference/statements/alter/partition.md index 23f10684ae7..948711e6d9e 100644 --- a/docs/en/sql-reference/statements/alter/partition.md +++ b/docs/en/sql-reference/statements/alter/partition.md @@ -16,7 +16,7 @@ The following operations with [partitions](../../../engines/table-engines/merget - [CLEAR COLUMN IN PARTITION](#alter_clear-column-partition) — Resets the value of a specified column in a partition. - [CLEAR INDEX IN PARTITION](#alter_clear-index-partition) — Resets the specified secondary index in a partition. - [FREEZE PARTITION](#alter_freeze-partition) — Creates a backup of a partition. -- [FETCH PART\|PARTITION](#alter_fetch-partition) — Downloads a part or partition from another server. +- [FETCH PARTITION\|PART](#alter_fetch-partition) — Downloads a part or partition from another server. - [MOVE PARTITION\|PART](#alter_move-partition) — Move partition/data part to another disk or volume. @@ -198,10 +198,10 @@ ALTER TABLE table_name CLEAR INDEX index_name IN PARTITION partition_expr The query works similar to `CLEAR COLUMN`, but it resets an index instead of a column data. -## FETCH PART|PARTITION {#alter_fetch-partition} +## FETCH PARTITION|PART {#alter_fetch-partition} ``` sql -ALTER TABLE table_name FETCH PART|PARTITION partition_expr FROM 'path-in-zookeeper' +ALTER TABLE table_name FETCH PARTITION|PART partition_expr FROM 'path-in-zookeeper' ``` Downloads a partition from another server. This query only works for the replicated tables. @@ -226,7 +226,7 @@ ALTER TABLE users ATTACH PART 201901_2_2_0; Note that: -- The `ALTER ... FETCH PART|PARTITION` query isn’t replicated. It places the part or partition to the `detached` directory only on the local server. +- The `ALTER ... FETCH PARTITION|PART` query isn’t replicated. It places the part or partition to the `detached` directory only on the local server. - The `ALTER TABLE ... ATTACH` query is replicated. It adds the data to all replicas. The data is added to one of the replicas from the `detached` directory, and to the others - from neighboring replicas. Before downloading, the system checks if the partition exists and the table structure matches. The most appropriate replica is selected automatically from the healthy replicas. diff --git a/src/Access/AccessType.h b/src/Access/AccessType.h index c7311997ba2..952cddba5f5 100644 --- a/src/Access/AccessType.h +++ b/src/Access/AccessType.h @@ -62,7 +62,7 @@ enum class AccessType enabled implicitly by the grant ALTER_TABLE */\ M(ALTER_SETTINGS, "ALTER SETTING, ALTER MODIFY SETTING, MODIFY SETTING", TABLE, ALTER_TABLE) /* allows to execute ALTER MODIFY SETTING */\ M(ALTER_MOVE_PARTITION, "ALTER MOVE PART, MOVE PARTITION, MOVE PART", TABLE, ALTER_TABLE) \ - M(ALTER_FETCH_PARTITION, "ALTER FETCH PART, FETCH PARTITION, FETCH PART", TABLE, ALTER_TABLE) \ + M(ALTER_FETCH_PARTITION, "ALTER FETCH PART, FETCH PARTITION", TABLE, ALTER_TABLE) \ M(ALTER_FREEZE_PARTITION, "FREEZE PARTITION, UNFREEZE", TABLE, ALTER_TABLE) \ \ M(ALTER_TABLE, "", GROUP, ALTER) \ diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 7658448976b..ad0463db889 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -549,8 +549,6 @@ M(579, INCORRECT_PART_TYPE) \ M(580, CANNOT_SET_ROUNDING_MODE) \ M(581, TOO_LARGE_DISTRIBUTED_DEPTH) \ - M(582, PART_DOESNT_EXIST) \ - M(583, PART_ALREADY_EXISTS) \ \ M(998, POSTGRESQL_CONNECTION_FAILURE) \ M(999, KEEPER_EXCEPTION) \ diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 98f1ee5560e..977ade9f1b8 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -112,11 +112,9 @@ namespace ErrorCodes extern const int NOT_A_LEADER; extern const int TABLE_WAS_NOT_DROPPED; extern const int PARTITION_ALREADY_EXISTS; - extern const int PART_ALREADY_EXISTS; extern const int TOO_MANY_RETRIES_TO_FETCH_PARTS; extern const int RECEIVED_ERROR_FROM_REMOTE_IO_SERVER; extern const int PARTITION_DOESNT_EXIST; - extern const int PART_DOESNT_EXIST; extern const int UNFINISHED; extern const int RECEIVED_ERROR_TOO_MANY_REQUESTS; extern const int TOO_MANY_FETCHES; @@ -132,6 +130,7 @@ namespace ErrorCodes extern const int UNKNOWN_POLICY; extern const int NO_SUCH_DATA_PART; extern const int INTERSERVER_SCHEME_DOESNT_MATCH; + extern const int DUPLICATED_PART_UUIDS; } namespace ActionLocks @@ -5390,12 +5389,12 @@ void StorageReplicatedMergeTree::fetchPartition( auto part_path = findReplicaHavingPart(part_name, from, zookeeper); if (part_path.empty()) - throw Exception("fetch part " + part_name + " not exists !", ErrorCodes::PART_DOESNT_EXIST); + throw Exception("fetch part " + part_name + " not exists !", ErrorCodes::NO_REPLICA_HAS_PART); /** Let's check that there is no such part in the `detached` directory (where we will write the downloaded parts). * Unreliable (there is a race condition) - such a part may appear a little later. */ - if (checkDetachPartIfExists(part_name)) - throw Exception("Detached part " + part_name + " already exists.", ErrorCodes::PART_ALREADY_EXISTS); + if (checkIfDetachedPartExists(part_name)) + throw Exception("Detached part " + part_name + " already exists.", ErrorCodes::DUPLICATED_PART_UUIDS); LOG_INFO(log, "Will fetch part {} from shard {} (zookeeper '{}')", part_name, from_, auxiliary_zookeeper_name); try @@ -5421,7 +5420,7 @@ void StorageReplicatedMergeTree::fetchPartition( /** Let's check that there is no such partition in the `detached` directory (where we will write the downloaded parts). * Unreliable (there is a race condition) - such a partition may appear a little later. */ - if (checkDetachPartitionIfExists(partition_id)) + if (checkIfDetachedPartitionExists(partition_id)) throw Exception("Detached partition " + partition_id + " already exists.", ErrorCodes::PARTITION_ALREADY_EXISTS); zkutil::Strings replicas; @@ -6947,7 +6946,7 @@ String StorageReplicatedMergeTree::findReplicaHavingPart( return {}; } -bool StorageReplicatedMergeTree::checkDetachPartIfExists(const String & part_name) +bool StorageReplicatedMergeTree::checkIfDetachedPartExists(const String & part_name) { Poco::DirectoryIterator dir_end; for (const std::string & path : getDataPaths()) @@ -6957,7 +6956,7 @@ bool StorageReplicatedMergeTree::checkDetachPartIfExists(const String & part_nam return false; } -bool StorageReplicatedMergeTree::checkDetachPartitionIfExists(const String & partition_name) +bool StorageReplicatedMergeTree::checkIfDetachedPartitionExists(const String & partition_name) { Poco::DirectoryIterator dir_end; for (const std::string & path : getDataPaths()) diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index bc59a1bdf5f..9122bdafbf0 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -525,8 +525,8 @@ private: static String findReplicaHavingPart(const String & part_name, const String & zookeeper_path_, zkutil::ZooKeeper::Ptr zookeeper_); bool checkReplicaHavePart(const String & replica, const String & part_name); - bool checkDetachPartIfExists(const String & part_name); - bool checkDetachPartitionIfExists(const String & partition_name); + bool checkIfDetachedPartExists(const String & part_name); + bool checkIfDetachedPartitionExists(const String & partition_name); /** Find replica having specified part or any part that covers it. * If active = true, consider only active replicas. diff --git a/tests/queries/0_stateless/01271_show_privileges.reference b/tests/queries/0_stateless/01271_show_privileges.reference index e2784f1dfd5..c8b8662dc3e 100644 --- a/tests/queries/0_stateless/01271_show_privileges.reference +++ b/tests/queries/0_stateless/01271_show_privileges.reference @@ -28,7 +28,7 @@ ALTER TTL ['ALTER MODIFY TTL','MODIFY TTL'] TABLE ALTER TABLE ALTER MATERIALIZE TTL ['MATERIALIZE TTL'] TABLE ALTER TABLE ALTER SETTINGS ['ALTER SETTING','ALTER MODIFY SETTING','MODIFY SETTING'] TABLE ALTER TABLE ALTER MOVE PARTITION ['ALTER MOVE PART','MOVE PARTITION','MOVE PART'] TABLE ALTER TABLE -ALTER FETCH PARTITION ['ALTER FETCH PART','FETCH PARTITION','FETCH PART'] TABLE ALTER TABLE +ALTER FETCH PARTITION ['ALTER FETCH PART','FETCH PARTITION'] TABLE ALTER TABLE ALTER FREEZE PARTITION ['FREEZE PARTITION','UNFREEZE'] TABLE ALTER TABLE ALTER TABLE [] \N ALTER ALTER VIEW REFRESH ['ALTER LIVE VIEW REFRESH','REFRESH VIEW'] VIEW ALTER VIEW From 96a95b05dbc80ab558181725b7a14c198670d5b9 Mon Sep 17 00:00:00 2001 From: songenjie Date: Wed, 14 Apr 2021 10:14:13 +0800 Subject: [PATCH 42/76] fix suggests --- tests/testflows/rbac/tests/privileges/grant_option.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testflows/rbac/tests/privileges/grant_option.py b/tests/testflows/rbac/tests/privileges/grant_option.py index a28350a78b9..bc8b73eb32f 100644 --- a/tests/testflows/rbac/tests/privileges/grant_option.py +++ b/tests/testflows/rbac/tests/privileges/grant_option.py @@ -89,7 +89,7 @@ def grant_option_check(grant_option_target, grant_target, user_name, table_type, @Examples("privilege", [ ("ALTER MOVE PARTITION",), ("ALTER MOVE PART",), ("MOVE PARTITION",), ("MOVE PART",), ("ALTER DELETE",), ("DELETE",), - ("ALTER FETCH PARTITION",), ("ALTER FETCH PART",), ("FETCH PARTITION",), ("FETCH PART",), + ("ALTER FETCH PARTITION",), ("ALTER FETCH PART",), ("FETCH PARTITION",), ("ALTER FREEZE PARTITION",), ("FREEZE PARTITION",), ("ALTER UPDATE",), ("UPDATE",), ("ALTER ADD COLUMN",), ("ADD COLUMN",), From ff8958965a095c5d7d1c67a523e1702c5df57ba6 Mon Sep 17 00:00:00 2001 From: songenjie Date: Wed, 14 Apr 2021 10:19:08 +0800 Subject: [PATCH 43/76] fix some docs --- docs/en/sql-reference/statements/grant.md | 2 +- website/blog/en/2016/how-to-update-data-in-clickhouse.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/statements/grant.md b/docs/en/sql-reference/statements/grant.md index daa020f9469..0a5c737b550 100644 --- a/docs/en/sql-reference/statements/grant.md +++ b/docs/en/sql-reference/statements/grant.md @@ -279,7 +279,7 @@ Allows executing [ALTER](../../sql-reference/statements/alter/index.md) queries - `ALTER MATERIALIZE TTL`. Level: `TABLE`. Aliases: `MATERIALIZE TTL` - `ALTER SETTINGS`. Level: `TABLE`. Aliases: `ALTER SETTING`, `ALTER MODIFY SETTING`, `MODIFY SETTING` - `ALTER MOVE PARTITION`. Level: `TABLE`. Aliases: `ALTER MOVE PART`, `MOVE PARTITION`, `MOVE PART` - - `ALTER FETCH PARTITION`. Level: `TABLE`. Aliases: `FETCH PARTITION`, `FETCH PART` + - `ALTER FETCH PARTITION`. Level: `TABLE`. Aliases: `ALTER FETCH PART`, `FETCH PARTITION`, `FETCH PART` - `ALTER FREEZE PARTITION`. Level: `TABLE`. Aliases: `FREEZE PARTITION` - `ALTER VIEW` Level: `GROUP` - `ALTER VIEW REFRESH`. Level: `VIEW`. Aliases: `ALTER LIVE VIEW REFRESH`, `REFRESH VIEW` diff --git a/website/blog/en/2016/how-to-update-data-in-clickhouse.md b/website/blog/en/2016/how-to-update-data-in-clickhouse.md index 7e9b938203f..ed713db2aee 100644 --- a/website/blog/en/2016/how-to-update-data-in-clickhouse.md +++ b/website/blog/en/2016/how-to-update-data-in-clickhouse.md @@ -67,7 +67,7 @@ There is a nice set of operations to work with partitions: - `DROP PARTITION` - Delete a partition. - `ATTACH PART|PARTITION` -- Add a new part or partition from the 'detached' directory to the table. - `FREEZE PARTITION` - Create a backup of a partition. -- `FETCH PART|PARTITION` - Download a part or partition from another server. +- `FETCH PARTITION|PART` - Download a part or partition from another server. We can do any data management operations on partitions level: move, copy and delete. Also, special DETACH and ATTACH operations are created to simplify data manipulation. DETACH detaches partition from table, moving all data to detached directory. Data is still there and you can copy it anywhere but detached data is not visible on request level. ATTACH is the opposite: attaches data from detached directory so it become visible. From 8566df9b7d4eb6592a496608852145f78bcb24da Mon Sep 17 00:00:00 2001 From: songenjie Date: Wed, 14 Apr 2021 17:11:59 +0800 Subject: [PATCH 44/76] fix some Suggest --- src/Storages/StorageReplicatedMergeTree.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 977ade9f1b8..d5c12628848 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -130,7 +130,7 @@ namespace ErrorCodes extern const int UNKNOWN_POLICY; extern const int NO_SUCH_DATA_PART; extern const int INTERSERVER_SCHEME_DOESNT_MATCH; - extern const int DUPLICATED_PART_UUIDS; + extern const int DUPLICATE_DATA_PART; } namespace ActionLocks @@ -5394,7 +5394,7 @@ void StorageReplicatedMergeTree::fetchPartition( * Unreliable (there is a race condition) - such a part may appear a little later. */ if (checkIfDetachedPartExists(part_name)) - throw Exception("Detached part " + part_name + " already exists.", ErrorCodes::DUPLICATED_PART_UUIDS); + throw Exception("Detached part " + part_name + " already exists.", ErrorCodes::DUPLICATE_DATA_PART); LOG_INFO(log, "Will fetch part {} from shard {} (zookeeper '{}')", part_name, from_, auxiliary_zookeeper_name); try From 88b4994c65740d7e0f35cfff67edd1ede8f82d18 Mon Sep 17 00:00:00 2001 From: songenjie Date: Wed, 14 Apr 2021 17:49:06 +0800 Subject: [PATCH 45/76] fix some Suggestfix some Suggest --- website/blog/en/2016/how-to-update-data-in-clickhouse.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/blog/en/2016/how-to-update-data-in-clickhouse.md b/website/blog/en/2016/how-to-update-data-in-clickhouse.md index ed713db2aee..22c2fa3ccc1 100644 --- a/website/blog/en/2016/how-to-update-data-in-clickhouse.md +++ b/website/blog/en/2016/how-to-update-data-in-clickhouse.md @@ -67,7 +67,7 @@ There is a nice set of operations to work with partitions: - `DROP PARTITION` - Delete a partition. - `ATTACH PART|PARTITION` -- Add a new part or partition from the 'detached' directory to the table. - `FREEZE PARTITION` - Create a backup of a partition. -- `FETCH PARTITION|PART` - Download a part or partition from another server. +- `FETCH PARTITION` - Download a partition from another server. We can do any data management operations on partitions level: move, copy and delete. Also, special DETACH and ATTACH operations are created to simplify data manipulation. DETACH detaches partition from table, moving all data to detached directory. Data is still there and you can copy it anywhere but detached data is not visible on request level. ATTACH is the opposite: attaches data from detached directory so it become visible. From 1fc040ac58c2cd00d5c6fa7bf64e0875e93e26ea Mon Sep 17 00:00:00 2001 From: songenjie Date: Wed, 14 Apr 2021 17:54:56 +0800 Subject: [PATCH 46/76] fix some Suggestfix some Suggest --- src/Storages/StorageReplicatedMergeTree.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index d5c12628848..f729f0a5be5 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -5389,19 +5389,19 @@ void StorageReplicatedMergeTree::fetchPartition( auto part_path = findReplicaHavingPart(part_name, from, zookeeper); if (part_path.empty()) - throw Exception("fetch part " + part_name + " not exists !", ErrorCodes::NO_REPLICA_HAS_PART); + throw Exception(ErrorCodes::PART_DOESNT_EXIST, "Part {} does not exist on any replica", part_name); /** Let's check that there is no such part in the `detached` directory (where we will write the downloaded parts). * Unreliable (there is a race condition) - such a part may appear a little later. */ if (checkIfDetachedPartExists(part_name)) - throw Exception("Detached part " + part_name + " already exists.", ErrorCodes::DUPLICATE_DATA_PART); + throw Exception(ErrorCodes::DUPLICATE_DATA_PART, "Detached part " + part_name + " already exists."); LOG_INFO(log, "Will fetch part {} from shard {} (zookeeper '{}')", part_name, from_, auxiliary_zookeeper_name); try { /// part name , metadata, part_path , true, 0, zookeeper if (!fetchPart(part_name, metadata_snapshot, part_path, true, 0, zookeeper)) - throw Exception("fetch part " + part_name + " failed! ", ErrorCodes::UNFINISHED); + throw Exception(ErrorCodes::UNFINISHED, "Failed to fetch part {} from {}", part_name, from_); } catch (const DB::Exception & e) { From 2534ed84d58c914dee424a843d9f32225f47ac25 Mon Sep 17 00:00:00 2001 From: songenjie Date: Wed, 14 Apr 2021 18:40:15 +0800 Subject: [PATCH 47/76] test_fetch_partition_from_auxiliary_zookeeper --- .../__init__.py | 0 .../configs/zookeeper_config.xml | 28 ------------- .../test.py | 40 ------------------- .../test.py | 22 +++++++--- 4 files changed, 16 insertions(+), 74 deletions(-) delete mode 100644 tests/integration/test_fetch_part_from_auxiliary_zookeeper/__init__.py delete mode 100644 tests/integration/test_fetch_part_from_auxiliary_zookeeper/configs/zookeeper_config.xml delete mode 100644 tests/integration/test_fetch_part_from_auxiliary_zookeeper/test.py diff --git a/tests/integration/test_fetch_part_from_auxiliary_zookeeper/__init__.py b/tests/integration/test_fetch_part_from_auxiliary_zookeeper/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/integration/test_fetch_part_from_auxiliary_zookeeper/configs/zookeeper_config.xml b/tests/integration/test_fetch_part_from_auxiliary_zookeeper/configs/zookeeper_config.xml deleted file mode 100644 index b2b0667ebbf..00000000000 --- a/tests/integration/test_fetch_part_from_auxiliary_zookeeper/configs/zookeeper_config.xml +++ /dev/null @@ -1,28 +0,0 @@ - - - - zoo1 - 2181 - - - zoo2 - 2181 - - - zoo3 - 2181 - - - - - - zoo1 - 2181 - - - zoo2 - 2181 - - - - diff --git a/tests/integration/test_fetch_part_from_auxiliary_zookeeper/test.py b/tests/integration/test_fetch_part_from_auxiliary_zookeeper/test.py deleted file mode 100644 index 17617f1c45c..00000000000 --- a/tests/integration/test_fetch_part_from_auxiliary_zookeeper/test.py +++ /dev/null @@ -1,40 +0,0 @@ - - -import pytest -from helpers.client import QueryRuntimeException -from helpers.cluster import ClickHouseCluster - -cluster = ClickHouseCluster(__file__) -node = cluster.add_instance("node", main_configs=["configs/zookeeper_config.xml"], with_zookeeper=True) - - -@pytest.fixture(scope="module") -def start_cluster(): - try: - cluster.start() - - yield cluster - finally: - cluster.shutdown() - - -def test_fetch_part_from_allowed_zookeeper(start_cluster): - node.query( - "CREATE TABLE simple (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', 'node') ORDER BY tuple() PARTITION BY date;" - ) - node.query("INSERT INTO simple VALUES ('2020-08-27', 1)") - - node.query( - "CREATE TABLE simple2 (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/1/simple', 'node') ORDER BY tuple() PARTITION BY date;" - ) - node.query( - "ALTER TABLE simple2 FETCH PART '20200827_0_0_0' FROM 'zookeeper2:/clickhouse/tables/0/simple';" - ) - node.query("ALTER TABLE simple2 ATTACH PART '20200827_0_0_0';") - - with pytest.raises(QueryRuntimeException): - node.query( - "ALTER TABLE simple2 FETCH PART '20200827_0_0_0' FROM 'zookeeper:/clickhouse/tables/0/simple';" - ) - - assert node.query("SELECT id FROM simple2").strip() == "1" diff --git a/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py b/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py index 0c94dfd3c48..d8d240349fc 100644 --- a/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py +++ b/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py @@ -18,7 +18,14 @@ def start_cluster(): cluster.shutdown() -def test_fetch_partition_from_allowed_zookeeper(start_cluster): +@pytest.mark.parametrize( + ('part', 'part_name'), + [ + ('PARTITION', '2020-08-27'), + ('PART', '20200827_0_0_0'), + ] +) +def test_fetch_part_from_allowed_zookeeper(start_cluster, part, part_name): node.query( "CREATE TABLE simple (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', 'node') ORDER BY tuple() PARTITION BY date;" ) @@ -27,14 +34,17 @@ def test_fetch_partition_from_allowed_zookeeper(start_cluster): node.query( "CREATE TABLE simple2 (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/1/simple', 'node') ORDER BY tuple() PARTITION BY date;" ) + node.query( - "ALTER TABLE simple2 FETCH PARTITION '2020-08-27' FROM 'zookeeper2:/clickhouse/tables/0/simple';" - ) - node.query("ALTER TABLE simple2 ATTACH PARTITION '2020-08-27';") + """ALTER TABLE simple2 FETCH {part} '{part_name}' FROM 'zookeeper2:/clickhouse/tables/0/simple';""".format( + part=part, part_name=part_name)) + + node.query("""ALTER TABLE simple2 ATTACH {part} '{part_name}';""".format( + part=part, part_name=part_name)) with pytest.raises(QueryRuntimeException): node.query( - "ALTER TABLE simple2 FETCH PARTITION '2020-08-27' FROM 'zookeeper:/clickhouse/tables/0/simple';" - ) + """ALTER TABLE simple2 FETCH {part} '{part_name}' FROM 'zookeeper:/clickhouse/tables/0/simple';""".format( + part=part, part_name=part_name)) assert node.query("SELECT id FROM simple2").strip() == "1" From c06c624fc7ae0acbde4b7daf4f044308e91df359 Mon Sep 17 00:00:00 2001 From: songenjie Date: Wed, 14 Apr 2021 18:55:42 +0800 Subject: [PATCH 48/76] fix build --- src/Storages/StorageReplicatedMergeTree.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index f729f0a5be5..10061af22e7 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -5389,7 +5389,7 @@ void StorageReplicatedMergeTree::fetchPartition( auto part_path = findReplicaHavingPart(part_name, from, zookeeper); if (part_path.empty()) - throw Exception(ErrorCodes::PART_DOESNT_EXIST, "Part {} does not exist on any replica", part_name); + throw Exception(ErrorCodes::NO_REPLICA_HAS_PART, "Part {} does not exist on any replica", part_name); /** Let's check that there is no such part in the `detached` directory (where we will write the downloaded parts). * Unreliable (there is a race condition) - such a part may appear a little later. */ From 382f702f592345789c071ba0ab28e26f4a247443 Mon Sep 17 00:00:00 2001 From: songenjie Date: Wed, 14 Apr 2021 20:04:59 +0800 Subject: [PATCH 49/76] test add param date --- .../test.py | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py b/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py index d8d240349fc..9553b0b64d3 100644 --- a/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py +++ b/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py @@ -1,5 +1,3 @@ - - import pytest from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster @@ -19,32 +17,32 @@ def start_cluster(): @pytest.mark.parametrize( - ('part', 'part_name'), + ('part', 'date', 'part_name'), [ - ('PARTITION', '2020-08-27'), - ('PART', '20200827_0_0_0'), + ('PARTITION', '2020-08-27', '2020-08-27'), + ('PART', '2020-08-28' '20200828_0_0_0'), ] ) -def test_fetch_part_from_allowed_zookeeper(start_cluster, part, part_name): +def test_fetch_part_from_allowed_zookeeper(start_cluster, part, date, part_name): node.query( - "CREATE TABLE simple (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', 'node') ORDER BY tuple() PARTITION BY date;" + "CREATE TABLE IF NOT EXISTS simple (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', 'node') ORDER BY tuple() PARTITION BY date;" ) - node.query("INSERT INTO simple VALUES ('2020-08-27', 1)") + + node.query("""INSERT INTO simple VALUES ('{date}', 1)""".format(date=date)) node.query( - "CREATE TABLE simple2 (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/1/simple', 'node') ORDER BY tuple() PARTITION BY date;" + "CREATE TABLE IF NOT EXISTS simple2 (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/1/simple', 'node') ORDER BY tuple() PARTITION BY date;" ) node.query( """ALTER TABLE simple2 FETCH {part} '{part_name}' FROM 'zookeeper2:/clickhouse/tables/0/simple';""".format( part=part, part_name=part_name)) - node.query("""ALTER TABLE simple2 ATTACH {part} '{part_name}';""".format( - part=part, part_name=part_name)) + node.query("""ALTER TABLE simple2 ATTACH {part} '{part_name}';""".format(part=part, part_name=part_name)) with pytest.raises(QueryRuntimeException): node.query( """ALTER TABLE simple2 FETCH {part} '{part_name}' FROM 'zookeeper:/clickhouse/tables/0/simple';""".format( part=part, part_name=part_name)) - assert node.query("SELECT id FROM simple2").strip() == "1" + assert node.query("""SELECT id FROM simple2 where date = '{date}'""".format(date=date)).strip() == "1" From ffd3b3d445036afe43bc941ac1b88a9b0f5cad2b Mon Sep 17 00:00:00 2001 From: songenjie Date: Wed, 14 Apr 2021 21:15:53 +0800 Subject: [PATCH 50/76] fix some docs --- .../test_fetch_partition_from_auxiliary_zookeeper/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py b/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py index 9553b0b64d3..7bce2d50011 100644 --- a/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py +++ b/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py @@ -20,7 +20,7 @@ def start_cluster(): ('part', 'date', 'part_name'), [ ('PARTITION', '2020-08-27', '2020-08-27'), - ('PART', '2020-08-28' '20200828_0_0_0'), + ('PART', '2020-08-28', '20200828_0_0_0'), ] ) def test_fetch_part_from_allowed_zookeeper(start_cluster, part, date, part_name): From 2c3abcaad12175b2545990e2f37515ba4c270523 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 14 Apr 2021 16:49:38 +0300 Subject: [PATCH 51/76] Updated test --- src/Dictionaries/DirectDictionary.cpp | 8 ++++++++ tests/performance/flat_dictionary.xml | 6 ++++-- tests/performance/hashed_dictionary.xml | 3 ++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/Dictionaries/DirectDictionary.cpp b/src/Dictionaries/DirectDictionary.cpp index bacb1a87dc9..ed5da3eead0 100644 --- a/src/Dictionaries/DirectDictionary.cpp +++ b/src/Dictionaries/DirectDictionary.cpp @@ -51,6 +51,14 @@ Columns DirectDictionary::getColumns( key_to_fetched_index.reserve(requested_keys.size()); auto fetched_columns_from_storage = request.makeAttributesResultColumns(); + for (size_t attribute_index = 0; attribute_index < request.attributesSize(); ++attribute_index) + { + if (!request.shouldFillResultColumnWithIndex(attribute_index)) + continue; + + auto & fetched_column_from_storage = fetched_columns_from_storage[attribute_index]; + fetched_column_from_storage->reserve(requested_keys.size()); + } size_t fetched_key_index = 0; diff --git a/tests/performance/flat_dictionary.xml b/tests/performance/flat_dictionary.xml index 56a94358eb9..a80631db541 100644 --- a/tests/performance/flat_dictionary.xml +++ b/tests/performance/flat_dictionary.xml @@ -53,7 +53,8 @@ - SELECT dictGet('default.simple_key_flat_dictionary', {column_name}, rand64() % toUInt64({elements_count})) + WITH rand64() % toUInt64({elements_count}) as key + SELECT dictGet('default.simple_key_flat_dictionary', {column_name}, key) FROM system.numbers LIMIT {elements_count} FORMAT Null; @@ -65,7 +66,8 @@ - SELECT dictHas('default.simple_key_flat_dictionary', rand64() % toUInt64(75000000)) + WITH rand64() % toUInt64(75000000) as key + SELECT dictHas('default.simple_key_flat_dictionary', key) FROM system.numbers LIMIT 75000000 FORMAT Null; diff --git a/tests/performance/hashed_dictionary.xml b/tests/performance/hashed_dictionary.xml index cd19ba035e5..5cbe1caeb23 100644 --- a/tests/performance/hashed_dictionary.xml +++ b/tests/performance/hashed_dictionary.xml @@ -95,7 +95,8 @@ FORMAT Null; - SELECT * FROM default.simple_key_hashed_dictionary; + SELECT * FROM default.simple_key_hashed_dictionary + FORMAT Null; WITH rand64() % toUInt64({elements_count}) as key From da6dc64e0468d54613e915bafe95083f687fc8d6 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 15 Apr 2021 00:34:46 +0300 Subject: [PATCH 52/76] jemalloc: set dirty_decay_ms/muzzy_decay_ms to 1 second --- contrib/jemalloc-cmake/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt index b174d4d361e..a82345975d1 100644 --- a/contrib/jemalloc-cmake/CMakeLists.txt +++ b/contrib/jemalloc-cmake/CMakeLists.txt @@ -34,9 +34,9 @@ if (OS_LINUX) # avoid spurious latencies and additional work associated with # MADV_DONTNEED. See # https://github.com/ClickHouse/ClickHouse/issues/11121 for motivation. - set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:10000") + set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:1000,dirty_decay_ms:1000") else() - set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0,muzzy_decay_ms:10000") + set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0,muzzy_decay_ms:1000,dirty_decay_ms:1000") endif() # CACHE variable is empty, to allow changing defaults without necessity # to purge cache From 7f7e04117d1094240def5f34bb7e44d5f1ba8762 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 15 Apr 2021 01:27:53 +0300 Subject: [PATCH 53/76] Arcadia, ARM, PowerPC --- src/Functions/divide/CMakeLists.txt | 26 +++++++++++++++++++------- src/Functions/divide/divide.cpp | 4 ++++ src/Functions/divide/divideImpl.cpp | 6 +++++- src/Functions/ya.make | 2 ++ 4 files changed, 30 insertions(+), 8 deletions(-) diff --git a/src/Functions/divide/CMakeLists.txt b/src/Functions/divide/CMakeLists.txt index 2bdd7e4c5ef..e5a10f0817c 100644 --- a/src/Functions/divide/CMakeLists.txt +++ b/src/Functions/divide/CMakeLists.txt @@ -1,10 +1,22 @@ -add_library(divide_impl_sse2 divideImpl.cpp) -target_compile_options(divide_impl_sse2 PRIVATE -msse2 -DNAMESPACE=SSE2) -target_link_libraries(divide_impl_sse2 libdivide) +# A library for integer division by constant with CPU dispatching. -add_library(divide_impl_avx2 divideImpl.cpp) -target_compile_options(divide_impl_avx2 PRIVATE -mavx2 -DNAMESPACE=AVX2) -target_link_libraries(divide_impl_avx2 libdivide) +if (ARCH_AMD64) + add_library(divide_impl_sse2 divideImpl.cpp) + target_compile_options(divide_impl_sse2 PRIVATE -msse2 -DNAMESPACE=SSE2) + target_link_libraries(divide_impl_sse2 libdivide) + + add_library(divide_impl_avx2 divideImpl.cpp) + target_compile_options(divide_impl_avx2 PRIVATE -mavx2 -DNAMESPACE=AVX2) + target_link_libraries(divide_impl_avx2 libdivide) + + set(IMPLEMENTATIONS divide_impl_sse2 divide_impl_avx2) +else () + add_library(divide_impl_generic divideImpl.cpp) + target_compile_options(divide_impl_generic PRIVATE -DNAMESPACE=Generic) + target_link_libraries(divide_impl_generic libdivide) + + set(IMPLEMENTATIONS divide_impl_generic) +endif () add_library(divide_impl divide.cpp) -target_link_libraries(divide_impl divide_impl_sse2 divide_impl_avx2 clickhouse_common_io) +target_link_libraries(divide_impl ${IMPLEMENTATIONS} clickhouse_common_io) diff --git a/src/Functions/divide/divide.cpp b/src/Functions/divide/divide.cpp index 1c3c11af312..7676c2cb02b 100644 --- a/src/Functions/divide/divide.cpp +++ b/src/Functions/divide/divide.cpp @@ -18,10 +18,14 @@ namespace AVX2 template void divideImpl(const A * __restrict a_pos, B b, ResultType * __restrict c_pos, size_t size) { +#if defined(__x86_64__) && !defined(ARCADIA_BUILD) if (DB::Cpu::CpuFlagsCache::have_AVX2) AVX2::divideImpl(a_pos, b, c_pos, size); else if (DB::Cpu::CpuFlagsCache::have_SSE2) SSE2::divideImpl(a_pos, b, c_pos, size); +#else + Generic::divideImpl(a_pos, b, c_pos, size); +#endif } diff --git a/src/Functions/divide/divideImpl.cpp b/src/Functions/divide/divideImpl.cpp index a62ce8126e2..f4c1a97d3ad 100644 --- a/src/Functions/divide/divideImpl.cpp +++ b/src/Functions/divide/divideImpl.cpp @@ -2,7 +2,11 @@ /// with different values of NAMESPACE and machine flags (sse2, avx2). #if !defined(NAMESPACE) -#error "NAMESPACE macro must be defined" + #if defined(ARCADIA_BUILD) + #define NAMESPACE Generic + #else + #error "NAMESPACE macro must be defined" + #endif #endif #if defined(__AVX2__) diff --git a/src/Functions/ya.make b/src/Functions/ya.make index 52ed54ec64f..660f7b115bf 100644 --- a/src/Functions/ya.make +++ b/src/Functions/ya.make @@ -229,6 +229,8 @@ SRCS( defaultValueOfTypeName.cpp demange.cpp divide.cpp + divide/divide.cpp + divide/divideImpl.cpp dumpColumnStructure.cpp e.cpp empty.cpp From 2ae8839e3dc63ace3f8744817a81b192274a3c27 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 15 Apr 2021 01:28:40 +0300 Subject: [PATCH 54/76] Style --- src/Functions/divide/divide.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Functions/divide/divide.h b/src/Functions/divide/divide.h index daf406038f2..1c17a461159 100644 --- a/src/Functions/divide/divide.h +++ b/src/Functions/divide/divide.h @@ -1,3 +1,5 @@ +#pragma once + #include template From 076c746e6d4835dc0a3239ef21065268f70af812 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 15 Apr 2021 01:35:21 +0300 Subject: [PATCH 55/76] Add perf test --- tests/performance/intDiv.xml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 tests/performance/intDiv.xml diff --git a/tests/performance/intDiv.xml b/tests/performance/intDiv.xml new file mode 100644 index 00000000000..c6fa0238986 --- /dev/null +++ b/tests/performance/intDiv.xml @@ -0,0 +1,5 @@ + + SELECT count() FROM numbers(200000000) WHERE NOT ignore(intDiv(number, 1000000000)) + SELECT count() FROM numbers(200000000) WHERE NOT ignore(divide(number, 1000000000)) + SELECT count() FROM numbers(200000000) WHERE NOT ignore(toUInt32(divide(number, 1000000000))) + From 15153e504ade2e7a0c841d0a36b42272d5b9d6b9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 15 Apr 2021 02:08:43 +0300 Subject: [PATCH 56/76] Fix unpleasant behaviour of Markdown format --- src/Processors/Formats/Impl/MarkdownRowOutputFormat.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/Processors/Formats/Impl/MarkdownRowOutputFormat.cpp b/src/Processors/Formats/Impl/MarkdownRowOutputFormat.cpp index 5108650ff0d..ee5d4193a45 100644 --- a/src/Processors/Formats/Impl/MarkdownRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/MarkdownRowOutputFormat.cpp @@ -21,16 +21,13 @@ void MarkdownRowOutputFormat::writePrefix() } writeCString("\n|", out); String left_alignment = ":-|"; - String central_alignment = ":-:|"; String right_alignment = "-:|"; for (size_t i = 0; i < columns; ++i) { - if (isInteger(types[i])) + if (types[i]->shouldAlignRightInPrettyFormats()) writeString(right_alignment, out); - else if (isString(types[i])) - writeString(left_alignment, out); else - writeString(central_alignment, out); + writeString(left_alignment, out); } writeChar('\n', out); } From 8bd77e1c0c0a12c3764d89545fc775b5e2acf3e5 Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Thu, 15 Apr 2021 11:58:14 +0300 Subject: [PATCH 57/76] Change markdown format test --- .../0_stateless/01231_markdown_format.reference | 10 +++++----- tests/queries/0_stateless/01231_markdown_format.sql | 6 ++++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/queries/0_stateless/01231_markdown_format.reference b/tests/queries/0_stateless/01231_markdown_format.reference index e2ec03b401a..65838bfede7 100644 --- a/tests/queries/0_stateless/01231_markdown_format.reference +++ b/tests/queries/0_stateless/01231_markdown_format.reference @@ -1,5 +1,5 @@ -| id | name | array | -|-:|:-|:-:| -| 1 | name1 | [1,2,3] | -| 2 | name2 | [4,5,6] | -| 3 | name3 | [7,8,9] | +| id | name | array | nullable | low_cardinality | decimal | +|-:|:-|:-|:-|:-|-:| +| 1 | name1 | [1,2,3] | Some long string | name1 | 1.110000 | +| 2 | name2 | [4,5,60000] | \N | Another long string | 222.222222 | +| 30000 | One more long string | [7,8,9] | name3 | name3 | 3.330000 | diff --git a/tests/queries/0_stateless/01231_markdown_format.sql b/tests/queries/0_stateless/01231_markdown_format.sql index 693664be1ab..287e9a0e91e 100644 --- a/tests/queries/0_stateless/01231_markdown_format.sql +++ b/tests/queries/0_stateless/01231_markdown_format.sql @@ -1,6 +1,8 @@ DROP TABLE IF EXISTS makrdown; -CREATE TABLE markdown (id UInt32, name String, array Array(Int8)) ENGINE = Memory; -INSERT INTO markdown VALUES (1, 'name1', [1,2,3]), (2, 'name2', [4,5,6]), (3, 'name3', [7,8,9]); +CREATE TABLE markdown (id UInt32, name String, array Array(Int32), nullable Nullable(String), low_cardinality LowCardinality(String), decimal Decimal32(6)) ENGINE = Memory; +INSERT INTO markdown VALUES (1, 'name1', [1,2,3], 'Some long string', 'name1', 1.11), (2, 'name2', [4,5,60000], Null, 'Another long string', 222.222222), (30000, 'One more long string', [7,8,9], 'name3', 'name3', 3.33); SELECT * FROM markdown FORMAT Markdown; DROP TABLE IF EXISTS markdown + + From b909899cc9a0665396c6486264889c15858f6800 Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Thu, 15 Apr 2021 11:59:56 +0300 Subject: [PATCH 58/76] Remove extra lines --- tests/queries/0_stateless/01231_markdown_format.sql | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/queries/0_stateless/01231_markdown_format.sql b/tests/queries/0_stateless/01231_markdown_format.sql index 287e9a0e91e..65c65389e12 100644 --- a/tests/queries/0_stateless/01231_markdown_format.sql +++ b/tests/queries/0_stateless/01231_markdown_format.sql @@ -4,5 +4,3 @@ INSERT INTO markdown VALUES (1, 'name1', [1,2,3], 'Some long string', 'name1', 1 SELECT * FROM markdown FORMAT Markdown; DROP TABLE IF EXISTS markdown - - From 75036debf4e0d91087948a22c4d2cd2242f9c1f1 Mon Sep 17 00:00:00 2001 From: Dmitry Krylov Date: Thu, 15 Apr 2021 19:25:30 +1000 Subject: [PATCH 59/76] Check type match of lambda and accumulator --- src/Functions/array/arrayFold.cpp | 17 ++++++++++++----- .../0_stateless/01813_array_fold_errors.sql | 1 + 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/Functions/array/arrayFold.cpp b/src/Functions/array/arrayFold.cpp index 5c80b01c5c9..bf0f019f0d8 100644 --- a/src/Functions/array/arrayFold.cpp +++ b/src/Functions/array/arrayFold.cpp @@ -9,8 +9,9 @@ namespace ErrorCodes { extern const int ILLEGAL_COLUMN; extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int SIZES_OF_ARRAYS_DOESNT_MATCH; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int SIZES_OF_ARRAYS_DOESNT_MATCH; + extern const int TYPE_MISMATCH; } @@ -26,8 +27,6 @@ public: bool isVariadic() const override { return true; } size_t getNumberOfArguments() const override { return 0; } - /// Called if at least one function argument is a lambda expression. - /// For argument-lambda expressions, it defines the types of arguments of these expressions. void getLambdaArgumentTypes(DataTypes & arguments) const override { if (arguments.size() < 3) @@ -64,8 +63,16 @@ public: if (!data_type_function) throw Exception("First argument for function " + getName() + " must be a function.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - /// The types of the remaining arguments are already checked in getLambdaArgumentTypes. - return DataTypePtr(arguments.back().type); + + auto const accumulator_type = arguments.back().type; + auto const lambda_type = data_type_function->getReturnType(); + if (! accumulator_type->equals(*lambda_type)) + throw Exception("Return type of lambda function must be the same as the accumulator type. " + "Inferred type of lambda " + lambda_type->getName() + ", " + + "inferred type of accumulator " + accumulator_type->getName() + ".", + ErrorCodes::TYPE_MISMATCH); + + return accumulator_type; } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override diff --git a/tests/queries/0_stateless/01813_array_fold_errors.sql b/tests/queries/0_stateless/01813_array_fold_errors.sql index 49fd085dfe2..9b3776d7cea 100644 --- a/tests/queries/0_stateless/01813_array_fold_errors.sql +++ b/tests/queries/0_stateless/01813_array_fold_errors.sql @@ -9,3 +9,4 @@ SELECT arrayFold(x,acc -> acc+x, number, toInt64(0)) FROM system.numbers LIMIT 1 SELECT arrayFold(x,y,acc -> acc + x * 2 + y * 3, [1,2,3,4], [5,6,7], toInt64(3)); -- { serverError 190 } SELECT arrayFold(x,acc -> acc + x * 2 + y * 3, [1,2,3,4], [5,6,7,8], toInt64(3)); -- { serverError 47 } SELECT arrayFold(x,acc -> acc + x * 2, [1,2,3,4], [5,6,7,8], toInt64(3)); -- { serverError 43 } +SELECT arrayFold(x,acc -> concat(acc,', ', x), [1, 2, 3, 4], '0') -- { serverError 44 } From b8a1ead3e9899ff4cbda7f8866bc7e6ff4323496 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Thu, 15 Apr 2021 13:51:40 +0300 Subject: [PATCH 60/76] Updated hashed_dictionary test --- tests/performance/hashed_dictionary.xml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tests/performance/hashed_dictionary.xml b/tests/performance/hashed_dictionary.xml index 5cbe1caeb23..26164b4f888 100644 --- a/tests/performance/hashed_dictionary.xml +++ b/tests/performance/hashed_dictionary.xml @@ -94,10 +94,6 @@ LIMIT {elements_count} FORMAT Null; - - SELECT * FROM default.simple_key_hashed_dictionary - FORMAT Null; - WITH rand64() % toUInt64({elements_count}) as key SELECT dictHas('default.simple_key_hashed_dictionary', key) @@ -113,10 +109,6 @@ LIMIT {elements_count} FORMAT Null; - - SELECT * FROM default.complex_key_hashed_dictionary - FORMAT Null; - WITH (rand64() % toUInt64({elements_count}), toString(rand64() % toUInt64({elements_count}))) as key SELECT dictHas('default.complex_key_hashed_dictionary', key) From d19b2cb9489b4185261dfe347b5c851e829b58f3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 15 Apr 2021 14:41:55 +0300 Subject: [PATCH 61/76] Fix build --- src/Functions/divide/divide.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/Functions/divide/divide.cpp b/src/Functions/divide/divide.cpp index 7676c2cb02b..5ab11df2a65 100644 --- a/src/Functions/divide/divide.cpp +++ b/src/Functions/divide/divide.cpp @@ -1,7 +1,7 @@ #include "divide.h" #include - +#if defined(__x86_64__) && !defined(ARCADIA_BUILD) namespace SSE2 { template @@ -13,6 +13,13 @@ namespace AVX2 template void divideImpl(const A * __restrict a_pos, B b, ResultType * __restrict c_pos, size_t size); } +#else +namespace Generic +{ + template + void divideImpl(const A * __restrict a_pos, B b, ResultType * __restrict c_pos, size_t size); +} +#endif template From bf51f94f37400f9701bd1390ea5ca6a1c16341d5 Mon Sep 17 00:00:00 2001 From: Dmitry Krylov Date: Thu, 15 Apr 2021 21:46:15 +1000 Subject: [PATCH 62/76] Fix constness --- src/Functions/array/arrayFold.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/array/arrayFold.cpp b/src/Functions/array/arrayFold.cpp index bf0f019f0d8..5fc7a304b03 100644 --- a/src/Functions/array/arrayFold.cpp +++ b/src/Functions/array/arrayFold.cpp @@ -72,7 +72,7 @@ public: + "inferred type of accumulator " + accumulator_type->getName() + ".", ErrorCodes::TYPE_MISMATCH); - return accumulator_type; + return DataTypePtr(accumulator_type); } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override From 77bc9e04c6df527ad245b21a307a6ae9ffa0af1c Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Thu, 15 Apr 2021 17:50:28 +0300 Subject: [PATCH 63/76] Updated zlib with apple linker fix --- contrib/zlib-ng | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/zlib-ng b/contrib/zlib-ng index b82d3497a5a..bf128f84df0 160000 --- a/contrib/zlib-ng +++ b/contrib/zlib-ng @@ -1 +1 @@ -Subproject commit b82d3497a5afc46dec3c5d07e4b163b169f251d7 +Subproject commit bf128f84df0806ec51c3513804222ae02007c4f3 From d2cf03ea41babefd715436f1f9e23e48fb3e6f8d Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 15 Apr 2021 21:00:16 +0300 Subject: [PATCH 64/76] Change logging from trace to debug for messages with rows/bytes --- src/Interpreters/Aggregator.cpp | 8 ++++---- src/Processors/Formats/IRowInputFormat.cpp | 2 +- src/Processors/Transforms/AggregatingInOrderTransform.cpp | 4 ++-- src/Processors/Transforms/AggregatingTransform.cpp | 4 ++-- src/Processors/Transforms/MergingAggregatedTransform.cpp | 2 +- src/Storages/Distributed/DirectoryMonitor.cpp | 6 +++--- src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp | 4 ++-- src/Storages/MergeTree/MergeTreePartsMover.cpp | 2 +- .../MergeTree/MergeTreeReverseSelectProcessor.cpp | 2 +- src/Storages/MergeTree/MergeTreeSelectProcessor.cpp | 2 +- src/Storages/MergeTree/MergeTreeSequentialSource.cpp | 4 ++-- src/Storages/StorageBuffer.cpp | 6 +++--- 12 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index 0f8b647096d..ad9cc0fc2d2 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -834,7 +834,7 @@ void Aggregator::writeToTemporaryFile(AggregatedDataVariants & data_variants, co ProfileEvents::increment(ProfileEvents::ExternalAggregationCompressedBytes, compressed_bytes); ProfileEvents::increment(ProfileEvents::ExternalAggregationUncompressedBytes, uncompressed_bytes); - LOG_TRACE(log, + LOG_DEBUG(log, "Written part in {} sec., {} rows, {} uncompressed, {} compressed," " {} uncompressed bytes per row, {} compressed bytes per row, compression rate: {}" " ({} rows/sec., {}/sec. uncompressed, {}/sec. compressed)", @@ -947,7 +947,7 @@ void Aggregator::writeToTemporaryFileImpl( /// `data_variants` will not destroy them in the destructor, they are now owned by ColumnAggregateFunction objects. data_variants.aggregator = nullptr; - LOG_TRACE(log, "Max size of temporary block: {} rows, {}.", max_temporary_block_size_rows, ReadableSize(max_temporary_block_size_bytes)); + LOG_DEBUG(log, "Max size of temporary block: {} rows, {}.", max_temporary_block_size_rows, ReadableSize(max_temporary_block_size_bytes)); } @@ -1481,7 +1481,7 @@ BlocksList Aggregator::convertToBlocks(AggregatedDataVariants & data_variants, b } double elapsed_seconds = watch.elapsedSeconds(); - LOG_TRACE(log, + LOG_DEBUG(log, "Converted aggregated data to blocks. {} rows, {} in {} sec. ({} rows/sec., {}/sec.)", rows, ReadableSize(bytes), elapsed_seconds, rows / elapsed_seconds, @@ -2109,7 +2109,7 @@ Block Aggregator::mergeBlocks(BlocksList & blocks, bool final) size_t rows = block.rows(); size_t bytes = block.bytes(); double elapsed_seconds = watch.elapsedSeconds(); - LOG_TRACE(log, "Merged partially aggregated blocks. {} rows, {}. in {} sec. ({} rows/sec., {}/sec.)", + LOG_DEBUG(log, "Merged partially aggregated blocks. {} rows, {}. in {} sec. ({} rows/sec., {}/sec.)", rows, ReadableSize(bytes), elapsed_seconds, rows / elapsed_seconds, ReadableSize(bytes / elapsed_seconds)); diff --git a/src/Processors/Formats/IRowInputFormat.cpp b/src/Processors/Formats/IRowInputFormat.cpp index 75a9abf6845..52e64a9d90d 100644 --- a/src/Processors/Formats/IRowInputFormat.cpp +++ b/src/Processors/Formats/IRowInputFormat.cpp @@ -190,7 +190,7 @@ Chunk IRowInputFormat::generate() if (num_errors && (params.allow_errors_num > 0 || params.allow_errors_ratio > 0)) { Poco::Logger * log = &Poco::Logger::get("IRowInputFormat"); - LOG_TRACE(log, "Skipped {} rows with errors while reading the input stream", num_errors); + LOG_DEBUG(log, "Skipped {} rows with errors while reading the input stream", num_errors); } readSuffix(); diff --git a/src/Processors/Transforms/AggregatingInOrderTransform.cpp b/src/Processors/Transforms/AggregatingInOrderTransform.cpp index 392e27166ef..d8b7742cdf4 100644 --- a/src/Processors/Transforms/AggregatingInOrderTransform.cpp +++ b/src/Processors/Transforms/AggregatingInOrderTransform.cpp @@ -214,8 +214,8 @@ IProcessor::Status AggregatingInOrderTransform::prepare() { output.push(std::move(to_push_chunk)); output.finish(); - LOG_TRACE(log, "Aggregated. {} to {} rows (from {})", src_rows, res_rows, - formatReadableSizeWithBinarySuffix(src_bytes)); + LOG_DEBUG(log, "Aggregated. {} to {} rows (from {})", + src_rows, res_rows, formatReadableSizeWithBinarySuffix(src_bytes)); return Status::Finished; } if (input.isFinished()) diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index c6907202d31..3400d06dae3 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -541,7 +541,7 @@ void AggregatingTransform::initGenerate() double elapsed_seconds = watch.elapsedSeconds(); size_t rows = variants.sizeWithoutOverflowRow(); - LOG_TRACE(log, "Aggregated. {} to {} rows (from {}) in {} sec. ({} rows/sec., {}/sec.)", + LOG_DEBUG(log, "Aggregated. {} to {} rows (from {}) in {} sec. ({} rows/sec., {}/sec.)", src_rows, rows, ReadableSize(src_bytes), elapsed_seconds, src_rows / elapsed_seconds, ReadableSize(src_bytes / elapsed_seconds)); @@ -599,7 +599,7 @@ void AggregatingTransform::initGenerate() pipe = Pipe::unitePipes(std::move(pipes)); } - LOG_TRACE(log, "Will merge {} temporary files of size {} compressed, {} uncompressed.", files.files.size(), ReadableSize(files.sum_size_compressed), ReadableSize(files.sum_size_uncompressed)); + LOG_DEBUG(log, "Will merge {} temporary files of size {} compressed, {} uncompressed.", files.files.size(), ReadableSize(files.sum_size_compressed), ReadableSize(files.sum_size_uncompressed)); addMergingAggregatedMemoryEfficientTransform(pipe, params, temporary_data_merge_threads); diff --git a/src/Processors/Transforms/MergingAggregatedTransform.cpp b/src/Processors/Transforms/MergingAggregatedTransform.cpp index 1a04f85fd9c..ddc58d830da 100644 --- a/src/Processors/Transforms/MergingAggregatedTransform.cpp +++ b/src/Processors/Transforms/MergingAggregatedTransform.cpp @@ -52,7 +52,7 @@ Chunk MergingAggregatedTransform::generate() if (!generate_started) { generate_started = true; - LOG_TRACE(log, "Read {} blocks of partially aggregated data, total {} rows.", total_input_blocks, total_input_rows); + LOG_DEBUG(log, "Read {} blocks of partially aggregated data, total {} rows.", total_input_blocks, total_input_rows); /// Exception safety. Make iterator valid in case any method below throws. next_block = blocks.begin(); diff --git a/src/Storages/Distributed/DirectoryMonitor.cpp b/src/Storages/Distributed/DirectoryMonitor.cpp index 2afa9747c60..29b69209253 100644 --- a/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/src/Storages/Distributed/DirectoryMonitor.cpp @@ -535,7 +535,7 @@ void StorageDistributedDirectoryMonitor::processFile(const std::string & file_pa ReadBufferFromFile in(file_path); const auto & distributed_header = readDistributedHeader(in, log); - LOG_TRACE(log, "Started processing `{}` ({} rows, {} bytes)", file_path, + LOG_DEBUG(log, "Started processing `{}` ({} rows, {} bytes)", file_path, formatReadableQuantity(distributed_header.rows), formatReadableSizeWithBinarySuffix(distributed_header.bytes)); @@ -631,7 +631,7 @@ struct StorageDistributedDirectoryMonitor::Batch Stopwatch watch; - LOG_TRACE(parent.log, "Sending a batch of {} files ({} rows, {} bytes).", file_indices.size(), + LOG_DEBUG(parent.log, "Sending a batch of {} files ({} rows, {} bytes).", file_indices.size(), formatReadableQuantity(total_rows), formatReadableSizeWithBinarySuffix(total_bytes)); @@ -876,7 +876,7 @@ void StorageDistributedDirectoryMonitor::processFilesWithBatching(const std::map if (!total_rows || !header) { - LOG_TRACE(log, "Processing batch {} with old format (no header/rows)", in.getFileName()); + LOG_DEBUG(log, "Processing batch {} with old format (no header/rows)", in.getFileName()); CompressedReadBuffer decompressing_in(in); NativeBlockInputStream block_in(decompressing_in, DBMS_TCP_PROTOCOL_VERSION); diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index ddb140989f6..7f7370e6f1f 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -1054,7 +1054,7 @@ QueryPlanPtr MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreams( false); /// Let's estimate total number of rows for progress bar. - LOG_TRACE(log, "Reading approx. {} rows with {} streams", total_rows, num_streams); + LOG_DEBUG(log, "Reading approx. {} rows with {} streams", total_rows, num_streams); for (size_t i = 0; i < num_streams; ++i) { @@ -1576,7 +1576,7 @@ QueryPlanPtr MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal( settings.preferred_block_size_bytes, false); - LOG_TRACE(log, "Reading approx. {} rows with {} streams", total_rows_in_lonely_parts, num_streams_for_lonely_parts); + LOG_DEBUG(log, "Reading approx. {} rows with {} streams", total_rows_in_lonely_parts, num_streams_for_lonely_parts); for (size_t i = 0; i < num_streams_for_lonely_parts; ++i) { diff --git a/src/Storages/MergeTree/MergeTreePartsMover.cpp b/src/Storages/MergeTree/MergeTreePartsMover.cpp index cb21f50f9a0..f9e3883d5e2 100644 --- a/src/Storages/MergeTree/MergeTreePartsMover.cpp +++ b/src/Storages/MergeTree/MergeTreePartsMover.cpp @@ -182,7 +182,7 @@ bool MergeTreePartsMover::selectPartsForMove( if (!parts_to_move.empty()) { - LOG_TRACE(log, "Selected {} parts to move according to storage policy rules and {} parts according to TTL rules, {} total", parts_to_move_by_policy_rules, parts_to_move_by_ttl_rules, ReadableSize(parts_to_move_total_size_bytes)); + LOG_DEBUG(log, "Selected {} parts to move according to storage policy rules and {} parts according to TTL rules, {} total", parts_to_move_by_policy_rules, parts_to_move_by_ttl_rules, ReadableSize(parts_to_move_total_size_bytes)); return true; } else diff --git a/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp index 1d3bb55eace..e9527efaa4a 100644 --- a/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp @@ -47,7 +47,7 @@ MergeTreeReverseSelectProcessor::MergeTreeReverseSelectProcessor( size_t total_rows = data_part->index_granularity.getRowsCountInRanges(all_mark_ranges); if (!quiet) - LOG_TRACE(log, "Reading {} ranges in reverse order from part {}, approx. {} rows starting from {}", + LOG_DEBUG(log, "Reading {} ranges in reverse order from part {}, approx. {} rows starting from {}", all_mark_ranges.size(), data_part->name, total_rows, data_part->index_granularity.getMarkStartingRow(all_mark_ranges.front().begin)); diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp index 47429745b0d..980afa170e9 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp @@ -47,7 +47,7 @@ MergeTreeSelectProcessor::MergeTreeSelectProcessor( size_t total_rows = data_part->index_granularity.getRowsCountInRanges(all_mark_ranges); if (!quiet) - LOG_TRACE(log, "Reading {} ranges from part {}, approx. {} rows starting from {}", + LOG_DEBUG(log, "Reading {} ranges from part {}, approx. {} rows starting from {}", all_mark_ranges.size(), data_part->name, total_rows, data_part->index_granularity.getMarkStartingRow(all_mark_ranges.front().begin)); diff --git a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp index 745a2860c56..e82b1966461 100644 --- a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp +++ b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp @@ -29,10 +29,10 @@ MergeTreeSequentialSource::MergeTreeSequentialSource( { /// Print column name but don't pollute logs in case of many columns. if (columns_to_read.size() == 1) - LOG_TRACE(log, "Reading {} marks from part {}, total {} rows starting from the beginning of the part, column {}", + LOG_DEBUG(log, "Reading {} marks from part {}, total {} rows starting from the beginning of the part, column {}", data_part->getMarksCount(), data_part->name, data_part->rows_count, columns_to_read.front()); else - LOG_TRACE(log, "Reading {} marks from part {}, total {} rows starting from the beginning of the part", + LOG_DEBUG(log, "Reading {} marks from part {}, total {} rows starting from the beginning of the part", data_part->getMarksCount(), data_part->name, data_part->rows_count); } diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index 5c4b4e7d1d8..c9bfb9e1ee7 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -542,7 +542,7 @@ public: { if (storage.destination_id) { - LOG_TRACE(storage.log, "Writing block with {} rows, {} bytes directly.", rows, bytes); + LOG_DEBUG(storage.log, "Writing block with {} rows, {} bytes directly.", rows, bytes); storage.writeBlockToDestination(block, destination); } return; @@ -804,7 +804,7 @@ void StorageBuffer::flushBuffer(Buffer & buffer, bool check_thresholds, bool loc if (!destination_id) { - LOG_TRACE(log, "Flushing buffer with {} rows (discarded), {} bytes, age {} seconds {}.", rows, bytes, time_passed, (check_thresholds ? "(bg)" : "(direct)")); + LOG_DEBUG(log, "Flushing buffer with {} rows (discarded), {} bytes, age {} seconds {}.", rows, bytes, time_passed, (check_thresholds ? "(bg)" : "(direct)")); return; } @@ -841,7 +841,7 @@ void StorageBuffer::flushBuffer(Buffer & buffer, bool check_thresholds, bool loc } UInt64 milliseconds = watch.elapsedMilliseconds(); - LOG_TRACE(log, "Flushing buffer with {} rows, {} bytes, age {} seconds, took {} ms {}.", rows, bytes, time_passed, milliseconds, (check_thresholds ? "(bg)" : "(direct)")); + LOG_DEBUG(log, "Flushing buffer with {} rows, {} bytes, age {} seconds, took {} ms {}.", rows, bytes, time_passed, milliseconds, (check_thresholds ? "(bg)" : "(direct)")); } From 19e04396295af4a15e6dddf674aaf16c6e285395 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 12 Apr 2021 09:04:38 +0300 Subject: [PATCH 65/76] Add ability to flush buffer only in background for StorageBuffer Add 3 new engine arguments: - flush_time - flush_rows - flush_bytes That will be checked only for background flush, this maybe useful if INSERT latency is "crucial". --- .../engines/table-engines/special/buffer.md | 12 ++- src/Common/ProfileEvents.cpp | 3 + src/Storages/StorageBuffer.cpp | 80 ++++++++++++++----- src/Storages/StorageBuffer.h | 17 ++-- ..._storage_buffer_flush_parameters.reference | 1 + .../01811_storage_buffer_flush_parameters.sql | 22 +++++ .../01817_storage_buffer_parameters.reference | 0 .../01817_storage_buffer_parameters.sql | 42 ++++++++++ 8 files changed, 147 insertions(+), 30 deletions(-) create mode 100644 tests/queries/0_stateless/01811_storage_buffer_flush_parameters.reference create mode 100644 tests/queries/0_stateless/01811_storage_buffer_flush_parameters.sql create mode 100644 tests/queries/0_stateless/01817_storage_buffer_parameters.reference create mode 100644 tests/queries/0_stateless/01817_storage_buffer_parameters.sql diff --git a/docs/en/engines/table-engines/special/buffer.md b/docs/en/engines/table-engines/special/buffer.md index bf6c08f8f6c..8245cd19e8c 100644 --- a/docs/en/engines/table-engines/special/buffer.md +++ b/docs/en/engines/table-engines/special/buffer.md @@ -18,11 +18,17 @@ Engine parameters: - `num_layers` – Parallelism layer. Physically, the table will be represented as `num_layers` of independent buffers. Recommended value: 16. - `min_time`, `max_time`, `min_rows`, `max_rows`, `min_bytes`, and `max_bytes` – Conditions for flushing data from the buffer. +Optional engine parameters: + +- `flush_time`, `flush_rows`, `flush_bytes` – Conditions for flushing data from the buffer, that will happen only in background (ommited or zero means no `flush*` parameters). + Data is flushed from the buffer and written to the destination table if all the `min*` conditions or at least one `max*` condition are met. -- `min_time`, `max_time` – Condition for the time in seconds from the moment of the first write to the buffer. -- `min_rows`, `max_rows` – Condition for the number of rows in the buffer. -- `min_bytes`, `max_bytes` – Condition for the number of bytes in the buffer. +Also if at least one `flush*` condition are met flush initiated in background, this is different from `max*`, since `flush*` allows you to configure background flushes separately to avoid adding latency for `INSERT` (into `Buffer`) queries. + +- `min_time`, `max_time`, `flush_time` – Condition for the time in seconds from the moment of the first write to the buffer. +- `min_rows`, `max_rows`, `flush_rows` – Condition for the number of rows in the buffer. +- `min_bytes`, `max_bytes`, `flush_bytes` – Condition for the number of bytes in the buffer. During the write operation, data is inserted to a `num_layers` number of random buffers. Or, if the data part to insert is large enough (greater than `max_rows` or `max_bytes`), it is written directly to the destination table, omitting the buffer. diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index d0876c5e69c..162d6e035cc 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -146,6 +146,9 @@ M(StorageBufferPassedTimeMaxThreshold, "") \ M(StorageBufferPassedRowsMaxThreshold, "") \ M(StorageBufferPassedBytesMaxThreshold, "") \ + M(StorageBufferPassedTimeFlushThreshold, "") \ + M(StorageBufferPassedRowsFlushThreshold, "") \ + M(StorageBufferPassedBytesFlushThreshold, "") \ M(StorageBufferLayerLockReadersWaitMilliseconds, "Time for waiting for Buffer layer during reading") \ M(StorageBufferLayerLockWritersWaitMilliseconds, "Time for waiting free Buffer layer to write to (can be used to tune Buffer layers)") \ \ diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index 5c4b4e7d1d8..7b03622431d 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -40,6 +40,9 @@ namespace ProfileEvents extern const Event StorageBufferPassedTimeMaxThreshold; extern const Event StorageBufferPassedRowsMaxThreshold; extern const Event StorageBufferPassedBytesMaxThreshold; + extern const Event StorageBufferPassedTimeFlushThreshold; + extern const Event StorageBufferPassedRowsFlushThreshold; + extern const Event StorageBufferPassedBytesFlushThreshold; extern const Event StorageBufferLayerLockReadersWaitMilliseconds; extern const Event StorageBufferLayerLockWritersWaitMilliseconds; } @@ -103,6 +106,7 @@ StorageBuffer::StorageBuffer( size_t num_shards_, const Thresholds & min_thresholds_, const Thresholds & max_thresholds_, + const Thresholds & flush_thresholds_, const StorageID & destination_id_, bool allow_materialized_) : IStorage(table_id_) @@ -110,6 +114,7 @@ StorageBuffer::StorageBuffer( , num_shards(num_shards_), buffers(num_shards_) , min_thresholds(min_thresholds_) , max_thresholds(max_thresholds_) + , flush_thresholds(flush_thresholds_) , destination_id(destination_id_) , allow_materialized(allow_materialized_) , log(&Poco::Logger::get("StorageBuffer (" + table_id_.getFullTableName() + ")")) @@ -602,7 +607,7 @@ private: { buffer.data = sorted_block.cloneEmpty(); } - else if (storage.checkThresholds(buffer, current_time, sorted_block.rows(), sorted_block.bytes())) + else if (storage.checkThresholds(buffer, /* direct= */true, current_time, sorted_block.rows(), sorted_block.bytes())) { /** If, after inserting the buffer, the constraints are exceeded, then we will reset the buffer. * This also protects against unlimited consumption of RAM, since if it is impossible to write to the table, @@ -713,7 +718,7 @@ bool StorageBuffer::supportsPrewhere() const return false; } -bool StorageBuffer::checkThresholds(const Buffer & buffer, time_t current_time, size_t additional_rows, size_t additional_bytes) const +bool StorageBuffer::checkThresholds(const Buffer & buffer, bool direct, time_t current_time, size_t additional_rows, size_t additional_bytes) const { time_t time_passed = 0; if (buffer.first_write_time) @@ -722,11 +727,11 @@ bool StorageBuffer::checkThresholds(const Buffer & buffer, time_t current_time, size_t rows = buffer.data.rows() + additional_rows; size_t bytes = buffer.data.bytes() + additional_bytes; - return checkThresholdsImpl(rows, bytes, time_passed); + return checkThresholdsImpl(direct, rows, bytes, time_passed); } -bool StorageBuffer::checkThresholdsImpl(size_t rows, size_t bytes, time_t time_passed) const +bool StorageBuffer::checkThresholdsImpl(bool direct, size_t rows, size_t bytes, time_t time_passed) const { if (time_passed > min_thresholds.time && rows > min_thresholds.rows && bytes > min_thresholds.bytes) { @@ -752,6 +757,27 @@ bool StorageBuffer::checkThresholdsImpl(size_t rows, size_t bytes, time_t time_p return true; } + if (!direct) + { + if (flush_thresholds.time && time_passed > flush_thresholds.time) + { + ProfileEvents::increment(ProfileEvents::StorageBufferPassedTimeFlushThreshold); + return true; + } + + if (flush_thresholds.rows && rows > flush_thresholds.rows) + { + ProfileEvents::increment(ProfileEvents::StorageBufferPassedRowsFlushThreshold); + return true; + } + + if (flush_thresholds.bytes && bytes > flush_thresholds.bytes) + { + ProfileEvents::increment(ProfileEvents::StorageBufferPassedBytesFlushThreshold); + return true; + } + } + return false; } @@ -785,7 +811,7 @@ void StorageBuffer::flushBuffer(Buffer & buffer, bool check_thresholds, bool loc if (check_thresholds) { - if (!checkThresholdsImpl(rows, bytes, time_passed)) + if (!checkThresholdsImpl(/* direct= */false, rows, bytes, time_passed)) return; } else @@ -1040,16 +1066,17 @@ void registerStorageBuffer(StorageFactory & factory) * * db, table - in which table to put data from buffer. * num_buckets - level of parallelism. - * min_time, max_time, min_rows, max_rows, min_bytes, max_bytes - conditions for flushing the buffer. + * min_time, max_time, min_rows, max_rows, min_bytes, max_bytes - conditions for flushing the buffer, + * flush_time, flush_rows, flush_bytes - conditions for flushing. */ factory.registerStorage("Buffer", [](const StorageFactory::Arguments & args) { ASTs & engine_args = args.engine_args; - if (engine_args.size() != 9) - throw Exception("Storage Buffer requires 9 parameters: " - " destination_database, destination_table, num_buckets, min_time, max_time, min_rows, max_rows, min_bytes, max_bytes.", + if (engine_args.size() < 9 || engine_args.size() > 12) + throw Exception("Storage Buffer requires from 9 to 12 parameters: " + " destination_database, destination_table, num_buckets, min_time, max_time, min_rows, max_rows, min_bytes, max_bytes[, flush_time, flush_rows, flush_bytes].", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); // Table and database name arguments accept expressions, evaluate them. @@ -1058,7 +1085,7 @@ void registerStorageBuffer(StorageFactory & factory) // After we evaluated all expressions, check that all arguments are // literals. - for (size_t i = 0; i < 9; i++) + for (size_t i = 0; i < engine_args.size(); i++) { if (!typeid_cast(engine_args[i].get())) { @@ -1068,17 +1095,29 @@ void registerStorageBuffer(StorageFactory & factory) } } - String destination_database = engine_args[0]->as().value.safeGet(); - String destination_table = engine_args[1]->as().value.safeGet(); + size_t i = 0; - UInt64 num_buckets = applyVisitor(FieldVisitorConvertToNumber(), engine_args[2]->as().value); + String destination_database = engine_args[i++]->as().value.safeGet(); + String destination_table = engine_args[i++]->as().value.safeGet(); - Int64 min_time = applyVisitor(FieldVisitorConvertToNumber(), engine_args[3]->as().value); - Int64 max_time = applyVisitor(FieldVisitorConvertToNumber(), engine_args[4]->as().value); - UInt64 min_rows = applyVisitor(FieldVisitorConvertToNumber(), engine_args[5]->as().value); - UInt64 max_rows = applyVisitor(FieldVisitorConvertToNumber(), engine_args[6]->as().value); - UInt64 min_bytes = applyVisitor(FieldVisitorConvertToNumber(), engine_args[7]->as().value); - UInt64 max_bytes = applyVisitor(FieldVisitorConvertToNumber(), engine_args[8]->as().value); + UInt64 num_buckets = applyVisitor(FieldVisitorConvertToNumber(), engine_args[i++]->as().value); + + StorageBuffer::Thresholds min; + StorageBuffer::Thresholds max; + StorageBuffer::Thresholds flush; + + min.time = applyVisitor(FieldVisitorConvertToNumber(), engine_args[i++]->as().value); + max.time = applyVisitor(FieldVisitorConvertToNumber(), engine_args[i++]->as().value); + min.rows = applyVisitor(FieldVisitorConvertToNumber(), engine_args[i++]->as().value); + max.rows = applyVisitor(FieldVisitorConvertToNumber(), engine_args[i++]->as().value); + min.bytes = applyVisitor(FieldVisitorConvertToNumber(), engine_args[i++]->as().value); + max.bytes = applyVisitor(FieldVisitorConvertToNumber(), engine_args[i++]->as().value); + if (engine_args.size() > i) + flush.time = applyVisitor(FieldVisitorConvertToNumber(), engine_args[i++]->as().value); + if (engine_args.size() > i) + flush.rows = applyVisitor(FieldVisitorConvertToNumber(), engine_args[i++]->as().value); + if (engine_args.size() > i) + flush.bytes = applyVisitor(FieldVisitorConvertToNumber(), engine_args[i++]->as().value); /// If destination_id is not set, do not write data from the buffer, but simply empty the buffer. StorageID destination_id = StorageID::createEmpty(); @@ -1094,8 +1133,7 @@ void registerStorageBuffer(StorageFactory & factory) args.constraints, args.getContext(), num_buckets, - StorageBuffer::Thresholds{min_time, min_rows, min_bytes}, - StorageBuffer::Thresholds{max_time, max_rows, max_bytes}, + min, max, flush, destination_id, static_cast(args.getLocalContext()->getSettingsRef().insert_allow_materialized_columns)); }, diff --git a/src/Storages/StorageBuffer.h b/src/Storages/StorageBuffer.h index b29bbf179f4..1747c024a74 100644 --- a/src/Storages/StorageBuffer.h +++ b/src/Storages/StorageBuffer.h @@ -35,6 +35,10 @@ namespace DB * Thresholds can be exceeded. For example, if max_rows = 1 000 000, the buffer already had 500 000 rows, * and a part of 800 000 rows is added, then there will be 1 300 000 rows in the buffer, and then such a block will be written to the subordinate table. * + * There are also separate thresholds for flush, those thresholds are checked only for non-direct flush. + * This maybe useful if you do not want to add extra latency for INSERT queries, + * so you can set max_rows=1e6 and flush_rows=500e3, then each 500e3 rows buffer will be flushed in background only. + * * When you destroy a Buffer table, all remaining data is flushed to the subordinate table. * The data in the buffer is not replicated, not logged to disk, not indexed. With a rough restart of the server, the data is lost. */ @@ -45,12 +49,11 @@ friend class BufferSource; friend class BufferBlockOutputStream; public: - /// Thresholds. struct Thresholds { - time_t time; /// The number of seconds from the insertion of the first row into the block. - size_t rows; /// The number of rows in the block. - size_t bytes; /// The number of (uncompressed) bytes in the block. + time_t time = 0; /// The number of seconds from the insertion of the first row into the block. + size_t rows = 0; /// The number of rows in the block. + size_t bytes = 0; /// The number of (uncompressed) bytes in the block. }; std::string getName() const override { return "Buffer"; } @@ -135,6 +138,7 @@ private: const Thresholds min_thresholds; const Thresholds max_thresholds; + const Thresholds flush_thresholds; StorageID destination_id; bool allow_materialized; @@ -153,8 +157,8 @@ private: /// are exceeded. If reset_block_structure is set - clears inner block /// structure inside buffer (useful in OPTIMIZE and ALTER). void flushBuffer(Buffer & buffer, bool check_thresholds, bool locked = false, bool reset_block_structure = false); - bool checkThresholds(const Buffer & buffer, time_t current_time, size_t additional_rows = 0, size_t additional_bytes = 0) const; - bool checkThresholdsImpl(size_t rows, size_t bytes, time_t time_passed) const; + bool checkThresholds(const Buffer & buffer, bool direct, time_t current_time, size_t additional_rows = 0, size_t additional_bytes = 0) const; + bool checkThresholdsImpl(bool direct, size_t rows, size_t bytes, time_t time_passed) const; /// `table` argument is passed, as it is sometimes evaluated beforehand. It must match the `destination`. void writeBlockToDestination(const Block & block, StoragePtr table); @@ -177,6 +181,7 @@ protected: size_t num_shards_, const Thresholds & min_thresholds_, const Thresholds & max_thresholds_, + const Thresholds & flush_thresholds_, const StorageID & destination_id, bool allow_materialized_); }; diff --git a/tests/queries/0_stateless/01811_storage_buffer_flush_parameters.reference b/tests/queries/0_stateless/01811_storage_buffer_flush_parameters.reference new file mode 100644 index 00000000000..209e3ef4b62 --- /dev/null +++ b/tests/queries/0_stateless/01811_storage_buffer_flush_parameters.reference @@ -0,0 +1 @@ +20 diff --git a/tests/queries/0_stateless/01811_storage_buffer_flush_parameters.sql b/tests/queries/0_stateless/01811_storage_buffer_flush_parameters.sql new file mode 100644 index 00000000000..dac68ad4ae8 --- /dev/null +++ b/tests/queries/0_stateless/01811_storage_buffer_flush_parameters.sql @@ -0,0 +1,22 @@ +drop table if exists data_01811; +drop table if exists buffer_01811; + +create table data_01811 (key Int) Engine=Memory(); +/* Buffer with flush_rows=1000 */ +create table buffer_01811 (key Int) Engine=Buffer(currentDatabase(), data_01811, + /* num_layers= */ 1, + /* min_time= */ 1, /* max_time= */ 86400, + /* min_rows= */ 1e9, /* max_rows= */ 1e6, + /* min_bytes= */ 0, /* max_bytes= */ 4e6, + /* flush_time= */ 86400, /* flush_rows= */ 10, /* flush_bytes= */0 +); + +insert into buffer_01811 select * from numbers(10); +insert into buffer_01811 select * from numbers(10); + +-- wait for background buffer flush +select sleep(3) format Null; +select count() from data_01811; + +drop table buffer_01811; +drop table data_01811; diff --git a/tests/queries/0_stateless/01817_storage_buffer_parameters.reference b/tests/queries/0_stateless/01817_storage_buffer_parameters.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01817_storage_buffer_parameters.sql b/tests/queries/0_stateless/01817_storage_buffer_parameters.sql new file mode 100644 index 00000000000..84727bc5d6b --- /dev/null +++ b/tests/queries/0_stateless/01817_storage_buffer_parameters.sql @@ -0,0 +1,42 @@ +drop table if exists data_01817; +drop table if exists buffer_01817; + +create table data_01817 (key Int) Engine=Null(); + +-- w/ flush_* +create table buffer_01817 (key Int) Engine=Buffer(currentDatabase(), data_01817, + /* num_layers= */ 1, + /* min_time= */ 1, /* max_time= */ 86400, + /* min_rows= */ 1e9, /* max_rows= */ 1e6, + /* min_bytes= */ 0, /* max_bytes= */ 4e6, + /* flush_time= */ 86400, /* flush_rows= */ 10, /* flush_bytes= */0 +); +drop table buffer_01817; + +-- w/o flush_* +create table buffer_01817 (key Int) Engine=Buffer(currentDatabase(), data_01817, + /* num_layers= */ 1, + /* min_time= */ 1, /* max_time= */ 86400, + /* min_rows= */ 1e9, /* max_rows= */ 1e6, + /* min_bytes= */ 0, /* max_bytes= */ 4e6 +); +drop table buffer_01817; + +-- not enough args +create table buffer_01817 (key Int) Engine=Buffer(currentDatabase(), data_01817, + /* num_layers= */ 1, + /* min_time= */ 1, /* max_time= */ 86400, + /* min_rows= */ 1e9, /* max_rows= */ 1e6, + /* min_bytes= */ 0 /* max_bytes= 4e6 */ +); -- { serverError 42 } +-- too much args +create table buffer_01817 (key Int) Engine=Buffer(currentDatabase(), data_01817, + /* num_layers= */ 1, + /* min_time= */ 1, /* max_time= */ 86400, + /* min_rows= */ 1e9, /* max_rows= */ 1e6, + /* min_bytes= */ 0, /* max_bytes= */ 4e6, + /* flush_time= */ 86400, /* flush_rows= */ 10, /* flush_bytes= */0, + 0 +); -- { serverError 42 } + +drop table data_01817; From 0ad6205fa6bd5849dce3071235ba7def6a49233c Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 15 Apr 2021 21:34:53 +0300 Subject: [PATCH 66/76] logs for debuging test failures with Replicated and Keeper --- src/Interpreters/DDLWorker.cpp | 13 ++++++++++++ .../ReplicatedMergeTreeCleanupThread.cpp | 13 +++++++++--- ...0953_zookeeper_suetin_deduplication_bug.sh | 8 +------- .../01305_replica_create_drop_zookeeper.sh | 20 +++++-------------- 4 files changed, 29 insertions(+), 25 deletions(-) diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index 6081f06b25f..1c023f757f8 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -372,7 +372,20 @@ void DDLWorker::scheduleTasks(bool reinitialized) } Strings queue_nodes = zookeeper->getChildren(queue_dir, nullptr, queue_updated_event); + size_t size_before_filtering = queue_nodes.size(); filterAndSortQueueNodes(queue_nodes); + /// The following message is too verbose, but it can be useful too debug mysterious test failures in CI + LOG_TRACE(log, "scheduleTasks: initialized={}, size_before_filtering={}, queue_size={}, " + "entries={}..{}, " + "first_failed_task_name={}, current_tasks_size={}," + "last_current_task={}," + "last_skipped_entry_name={}", + initialized, size_before_filtering, queue_nodes.size(), + queue_nodes.empty() ? "none" : queue_nodes.front(), queue_nodes.empty() ? "none" : queue_nodes.back(), + first_failed_task_name ? *first_failed_task_name : "none", current_tasks.size(), + current_tasks.empty() ? "none" : current_tasks.back()->entry_name, + last_skipped_entry_name ? *last_skipped_entry_name : "none"); + if (max_tasks_in_queue < queue_nodes.size()) cleanup_event->set(); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp index 792a77d5e1a..502c6215a9a 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp @@ -342,6 +342,15 @@ void ReplicatedMergeTreeCleanupThread::clearOldBlocks() timed_blocks.begin(), timed_blocks.end(), block_threshold, NodeWithStat::greaterByTime); auto first_outdated_block = std::min(first_outdated_block_fixed_threshold, first_outdated_block_time_threshold); + auto num_nodes_to_delete = timed_blocks.end() - first_outdated_block; + if (!num_nodes_to_delete) + return; + + auto last_outdated_block = timed_blocks.end() - 1; + LOG_TRACE(log, "Will clear {} old blocks from {} (ctime {}) to {} (ctime {})", num_nodes_to_delete, + first_outdated_block->node, first_outdated_block->ctime, + last_outdated_block->node, last_outdated_block->ctime); + zkutil::AsyncResponses try_remove_futures; for (auto it = first_outdated_block; it != timed_blocks.end(); ++it) { @@ -372,9 +381,7 @@ void ReplicatedMergeTreeCleanupThread::clearOldBlocks() first_outdated_block++; } - auto num_nodes_to_delete = timed_blocks.end() - first_outdated_block; - if (num_nodes_to_delete) - LOG_TRACE(log, "Cleared {} old blocks from ZooKeeper", num_nodes_to_delete); + LOG_TRACE(log, "Cleared {} old blocks from ZooKeeper", num_nodes_to_delete); } diff --git a/tests/queries/0_stateless/00953_zookeeper_suetin_deduplication_bug.sh b/tests/queries/0_stateless/00953_zookeeper_suetin_deduplication_bug.sh index baa2b0cf53f..71ca29bfd96 100755 --- a/tests/queries/0_stateless/00953_zookeeper_suetin_deduplication_bug.sh +++ b/tests/queries/0_stateless/00953_zookeeper_suetin_deduplication_bug.sh @@ -21,15 +21,12 @@ ORDER BY (engine_id) SETTINGS replicated_deduplication_window = 2, cleanup_delay_period=4, cleanup_delay_period_random_add=0;" $CLICKHOUSE_CLIENT --query="INSERT INTO elog VALUES (toDate('2018-10-01'), 1, 'hello')" -sleep 1 $CLICKHOUSE_CLIENT --query="INSERT INTO elog VALUES (toDate('2018-10-01'), 2, 'hello')" -sleep 1 $CLICKHOUSE_CLIENT --query="INSERT INTO elog VALUES (toDate('2018-10-01'), 3, 'hello')" $CLICKHOUSE_CLIENT --query="SELECT count(*) from elog" # 3 rows count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/s1/blocks'") - while [[ $count != 2 ]] do sleep 1 @@ -39,9 +36,8 @@ done $CLICKHOUSE_CLIENT --query="INSERT INTO elog VALUES (toDate('2018-10-01'), 1, 'hello')" $CLICKHOUSE_CLIENT --query="SELECT count(*) from elog" # 4 rows + count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/s1/blocks'") - - while [[ $count != 2 ]] do sleep 1 @@ -53,12 +49,10 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO elog VALUES (toDate('2018-10-01'), 2, 'h $CLICKHOUSE_CLIENT --query="SELECT count(*) from elog" # 5 rows count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/s1/blocks'") - while [[ $count != 2 ]] do sleep 1 count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/s1/blocks'") - done $CLICKHOUSE_CLIENT --query="INSERT INTO elog VALUES (toDate('2018-10-01'), 2, 'hello')" diff --git a/tests/queries/0_stateless/01305_replica_create_drop_zookeeper.sh b/tests/queries/0_stateless/01305_replica_create_drop_zookeeper.sh index 01bb9af461c..e7b8091284a 100755 --- a/tests/queries/0_stateless/01305_replica_create_drop_zookeeper.sh +++ b/tests/queries/0_stateless/01305_replica_create_drop_zookeeper.sh @@ -8,21 +8,11 @@ set -e function thread() { - db_engine=`$CLICKHOUSE_CLIENT -q "SELECT engine FROM system.databases WHERE name='$CLICKHOUSE_DATABASE'"` - if [[ $db_engine == "Atomic" ]]; then - # Ignore "Replica already exists" exception - while true; do - $CLICKHOUSE_CLIENT -n -q "DROP TABLE IF EXISTS test_table_$1 NO DELAY; - CREATE TABLE test_table_$1 (a UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_table', 'r_$1') ORDER BY tuple();" 2>&1 | - grep -vP '(^$)|(^Received exception from server)|(^\d+\. )|because the last replica of the table was dropped right now|is already started to be removing by another replica right now|is already finished removing by another replica right now|Removing leftovers from table|Another replica was suddenly created|was successfully removed from ZooKeeper|was created by another server at the same moment|was suddenly removed|some other replicas were created at the same time|already exists' - done - else - while true; do - $CLICKHOUSE_CLIENT -n -q "DROP TABLE IF EXISTS test_table_$1; - CREATE TABLE test_table_$1 (a UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_table', 'r_$1') ORDER BY tuple();" 2>&1 | - grep -vP '(^$)|(^Received exception from server)|(^\d+\. )|because the last replica of the table was dropped right now|is already started to be removing by another replica right now|is already finished removing by another replica right now|Removing leftovers from table|Another replica was suddenly created|was successfully removed from ZooKeeper|was created by another server at the same moment|was suddenly removed|some other replicas were created at the same time' - done - fi + while true; do + $CLICKHOUSE_CLIENT -n -q "DROP TABLE IF EXISTS test_table_$1 SYNC; + CREATE TABLE test_table_$1 (a UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_table', 'r_$1') ORDER BY tuple();" 2>&1 | + grep -vP '(^$)|(^Received exception from server)|(^\d+\. )|because the last replica of the table was dropped right now|is already started to be removing by another replica right now|is already finished removing by another replica right now|Removing leftovers from table|Another replica was suddenly created|was successfully removed from ZooKeeper|was created by another server at the same moment|was suddenly removed|some other replicas were created at the same time' + done } From 09571ca91fca2f525d2e5b3a8cea61abece6f61a Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Thu, 15 Apr 2021 22:22:40 +0300 Subject: [PATCH 67/76] Updated zlib-ng submodule --- contrib/zlib-ng | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/zlib-ng b/contrib/zlib-ng index bf128f84df0..527425a08cb 160000 --- a/contrib/zlib-ng +++ b/contrib/zlib-ng @@ -1 +1 @@ -Subproject commit bf128f84df0806ec51c3513804222ae02007c4f3 +Subproject commit 527425a08cbdce33acff00eaf8f83d6bdb6b29ae From 4affe01ffbc3280bd45ea3a1d06a7bc6e45cce3a Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Thu, 15 Apr 2021 23:30:41 +0300 Subject: [PATCH 68/76] Backport zlib x86 arm check features constructor --- contrib/zlib-ng | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/zlib-ng b/contrib/zlib-ng index 527425a08cb..16b42c7a030 160000 --- a/contrib/zlib-ng +++ b/contrib/zlib-ng @@ -1 +1 @@ -Subproject commit 527425a08cbdce33acff00eaf8f83d6bdb6b29ae +Subproject commit 16b42c7a03097ca3df67d90246a0d9bf826734e1 From 9110a76d00807baf9dc368cd613512298188269b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 16 Apr 2021 00:14:37 +0300 Subject: [PATCH 69/76] Reordered settings to avoid confusion --- src/Core/Settings.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index d31073ae932..ff58f5d4e5f 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -252,8 +252,6 @@ class IColumn; * Almost all limits apply to each stream individually. \ */ \ \ - M(UInt64, limit, 0, "Limit on read rows from the most 'end' result for select query, default 0 means no limit length", 0) \ - M(UInt64, offset, 0, "Offset on read rows from the most 'end' result for select query", 0) \ M(UInt64, max_rows_to_read, 0, "Limit on read rows from the most 'deep' sources. That is, only in the deepest subquery. When reading from a remote server, it is only checked on a remote server.", 0) \ M(UInt64, max_bytes_to_read, 0, "Limit on read bytes (after decompression) from the most 'deep' sources. That is, only in the deepest subquery. When reading from a remote server, it is only checked on a remote server.", 0) \ M(OverflowMode, read_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.", 0) \ @@ -464,6 +462,8 @@ class IColumn; \ M(Bool, database_replicated_ddl_output, true, "Obsolete setting, does nothing. Will be removed after 2021-09-08", 0) \ M(HandleKafkaErrorMode, handle_kafka_error_mode, HandleKafkaErrorMode::DEFAULT, "How to handle errors for Kafka engine. Passible values: default, stream.", 0) \ + M(UInt64, limit, 0, "Limit on read rows from the most 'end' result for select query, default 0 means no limit length", 0) \ + M(UInt64, offset, 0, "Offset on read rows from the most 'end' result for select query", 0) \ // End of COMMON_SETTINGS // Please add settings related to formats into the FORMAT_FACTORY_SETTINGS below. From 55b1fc5a21a92de64bbb525111375fd2f66648c2 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Fri, 16 Apr 2021 00:18:52 +0300 Subject: [PATCH 70/76] Updated zlib-ng --- contrib/zlib-ng | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/zlib-ng b/contrib/zlib-ng index 16b42c7a030..28dfdaa8a3c 160000 --- a/contrib/zlib-ng +++ b/contrib/zlib-ng @@ -1 +1 @@ -Subproject commit 16b42c7a03097ca3df67d90246a0d9bf826734e1 +Subproject commit 28dfdaa8a3c5add48dcaf56086a9306a357e6e6b From 9238d8e54aea2eb2a497d314f10f870f6f9cba5d Mon Sep 17 00:00:00 2001 From: madianjun Date: Fri, 16 Apr 2021 14:28:52 +0800 Subject: [PATCH 71/76] Fix exception message for parts_to_throw_insert --- src/Storages/MergeTree/MergeTreeData.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index ee8e15008cb..2008edb8919 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -2563,7 +2563,7 @@ void MergeTreeData::delayInsertOrThrowIfNeeded(Poco::Event * until) const ProfileEvents::increment(ProfileEvents::RejectedInserts); throw Exception( ErrorCodes::TOO_MANY_PARTS, - "Too many parts ({}). Parts cleaning are processing significantly slower than inserts", + "Too many parts ({}). Merges are processing significantly slower than inserts", parts_count_in_partition); } From 75d18a6d278ac519cefe44515c9b4dce5f00d95e Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Fri, 16 Apr 2021 10:45:53 +0300 Subject: [PATCH 72/76] Updated zlib-ng aarch64 --- contrib/zlib-ng | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/zlib-ng b/contrib/zlib-ng index 28dfdaa8a3c..4039bb46239 160000 --- a/contrib/zlib-ng +++ b/contrib/zlib-ng @@ -1 +1 @@ -Subproject commit 28dfdaa8a3c5add48dcaf56086a9306a357e6e6b +Subproject commit 4039bb4623905e73c6e32a0c022f144bab87b2b3 From 29281ea6e062294652210d3a3011ce0998956f28 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 16 Apr 2021 10:54:11 +0300 Subject: [PATCH 73/76] jemalloc: set muzzy_decay_ms/dirty_decay_ms to 5s --- contrib/jemalloc-cmake/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt index a82345975d1..830a280465c 100644 --- a/contrib/jemalloc-cmake/CMakeLists.txt +++ b/contrib/jemalloc-cmake/CMakeLists.txt @@ -34,9 +34,9 @@ if (OS_LINUX) # avoid spurious latencies and additional work associated with # MADV_DONTNEED. See # https://github.com/ClickHouse/ClickHouse/issues/11121 for motivation. - set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:1000,dirty_decay_ms:1000") + set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:5000,dirty_decay_ms:5000") else() - set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0,muzzy_decay_ms:1000,dirty_decay_ms:1000") + set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0,muzzy_decay_ms:5000,dirty_decay_ms:5000") endif() # CACHE variable is empty, to allow changing defaults without necessity # to purge cache From 88e2d28666e7d614d9187121fef550f20a1ede5b Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Fri, 16 Apr 2021 00:16:35 +0300 Subject: [PATCH 74/76] Improve documentation for CREATE ROW POLICY command. --- .../statements/create/row-policy.md | 71 ++++++++++++++----- .../statements/create/row-policy.md | 65 +++++++++++++---- 2 files changed, 105 insertions(+), 31 deletions(-) diff --git a/docs/en/sql-reference/statements/create/row-policy.md b/docs/en/sql-reference/statements/create/row-policy.md index cbe639c6fc5..6f769fb1dca 100644 --- a/docs/en/sql-reference/statements/create/row-policy.md +++ b/docs/en/sql-reference/statements/create/row-policy.md @@ -5,39 +5,78 @@ toc_title: ROW POLICY # CREATE ROW POLICY {#create-row-policy-statement} -Creates [filters for rows](../../../operations/access-rights.md#row-policy-management), which a user can read from a table. +Creates a [row policy](../../../operations/access-rights.md#row-policy-management), i.e. a filter used to determine which rows a user can read from a table. Syntax: ``` sql CREATE [ROW] POLICY [IF NOT EXISTS | OR REPLACE] policy_name1 [ON CLUSTER cluster_name1] ON [db1.]table1 [, policy_name2 [ON CLUSTER cluster_name2] ON [db2.]table2 ...] + [FOR SELECT] USING condition [AS {PERMISSIVE | RESTRICTIVE}] - [FOR SELECT] - [USING condition] [TO {role1 [, role2 ...] | ALL | ALL EXCEPT role1 [, role2 ...]}] ``` `ON CLUSTER` clause allows creating row policies on a cluster, see [Distributed DDL](../../../sql-reference/distributed-ddl.md). -## AS Clause {#create-row-policy-as} +## USING Clause {#create-row-policy-using} -Using this section you can create permissive or restrictive policies. - -Permissive policy grants access to rows. Permissive policies which apply to the same table are combined together using the boolean `OR` operator. Policies are permissive by default. - -Restrictive policy restricts access to rows. Restrictive policies which apply to the same table are combined together using the boolean `AND` operator. - -Restrictive policies apply to rows that passed the permissive filters. If you set restrictive policies but no permissive policies, the user can’t get any row from the table. +Allows to specify a condition to filter rows. An user will see a row if the condition is calculated to non-zero for the row. ## TO Clause {#create-row-policy-to} -In the section `TO` you can provide a mixed list of roles and users, for example, `CREATE ROW POLICY ... TO accountant, john@localhost`. +In the section `TO` you can provide a list of users and roles this policy should work for. For example, `CREATE ROW POLICY ... TO accountant, john@localhost`. -Keyword `ALL` means all the ClickHouse users including current user. Keywords `ALL EXCEPT` allow to exclude some users from the all users list, for example, `CREATE ROW POLICY ... TO ALL EXCEPT accountant, john@localhost` +Keyword `ALL` means all the ClickHouse users including current user. Keyword `ALL EXCEPT` allow to exclude some users from the all users list, for example, `CREATE ROW POLICY ... TO ALL EXCEPT accountant, john@localhost` -## Examples {#examples} +!!! note "Note" + If there are no row policies defined for a table then any user can `SELECT` all the row from the table. + Defining one or more row policies for the table makes the access to the table depending on the row policies no matter if + those row policies are defined for the current user or not. For example, the following row policy -`CREATE ROW POLICY filter ON mydb.mytable FOR SELECT USING a<1000 TO accountant, john@localhost` + `CREATE ROW POLICY pol1 ON mydb.table1 USING b=1 TO mira, peter` -`CREATE ROW POLICY filter ON mydb.mytable FOR SELECT USING a<1000 TO ALL EXCEPT mira` + forbids the users `mira` and `peter` to see the rows with `b != 1`, and any non-mentioned user (e.g., the user `paul`) will see no rows from `mydb.table1` at all! If that isn't desirable you can fix it by adding one more row policy, for example: + + `CREATE ROW POLICY pol2 ON mydb.table1 USING 1 TO ALL EXCEPT mira, peter` + +## AS Clause {#create-row-policy-as} + +It's allowed to have more than one policy enabled on the same table for the same user at the one time. +So we need a way to combine the conditions from multiple policies. +By default policies are combined using the boolean `OR` operator. For example, the following policies + +``` sql +CREATE ROW POLICY pol1 ON mydb.table1 USING b=1 TO mira, peter +CREATE ROW POLICY pol2 ON mydb.table1 USING c=2 TO peter, antonio +``` + +enables the user `peter` to see rows with either `b=1` or `c=2`. + +The `AS` clause specifies how policies should be combined with other policies. Policies can be either permissive or restrictive. +By default policies are permissive, which means they are combined using the boolean `OR` operator. + +A policy can be defined as restrictive as an alternative. Restrictive policies are combined using the boolean `AND` operator. +Here is the formula: + +``` +row_is_visible = (one or more of the permissive policies' conditions are non-zero) AND (all of the restrictive policies's conditions are non-zero)` +``` + +For example, the following policies + +``` sql +CREATE ROW POLICY pol1 ON mydb.table1 USING b=1 TO mira, peter +CREATE ROW POLICY pol2 ON mydb.table1 USING c=2 AS RESTRICTIVE TO peter, antonio +``` + +enables the user `peter` to see rows only if both `b=1` AND `c=2`. + + +## Examples + +`CREATE ROW POLICY filter1 ON mydb.mytable USING a<1000 TO accountant, john@localhost` + +`CREATE ROW POLICY filter2 ON mydb.mytable USING a<1000 AND b=5 TO ALL EXCEPT mira` + +`CREATE ROW POLICY filter3 ON mydb.mytable USING 1 TO admin` diff --git a/docs/ru/sql-reference/statements/create/row-policy.md b/docs/ru/sql-reference/statements/create/row-policy.md index 88709598906..95fa29ff48a 100644 --- a/docs/ru/sql-reference/statements/create/row-policy.md +++ b/docs/ru/sql-reference/statements/create/row-policy.md @@ -5,7 +5,7 @@ toc_title: "Политика доступа" # CREATE ROW POLICY {#create-row-policy-statement} -Создает [фильтры для строк](../../../operations/access-rights.md#row-policy-management), которые пользователь может прочесть из таблицы. +Создает [политики доступа к строкам](../../../operations/access-rights.md#row-policy-management), т.е. фильтры, которые определяют, какие строки пользователь может читать из таблицы. Синтаксис: @@ -13,33 +13,68 @@ toc_title: "Политика доступа" CREATE [ROW] POLICY [IF NOT EXISTS | OR REPLACE] policy_name1 [ON CLUSTER cluster_name1] ON [db1.]table1 [, policy_name2 [ON CLUSTER cluster_name2] ON [db2.]table2 ...] [AS {PERMISSIVE | RESTRICTIVE}] - [FOR SELECT] - [USING condition] + [FOR SELECT] USING condition [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] ``` -Секция `ON CLUSTER` позволяет создавать фильтры для строк на кластере, см. [Распределенные DDL запросы](../../../sql-reference/distributed-ddl.md). +Секция `ON CLUSTER` позволяет создавать политики на кластере, см. [Распределенные DDL запросы](../../../sql-reference/distributed-ddl.md). -## Секция AS {#create-row-policy-as} +## USING Clause {#create-row-policy-using} -С помощью данной секции можно создать политику разрешения или ограничения. - -Политика разрешения предоставляет доступ к строкам. Разрешительные политики, которые применяются к одной таблице, объединяются с помощью логического оператора `OR`. Политики являются разрешительными по умолчанию. - -Политика ограничения запрещает доступ к строкам. Ограничительные политики, которые применяются к одной таблице, объединяются логическим оператором `AND`. - -Ограничительные политики применяются к строкам, прошедшим фильтр разрешительной политики. Если вы не зададите разрешительные политики, пользователь не сможет обращаться ни к каким строкам из таблицы. +Секция `USING` указывает условие для фильтрации строк. Пользователь может видеть строку, если это условие, вычисленное для строки, дает ненулевой результат. ## Секция TO {#create-row-policy-to} -В секции `TO` вы можете перечислить как роли, так и пользователей. Например, `CREATE ROW POLICY ... TO accountant, john@localhost`. +В секции `TO` перечисляются пользователи и роли, для которых должна действовать политика. Например, `CREATE ROW POLICY ... TO accountant, john@localhost`. Ключевым словом `ALL` обозначаются все пользователи, включая текущего. Ключевые слова `ALL EXCEPT` позволяют исключить пользователей из списка всех пользователей. Например, `CREATE ROW POLICY ... TO ALL EXCEPT accountant, john@localhost` +!!! note "Note" + Если для таблицы не задано ни одной политики доступа к строкам, то любой пользователь может выполнить `SELECT` и получить все строки таблицы. + Если определить хотя бы одну политику для таблицы, до доступ к строкам будет управляться этими политиками, причем для всех пользователей + (даже для тех, для кого политики не определялись). Например, следующая политика + + `CREATE ROW POLICY pol1 ON mydb.table1 USING b=1 TO mira, peter` + + запретит пользователям `mira` и `peter` видеть строки с `b != 1`, и еще запретит всем остальным пользователям (например, пользователю `paul`) + видеть какие-либо строки вообще из таблицы `mydb.table1`! Если это нежелательно, такое поведение можно исправить, определив дополнительную политику: + + `CREATE ROW POLICY pol2 ON mydb.table1 USING 1 TO ALL EXCEPT mira, peter` + +## Секция AS {#create-row-policy-as} + +Может быть одновременно активно более одной политики для одной и той же таблицы и одного и того же пользователя. +Поэтому нам нужен способ комбинировать политики. По умолчанию политики комбинируются с использованием логического оператора `OR`. +Например, политики: + +``` sql +CREATE ROW POLICY pol1 ON mydb.table1 USING b=1 TO mira, peter +CREATE ROW POLICY pol2 ON mydb.table1 USING c=2 TO peter, antonio +``` + +разрешат пользователю с именем `peter` видеть строки, для которых будет верно `b=1` или `c=2`. + +Секция `AS` указывает, как политики должны комбинироваться с другими политиками. Политики могут быть или разрешительными (`PERMISSIVE`), или ограничительными (`RESTRICTIVE`). По умолчанию политики создаются разрешительными (`PERMISSIVE`); такие политики комбинируются с использованием логического оператора `OR`. +Ограничительные (`RESTRICTIVE`) политики комбинируются с использованием логического оператора `AND`. +Используется следующая формула: + +`строка_видима = (одна или больше permissive-политик дала ненулевой результат проверки условия) И (все restrictive-политики дали ненулевой результат проверки условия)` + +Например, политики + +``` sql +CREATE ROW POLICY pol1 ON mydb.table1 USING b=1 TO mira, peter +CREATE ROW POLICY pol2 ON mydb.table1 USING c=2 AS RESTRICTIVE TO peter, antonio +``` + +разрешат пользователю с именем `peter` видеть только те строки, для которых будет одновременно `b=1` и `c=2`. + ## Примеры -`CREATE ROW POLICY filter ON mydb.mytable FOR SELECT USING a<1000 TO accountant, john@localhost` +`CREATE ROW POLICY filter1 ON mydb.mytable USING a<1000 TO accountant, john@localhost` -`CREATE ROW POLICY filter ON mydb.mytable FOR SELECT USING a<1000 TO ALL EXCEPT mira` +`CREATE ROW POLICY filter2 ON mydb.mytable USING a<1000 AND b=5 TO ALL EXCEPT mira` + +`CREATE ROW POLICY filter3 ON mydb.mytable USING 1 TO admin` \ No newline at end of file From 4d323fa556677a769d588aba4aafa76900b134b4 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Fri, 16 Apr 2021 13:59:54 +0300 Subject: [PATCH 75/76] MacOS clang build instructions fix --- docs/en/development/build-osx.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/development/build-osx.md b/docs/en/development/build-osx.md index 886e85bbf86..29fdf0f324c 100644 --- a/docs/en/development/build-osx.md +++ b/docs/en/development/build-osx.md @@ -62,7 +62,7 @@ $ cd ClickHouse $ rm -rf build $ mkdir build $ cd build -$ cmake -DCMAKE_C_COMPILER=$(brew --prefix llvm)/bin/clang -DCMAKE_CXX_COMPILER==$(brew --prefix llvm)/bin/clang++ -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_JEMALLOC=OFF .. +$ cmake -DCMAKE_C_COMPILER=$(brew --prefix llvm)/bin/clang -DCMAKE_CXX_COMPILER=$(brew --prefix llvm)/bin/clang++ -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_JEMALLOC=OFF .. $ cmake --build . --config RelWithDebInfo $ cd .. ``` From 440efb6fb684b7890bac0d37977ce4d55b9b02b7 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Fri, 16 Apr 2021 14:22:23 +0300 Subject: [PATCH 76/76] Update arrayFold.cpp --- src/Functions/array/arrayFold.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Functions/array/arrayFold.cpp b/src/Functions/array/arrayFold.cpp index 5fc7a304b03..21a228929ad 100644 --- a/src/Functions/array/arrayFold.cpp +++ b/src/Functions/array/arrayFold.cpp @@ -138,7 +138,8 @@ public: size_t arr_cursor = 0; for (size_t irow = 0; irow < column_first_array->size(); ++irow) // for each row of result { - // Make accumulator column for this row + // Make accumulator column for this row. We initialize it + // with the starting value given as the last argument. ColumnWithTypeAndName accumulator_column = arguments.back(); ColumnPtr acc(accumulator_column.column->cut(irow, 1)); auto accumulator = ColumnWithTypeAndName(acc,