From b914c4262df43a9bcc901324f4271f9c756a6135 Mon Sep 17 00:00:00 2001 From: liyang Date: Fri, 17 Jan 2020 21:52:02 +0800 Subject: [PATCH 001/712] add array function auc --- dbms/src/Functions/array/arrayScalarProduct.h | 157 ++++++++++++++++++ dbms/src/Functions/array/auc.cpp | 97 +++++++++++ .../array/registerFunctionsArray.cpp | 3 +- .../0_stateless/01064_array_auc.reference | 1 + .../queries/0_stateless/01064_array_auc.sql | 1 + .../functions/array_functions.md | 6 + 6 files changed, 264 insertions(+), 1 deletion(-) create mode 100644 dbms/src/Functions/array/arrayScalarProduct.h create mode 100644 dbms/src/Functions/array/auc.cpp create mode 100644 dbms/tests/queries/0_stateless/01064_array_auc.reference create mode 100644 dbms/tests/queries/0_stateless/01064_array_auc.sql diff --git a/dbms/src/Functions/array/arrayScalarProduct.h b/dbms/src/Functions/array/arrayScalarProduct.h new file mode 100644 index 00000000000..9d25f551483 --- /dev/null +++ b/dbms/src/Functions/array/arrayScalarProduct.h @@ -0,0 +1,157 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; + extern const int ILLEGAL_COLUMN; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; +} + +template +class FunctionArrayScalarProduct : public IFunction +{ +public: + static constexpr auto name = Name::name; + static FunctionPtr create(const Context & context) { return std::make_shared(context); } + FunctionArrayScalarProduct(const Context & context_): context(context_) {} + +private: + using ResultColumnType = ColumnVector; + + template + bool executeNumber(Block & block, const ColumnNumbers & arguments, size_t result) + { + return executeNumberNumber(block, arguments, result) + || executeNumberNumber(block, arguments, result) + || executeNumberNumber(block, arguments, result) + || executeNumberNumber(block, arguments, result) + || executeNumberNumber(block, arguments, result) + || executeNumberNumber(block, arguments, result) + || executeNumberNumber(block, arguments, result) + || executeNumberNumber(block, arguments, result) + || executeNumberNumber(block, arguments, result) + || executeNumberNumber(block, arguments, result); + } + + + template + bool executeNumberNumber(Block & block, const ColumnNumbers & arguments, size_t result) + { + ColumnPtr col1 = block.getByPosition(arguments[0]).column->convertToFullColumnIfConst(); + ColumnPtr col2 = block.getByPosition(arguments[1]).column->convertToFullColumnIfConst(); + if (! col1 || ! col2) + return false; + + const ColumnArray* col_array1 = checkAndGetColumn(col1.get()); + const ColumnArray* col_array2 = checkAndGetColumn(col2.get()); + if (! col_array1 || ! col_array2) + return false; + + const ColumnVector * col_nested1 = checkAndGetColumn>(col_array1->getData()); + const ColumnVector * col_nested2 = checkAndGetColumn>(col_array2->getData()); + if (! col_nested1 || ! col_nested2) + return false; + + auto col_res = ResultColumnType::create(); + vector(col_nested1->getData(), col_array1->getOffsets(), + col_nested2->getData(), col_array2->getOffsets(), col_res->getData()); + block.getByPosition(result).column = std::move(col_res); + return true; + } + + template + static void vector(const PaddedPODArray & data1, const ColumnArray::Offsets & offsets1, + const PaddedPODArray & data2, const ColumnArray::Offsets & offsets2, + PaddedPODArray & result) + { + size_t size = offsets1.size(); + result.resize(size); + + ColumnArray::Offset current_offset1 = 0; + ColumnArray::Offset current_offset2 = 0; + for (size_t i = 0; i < size; ++i) { + size_t array1_size = offsets1[i] - current_offset1; + size_t array2_size = offsets2[i] - current_offset2; + result[i] = Method::apply(data1, current_offset1, array1_size, data2, current_offset2, array2_size); + + current_offset1 = offsets1[i]; + current_offset2 = offsets2[i]; + } + } + +public: + /// Get function name. + String getName() const override + { + return name; + } + + size_t getNumberOfArguments() const override { return 2; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + // Basic type check + std::vector nested_types(2, nullptr); + for (size_t i = 0; i < getNumberOfArguments(); ++i) + { + const DataTypeArray * array_type = checkAndGetDataType(arguments[i].get()); + if (!array_type) + throw Exception("All argument for function " + getName() + " must be an array.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + auto & nested_type = array_type->getNestedType(); + WhichDataType which(nested_type); + bool is_number = which.isNativeInt() || which.isNativeUInt() || which.isFloat(); + if (! is_number) + { + throw Exception(getName() + " cannot process values of type " + nested_type->getName(), + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + nested_types[i] = nested_type; + } + + // Detail type check in Method, then return ReturnType + return Method::getReturnType(nested_types[0], nested_types[1]); + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /* input_rows_count */) override + { + if (!(executeNumber(block, arguments, result) + || executeNumber(block, arguments, result) + || executeNumber(block, arguments, result) + || executeNumber(block, arguments, result) + || executeNumber(block, arguments, result) + || executeNumber(block, arguments, result) + || executeNumber(block, arguments, result) + || executeNumber(block, arguments, result) + || executeNumber(block, arguments, result) + || executeNumber(block, arguments, result))) + throw Exception{"Illegal column " + block.getByPosition(arguments[0]).column->getName() + + " of first argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN}; + + } + +private: + const Context & context; +}; + +} \ No newline at end of file diff --git a/dbms/src/Functions/array/auc.cpp b/dbms/src/Functions/array/auc.cpp new file mode 100644 index 00000000000..2ac7d85a501 --- /dev/null +++ b/dbms/src/Functions/array/auc.cpp @@ -0,0 +1,97 @@ +#include +#include +#include +#include "arrayScalarProduct.h" + +namespace DB +{ + +struct NameAUC { static constexpr auto name = "auc"; }; + +class AUCImpl +{ +public: + using ResultType = Float64; + + struct ScoreLabel + { + ResultType score; + UInt8 label; + }; + + static DataTypePtr getReturnType(const DataTypePtr & /* nested_type1 */, const DataTypePtr & nested_type2) + { + WhichDataType which2(nested_type2); + if (! which2.isUInt8()) { + throw Exception(std::string(NameAUC::name) + "lable type must be UInt8", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + return std::make_shared>(); + } + + template + static ResultType apply(const PaddedPODArray & scores, ColumnArray::Offset score_offset, size_t score_len, + const PaddedPODArray & labels, ColumnArray::Offset label_offset, size_t label_len) + { + if (score_len != label_len) + throw Exception{"Unmatched length of arrays in " + std::string(NameAUC::name), + ErrorCodes::LOGICAL_ERROR}; + if (score_len == 0) + return {}; + + // Order pairs of score and lable by score ascending + size_t num_pos = 0; + size_t num_neg = 0; + std::vector pairs(score_len); + for (size_t i = 0; i < score_len; ++i) + { + pairs[i].score = scores[i + score_offset]; + pairs[i].label = (labels[i + label_offset] ? 1 : 0); + if (pairs[i].label) + ++num_pos; + else + ++num_neg; + } + std::sort(pairs.begin(), pairs.end(), + [](const auto & lhs, const auto & rhs) {return lhs.score < rhs.score; }); + + // Calculate AUC + size_t curr_cnt = 0; + size_t curr_pos_cnt = 0; + size_t curr_sum = 0; + ResultType last_score = -1; + ResultType rank_sum = 0; + for (size_t i = 0; i < pairs.size(); ++i) + { + if (pairs[i].score == last_score) + { + curr_sum += i + 1; + ++curr_cnt; + if (pairs[i].label) + ++curr_pos_cnt; + } + else + { + if (i > 0) + rank_sum += ResultType(curr_sum * curr_pos_cnt) / curr_cnt; + curr_sum = i + 1; + curr_cnt = 1; + curr_pos_cnt = pairs[i].label ? 1 : 0; + } + last_score = pairs[i].score; + } + rank_sum += ResultType(curr_sum * curr_pos_cnt) / curr_cnt; + return (rank_sum - num_pos*(num_pos+1)/2)/(num_pos * num_neg); + } +}; + +/// auc(array_score, array_label) - Calculate AUC with array of score and label +using FunctionAUC = FunctionArrayScalarProduct; + +void registerFunctionAUC(FunctionFactory & factory) +{ + factory.registerFunction(); +} + + +} \ No newline at end of file diff --git a/dbms/src/Functions/array/registerFunctionsArray.cpp b/dbms/src/Functions/array/registerFunctionsArray.cpp index ababc7603e3..96f19f547c0 100644 --- a/dbms/src/Functions/array/registerFunctionsArray.cpp +++ b/dbms/src/Functions/array/registerFunctionsArray.cpp @@ -33,7 +33,7 @@ void registerFunctionArrayDistinct(FunctionFactory & factory); void registerFunctionArrayFlatten(FunctionFactory & factory); void registerFunctionArrayWithConstant(FunctionFactory & factory); void registerFunctionArrayZip(FunctionFactory & factory); - +void registerFunctionAUC(FunctionFactory &); void registerFunctionsArray(FunctionFactory & factory) { @@ -67,6 +67,7 @@ void registerFunctionsArray(FunctionFactory & factory) registerFunctionArrayFlatten(factory); registerFunctionArrayWithConstant(factory); registerFunctionArrayZip(factory); + registerFunctionAUC(factory); } } diff --git a/dbms/tests/queries/0_stateless/01064_array_auc.reference b/dbms/tests/queries/0_stateless/01064_array_auc.reference new file mode 100644 index 00000000000..d6e1fa0f619 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01064_array_auc.reference @@ -0,0 +1 @@ +0.75 \ No newline at end of file diff --git a/dbms/tests/queries/0_stateless/01064_array_auc.sql b/dbms/tests/queries/0_stateless/01064_array_auc.sql new file mode 100644 index 00000000000..33b54e7db45 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01064_array_auc.sql @@ -0,0 +1 @@ +select select auc([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1]) \ No newline at end of file diff --git a/docs/en/query_language/functions/array_functions.md b/docs/en/query_language/functions/array_functions.md index 6d5d9ccd6bf..64f631a84e4 100644 --- a/docs/en/query_language/functions/array_functions.md +++ b/docs/en/query_language/functions/array_functions.md @@ -896,5 +896,11 @@ Result: │ [('a','d'),('b','e'),('c','f')] │ └────────────────────────────────────────────┘ ``` +## auc(arr_scores, arr_labels) + +Returns AUC(Area Under the Curve, which is a concept in machine learning, see more details: https://developers.google.com/machine-learning/crash-course/classification/roc-and-auc); + +`arr_scores` represents scores prediction model gives, while `arr_labels` represents labels of samples, usually 1 for positive sample and 0 for negtive sample. + [Original article](https://clickhouse.yandex/docs/en/query_language/functions/array_functions/) From 1dab2b756d08c4a4e0f0d8ba7bc2d0a21dce52f5 Mon Sep 17 00:00:00 2001 From: liyang Date: Sat, 18 Jan 2020 12:16:34 +0800 Subject: [PATCH 002/712] fix code style --- dbms/src/Functions/array/arrayScalarProduct.h | 104 ++++++++---------- dbms/src/Functions/array/auc.cpp | 53 +++++---- 2 files changed, 75 insertions(+), 82 deletions(-) diff --git a/dbms/src/Functions/array/arrayScalarProduct.h b/dbms/src/Functions/array/arrayScalarProduct.h index 9d25f551483..82e73b8c17a 100644 --- a/dbms/src/Functions/array/arrayScalarProduct.h +++ b/dbms/src/Functions/array/arrayScalarProduct.h @@ -1,24 +1,23 @@ -#include -#include -#include +#include +#include +#include +#include +#include #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include #include +#include +#include +#include +#include namespace DB { - namespace ErrorCodes { extern const int LOGICAL_ERROR; @@ -32,7 +31,7 @@ class FunctionArrayScalarProduct : public IFunction public: static constexpr auto name = Name::name; static FunctionPtr create(const Context & context) { return std::make_shared(context); } - FunctionArrayScalarProduct(const Context & context_): context(context_) {} + FunctionArrayScalarProduct(const Context & context_) : context(context_) {} private: using ResultColumnType = ColumnVector; @@ -40,16 +39,11 @@ private: template bool executeNumber(Block & block, const ColumnNumbers & arguments, size_t result) { - return executeNumberNumber(block, arguments, result) - || executeNumberNumber(block, arguments, result) - || executeNumberNumber(block, arguments, result) - || executeNumberNumber(block, arguments, result) - || executeNumberNumber(block, arguments, result) - || executeNumberNumber(block, arguments, result) - || executeNumberNumber(block, arguments, result) - || executeNumberNumber(block, arguments, result) - || executeNumberNumber(block, arguments, result) - || executeNumberNumber(block, arguments, result); + return executeNumberNumber(block, arguments, result) || executeNumberNumber(block, arguments, result) + || executeNumberNumber(block, arguments, result) || executeNumberNumber(block, arguments, result) + || executeNumberNumber(block, arguments, result) || executeNumberNumber(block, arguments, result) + || executeNumberNumber(block, arguments, result) || executeNumberNumber(block, arguments, result) + || executeNumberNumber(block, arguments, result) || executeNumberNumber(block, arguments, result); } @@ -58,29 +52,31 @@ private: { ColumnPtr col1 = block.getByPosition(arguments[0]).column->convertToFullColumnIfConst(); ColumnPtr col2 = block.getByPosition(arguments[1]).column->convertToFullColumnIfConst(); - if (! col1 || ! col2) + if (!col1 || !col2) return false; - const ColumnArray* col_array1 = checkAndGetColumn(col1.get()); - const ColumnArray* col_array2 = checkAndGetColumn(col2.get()); - if (! col_array1 || ! col_array2) + const ColumnArray * col_array1 = checkAndGetColumn(col1.get()); + const ColumnArray * col_array2 = checkAndGetColumn(col2.get()); + if (!col_array1 || !col_array2) return false; const ColumnVector * col_nested1 = checkAndGetColumn>(col_array1->getData()); const ColumnVector * col_nested2 = checkAndGetColumn>(col_array2->getData()); - if (! col_nested1 || ! col_nested2) + if (!col_nested1 || !col_nested2) return false; auto col_res = ResultColumnType::create(); - vector(col_nested1->getData(), col_array1->getOffsets(), - col_nested2->getData(), col_array2->getOffsets(), col_res->getData()); + vector(col_nested1->getData(), col_array1->getOffsets(), col_nested2->getData(), col_array2->getOffsets(), col_res->getData()); block.getByPosition(result).column = std::move(col_res); return true; } template - static void vector(const PaddedPODArray & data1, const ColumnArray::Offsets & offsets1, - const PaddedPODArray & data2, const ColumnArray::Offsets & offsets2, + static void vector( + const PaddedPODArray & data1, + const ColumnArray::Offsets & offsets1, + const PaddedPODArray & data2, + const ColumnArray::Offsets & offsets2, PaddedPODArray & result) { size_t size = offsets1.size(); @@ -88,7 +84,8 @@ private: ColumnArray::Offset current_offset1 = 0; ColumnArray::Offset current_offset2 = 0; - for (size_t i = 0; i < size; ++i) { + for (size_t i = 0; i < size; ++i) + { size_t array1_size = offsets1[i] - current_offset1; size_t array2_size = offsets2[i] - current_offset2; result[i] = Method::apply(data1, current_offset1, array1_size, data2, current_offset2, array2_size); @@ -100,10 +97,7 @@ private: public: /// Get function name. - String getName() const override - { - return name; - } + String getName() const override { return name; } size_t getNumberOfArguments() const override { return 2; } @@ -111,20 +105,19 @@ public: { // Basic type check std::vector nested_types(2, nullptr); - for (size_t i = 0; i < getNumberOfArguments(); ++i) + for (size_t i = 0; i < getNumberOfArguments(); ++i) { const DataTypeArray * array_type = checkAndGetDataType(arguments[i].get()); if (!array_type) - throw Exception("All argument for function " + getName() + " must be an array.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - + throw Exception("All argument for function " + getName() + " must be an array.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + auto & nested_type = array_type->getNestedType(); WhichDataType which(nested_type); bool is_number = which.isNativeInt() || which.isNativeUInt() || which.isFloat(); - if (! is_number) + if (!is_number) { - throw Exception(getName() + " cannot process values of type " + nested_type->getName(), - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception( + getName() + " cannot process values of type " + nested_type->getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } nested_types[i] = nested_type; } @@ -135,19 +128,14 @@ public: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /* input_rows_count */) override { - if (!(executeNumber(block, arguments, result) - || executeNumber(block, arguments, result) - || executeNumber(block, arguments, result) - || executeNumber(block, arguments, result) - || executeNumber(block, arguments, result) - || executeNumber(block, arguments, result) - || executeNumber(block, arguments, result) - || executeNumber(block, arguments, result) - || executeNumber(block, arguments, result) - || executeNumber(block, arguments, result))) - throw Exception{"Illegal column " + block.getByPosition(arguments[0]).column->getName() - + " of first argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN}; - + if (!(executeNumber(block, arguments, result) || executeNumber(block, arguments, result) + || executeNumber(block, arguments, result) || executeNumber(block, arguments, result) + || executeNumber(block, arguments, result) || executeNumber(block, arguments, result) + || executeNumber(block, arguments, result) || executeNumber(block, arguments, result) + || executeNumber(block, arguments, result) || executeNumber(block, arguments, result))) + throw Exception{"Illegal column " + block.getByPosition(arguments[0]).column->getName() + " of first argument of function " + + getName(), + ErrorCodes::ILLEGAL_COLUMN}; } private: diff --git a/dbms/src/Functions/array/auc.cpp b/dbms/src/Functions/array/auc.cpp index 2ac7d85a501..7091452ea0b 100644 --- a/dbms/src/Functions/array/auc.cpp +++ b/dbms/src/Functions/array/auc.cpp @@ -1,19 +1,21 @@ -#include #include +#include #include #include "arrayScalarProduct.h" namespace DB { +struct NameAUC +{ + static constexpr auto name = "auc"; +}; -struct NameAUC { static constexpr auto name = "auc"; }; - -class AUCImpl +class AUCImpl { public: using ResultType = Float64; - struct ScoreLabel + struct ScoreLabel { ResultType score; UInt8 label; @@ -22,38 +24,41 @@ public: static DataTypePtr getReturnType(const DataTypePtr & /* nested_type1 */, const DataTypePtr & nested_type2) { WhichDataType which2(nested_type2); - if (! which2.isUInt8()) { - throw Exception(std::string(NameAUC::name) + "lable type must be UInt8", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + if (!which2.isUInt8()) + { + throw Exception(std::string(NameAUC::name) + "lable type must be UInt8", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } return std::make_shared>(); } template - static ResultType apply(const PaddedPODArray & scores, ColumnArray::Offset score_offset, size_t score_len, - const PaddedPODArray & labels, ColumnArray::Offset label_offset, size_t label_len) + static ResultType apply( + const PaddedPODArray & scores, + ColumnArray::Offset score_offset, + size_t score_len, + const PaddedPODArray & labels, + ColumnArray::Offset label_offset, + size_t label_len) { - if (score_len != label_len) - throw Exception{"Unmatched length of arrays in " + std::string(NameAUC::name), - ErrorCodes::LOGICAL_ERROR}; - if (score_len == 0) + if (score_len != label_len) + throw Exception{"Unmatched length of arrays in " + std::string(NameAUC::name), ErrorCodes::LOGICAL_ERROR}; + if (score_len == 0) return {}; // Order pairs of score and lable by score ascending size_t num_pos = 0; size_t num_neg = 0; std::vector pairs(score_len); - for (size_t i = 0; i < score_len; ++i) + for (size_t i = 0; i < score_len; ++i) { pairs[i].score = scores[i + score_offset]; pairs[i].label = (labels[i + label_offset] ? 1 : 0); - if (pairs[i].label) + if (pairs[i].label) ++num_pos; - else + else ++num_neg; } - std::sort(pairs.begin(), pairs.end(), - [](const auto & lhs, const auto & rhs) {return lhs.score < rhs.score; }); + std::sort(pairs.begin(), pairs.end(), [](const auto & lhs, const auto & rhs) { return lhs.score < rhs.score; }); // Calculate AUC size_t curr_cnt = 0; @@ -61,18 +66,18 @@ public: size_t curr_sum = 0; ResultType last_score = -1; ResultType rank_sum = 0; - for (size_t i = 0; i < pairs.size(); ++i) + for (size_t i = 0; i < pairs.size(); ++i) { if (pairs[i].score == last_score) { curr_sum += i + 1; ++curr_cnt; - if (pairs[i].label) + if (pairs[i].label) ++curr_pos_cnt; } - else + else { - if (i > 0) + if (i > 0) rank_sum += ResultType(curr_sum * curr_pos_cnt) / curr_cnt; curr_sum = i + 1; curr_cnt = 1; @@ -81,7 +86,7 @@ public: last_score = pairs[i].score; } rank_sum += ResultType(curr_sum * curr_pos_cnt) / curr_cnt; - return (rank_sum - num_pos*(num_pos+1)/2)/(num_pos * num_neg); + return (rank_sum - num_pos * (num_pos + 1) / 2) / (num_pos * num_neg); } }; From 07f2d88ed4230ecb724fc4c3ba57b1910e677fcf Mon Sep 17 00:00:00 2001 From: liyang Date: Sat, 18 Jan 2020 14:46:06 +0800 Subject: [PATCH 003/712] fix bug --- dbms/tests/queries/0_stateless/01064_array_auc.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/tests/queries/0_stateless/01064_array_auc.sql b/dbms/tests/queries/0_stateless/01064_array_auc.sql index 33b54e7db45..8886426f098 100644 --- a/dbms/tests/queries/0_stateless/01064_array_auc.sql +++ b/dbms/tests/queries/0_stateless/01064_array_auc.sql @@ -1 +1 @@ -select select auc([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1]) \ No newline at end of file +select auc([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1]) From c66fc0c8b58af73237c7fae99b96dc93cd86b32b Mon Sep 17 00:00:00 2001 From: liyang Date: Sat, 18 Jan 2020 18:03:14 +0800 Subject: [PATCH 004/712] fix bug --- dbms/src/Functions/array/arrayScalarProduct.h | 2 +- dbms/src/Functions/array/auc.cpp | 2 +- dbms/tests/queries/0_stateless/01064_array_auc.reference | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dbms/src/Functions/array/arrayScalarProduct.h b/dbms/src/Functions/array/arrayScalarProduct.h index 82e73b8c17a..c450f41f48a 100644 --- a/dbms/src/Functions/array/arrayScalarProduct.h +++ b/dbms/src/Functions/array/arrayScalarProduct.h @@ -142,4 +142,4 @@ private: const Context & context; }; -} \ No newline at end of file +} diff --git a/dbms/src/Functions/array/auc.cpp b/dbms/src/Functions/array/auc.cpp index 7091452ea0b..7a2daf0fdd0 100644 --- a/dbms/src/Functions/array/auc.cpp +++ b/dbms/src/Functions/array/auc.cpp @@ -99,4 +99,4 @@ void registerFunctionAUC(FunctionFactory & factory) } -} \ No newline at end of file +} diff --git a/dbms/tests/queries/0_stateless/01064_array_auc.reference b/dbms/tests/queries/0_stateless/01064_array_auc.reference index d6e1fa0f619..39c64a9f26e 100644 --- a/dbms/tests/queries/0_stateless/01064_array_auc.reference +++ b/dbms/tests/queries/0_stateless/01064_array_auc.reference @@ -1 +1 @@ -0.75 \ No newline at end of file +0.75 From 2fdd211635d3d6d7800ce0cdd63d6b70d7ba26b8 Mon Sep 17 00:00:00 2001 From: liyang Date: Tue, 21 Jan 2020 16:32:35 +0800 Subject: [PATCH 005/712] rename auc to arrayAUC --- .../src/Functions/array/auc.cpp => array_auc.cpp | 16 ++++++++-------- .../queries/0_stateless/01064_array_auc.sql | 2 +- .../query_language/functions/array_functions.md | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) rename dbms/src/Functions/array/auc.cpp => array_auc.cpp (84%) diff --git a/dbms/src/Functions/array/auc.cpp b/array_auc.cpp similarity index 84% rename from dbms/src/Functions/array/auc.cpp rename to array_auc.cpp index 7a2daf0fdd0..41739446e46 100644 --- a/dbms/src/Functions/array/auc.cpp +++ b/array_auc.cpp @@ -5,12 +5,12 @@ namespace DB { -struct NameAUC +struct NameArrayAUC { - static constexpr auto name = "auc"; + static constexpr auto name = "arrayAUC"; }; -class AUCImpl +class ArrayAUCImpl { public: using ResultType = Float64; @@ -26,7 +26,7 @@ public: WhichDataType which2(nested_type2); if (!which2.isUInt8()) { - throw Exception(std::string(NameAUC::name) + "lable type must be UInt8", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception(std::string(NameArrayAUC::name) + "lable type must be UInt8", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } return std::make_shared>(); } @@ -41,7 +41,7 @@ public: size_t label_len) { if (score_len != label_len) - throw Exception{"Unmatched length of arrays in " + std::string(NameAUC::name), ErrorCodes::LOGICAL_ERROR}; + throw Exception{"Unmatched length of arrays in " + std::string(NameArrayAUC::name), ErrorCodes::LOGICAL_ERROR}; if (score_len == 0) return {}; @@ -91,11 +91,11 @@ public: }; /// auc(array_score, array_label) - Calculate AUC with array of score and label -using FunctionAUC = FunctionArrayScalarProduct; +using FunctionArrayAUC = FunctionArrayScalarProduct; -void registerFunctionAUC(FunctionFactory & factory) +void registerFunctionArrayAUC(FunctionFactory & factory) { - factory.registerFunction(); + factory.registerFunction(); } diff --git a/dbms/tests/queries/0_stateless/01064_array_auc.sql b/dbms/tests/queries/0_stateless/01064_array_auc.sql index 8886426f098..ca270937f63 100644 --- a/dbms/tests/queries/0_stateless/01064_array_auc.sql +++ b/dbms/tests/queries/0_stateless/01064_array_auc.sql @@ -1 +1 @@ -select auc([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1]) +select arrayAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1]) diff --git a/docs/en/query_language/functions/array_functions.md b/docs/en/query_language/functions/array_functions.md index 64f631a84e4..8c9b2a7c151 100644 --- a/docs/en/query_language/functions/array_functions.md +++ b/docs/en/query_language/functions/array_functions.md @@ -896,7 +896,7 @@ Result: │ [('a','d'),('b','e'),('c','f')] │ └────────────────────────────────────────────┘ ``` -## auc(arr_scores, arr_labels) +## arrayAUC(arr_scores, arr_labels) Returns AUC(Area Under the Curve, which is a concept in machine learning, see more details: https://developers.google.com/machine-learning/crash-course/classification/roc-and-auc); From b9076ef247b7765e2839c170debe6e10f4f202b4 Mon Sep 17 00:00:00 2001 From: liyang Date: Tue, 21 Jan 2020 16:59:09 +0800 Subject: [PATCH 006/712] mv file --- array_auc.cpp => dbms/src/Functions/array/array_auc.cpp | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename array_auc.cpp => dbms/src/Functions/array/array_auc.cpp (100%) diff --git a/array_auc.cpp b/dbms/src/Functions/array/array_auc.cpp similarity index 100% rename from array_auc.cpp rename to dbms/src/Functions/array/array_auc.cpp From 3d9f49c0baff29d6d43d332cfb42e8d6c44d5e01 Mon Sep 17 00:00:00 2001 From: liyang Date: Tue, 21 Jan 2020 17:03:35 +0800 Subject: [PATCH 007/712] commitagin --- dbms/src/Functions/array/registerFunctionsArray.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/dbms/src/Functions/array/registerFunctionsArray.cpp b/dbms/src/Functions/array/registerFunctionsArray.cpp index 96f19f547c0..cc360d80979 100644 --- a/dbms/src/Functions/array/registerFunctionsArray.cpp +++ b/dbms/src/Functions/array/registerFunctionsArray.cpp @@ -1,6 +1,5 @@ namespace DB { - class FunctionFactory; void registerFunctionArray(FunctionFactory & factory); @@ -33,7 +32,7 @@ void registerFunctionArrayDistinct(FunctionFactory & factory); void registerFunctionArrayFlatten(FunctionFactory & factory); void registerFunctionArrayWithConstant(FunctionFactory & factory); void registerFunctionArrayZip(FunctionFactory & factory); -void registerFunctionAUC(FunctionFactory &); +void registerFunctionArrayAUC(FunctionFactory &); void registerFunctionsArray(FunctionFactory & factory) { @@ -67,8 +66,7 @@ void registerFunctionsArray(FunctionFactory & factory) registerFunctionArrayFlatten(factory); registerFunctionArrayWithConstant(factory); registerFunctionArrayZip(factory); - registerFunctionAUC(factory); + registerFunctionArrayAUC(factory); } } - From b470ff7aaedd10c24752d68fa7899d52f258980a Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 3 Feb 2020 06:11:12 +0300 Subject: [PATCH 008/712] Update array_functions.md --- docs/en/query_language/functions/array_functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/query_language/functions/array_functions.md b/docs/en/query_language/functions/array_functions.md index 8c9b2a7c151..35aa9c98dfb 100644 --- a/docs/en/query_language/functions/array_functions.md +++ b/docs/en/query_language/functions/array_functions.md @@ -898,7 +898,7 @@ Result: ``` ## arrayAUC(arr_scores, arr_labels) -Returns AUC(Area Under the Curve, which is a concept in machine learning, see more details: https://developers.google.com/machine-learning/crash-course/classification/roc-and-auc); +Returns AUC (Area Under the Curve, which is a concept in machine learning, see more details: https://developers.google.com/machine-learning/crash-course/classification/roc-and-auc). `arr_scores` represents scores prediction model gives, while `arr_labels` represents labels of samples, usually 1 for positive sample and 0 for negtive sample. From 71d46bb5219e620126347643708990dda8a25f0e Mon Sep 17 00:00:00 2001 From: liyang Date: Sat, 8 Feb 2020 14:31:03 +0800 Subject: [PATCH 009/712] modify to suggestions --- dbms/src/Functions/array/arrayScalarProduct.h | 5 +-- dbms/src/Functions/array/array_auc.cpp | 43 ++++++++++++++++--- .../functions/array_functions.md | 28 ++++++++++-- 3 files changed, 63 insertions(+), 13 deletions(-) diff --git a/dbms/src/Functions/array/arrayScalarProduct.h b/dbms/src/Functions/array/arrayScalarProduct.h index c450f41f48a..87f618b5b4f 100644 --- a/dbms/src/Functions/array/arrayScalarProduct.h +++ b/dbms/src/Functions/array/arrayScalarProduct.h @@ -109,11 +109,10 @@ public: { const DataTypeArray * array_type = checkAndGetDataType(arguments[i].get()); if (!array_type) - throw Exception("All argument for function " + getName() + " must be an array.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception("All arguments for function " + getName() + " must be an array.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); auto & nested_type = array_type->getNestedType(); - WhichDataType which(nested_type); - bool is_number = which.isNativeInt() || which.isNativeUInt() || which.isFloat(); + bool is_number = isNativeNumber(nested_type) if (!is_number) { throw Exception( diff --git a/dbms/src/Functions/array/array_auc.cpp b/dbms/src/Functions/array/array_auc.cpp index 41739446e46..861b617e3ea 100644 --- a/dbms/src/Functions/array/array_auc.cpp +++ b/dbms/src/Functions/array/array_auc.cpp @@ -18,16 +18,35 @@ public: struct ScoreLabel { ResultType score; - UInt8 label; + bool label; }; - static DataTypePtr getReturnType(const DataTypePtr & /* nested_type1 */, const DataTypePtr & nested_type2) + static DataTypePtr getReturnType(const DataTypePtr & /* score_type */, const DataTypePtr & label_type) { - WhichDataType which2(nested_type2); - if (!which2.isUInt8()) + WhichDataType which(label_type); + // Labels values are either {0, 1} or {-1, 1}, and its type must be one of (Enum8, UInt8, Int8) + if (!which.isUInt8() && !which.isEnum8() && !which.isInt8()) { throw Exception(std::string(NameArrayAUC::name) + "lable type must be UInt8", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } + + if (which.isEnum8()) + { + auto type8 = checkAndGetDataType(label_type.get()); + if (type8) + throw Exception(std::string(NameArrayAUC::name) + "lable type not valid Enum8", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + std::set valSet; + const auto & values = type8->getValues(); + for (const auto & value : values) + { + valSet.insert(value.second); + } + + if (valSet != {0, 1} || valSet != {-1, 1}) + throw Exception( + std::string(NameArrayAUC::name) + "lable values must be {0, 1} or {-1, 1}", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } return std::make_shared>(); } @@ -45,25 +64,35 @@ public: if (score_len == 0) return {}; - // Order pairs of score and lable by score ascending + // Calculate positive and negative label number and restore scores and labels in vector size_t num_pos = 0; size_t num_neg = 0; + std::set labelValSet; std::vector pairs(score_len); for (size_t i = 0; i < score_len; ++i) { pairs[i].score = scores[i + score_offset]; - pairs[i].label = (labels[i + label_offset] ? 1 : 0); + pairs[i].label = (labels[i + label_offset] == 1); if (pairs[i].label) ++num_pos; else ++num_neg; + + labelValSet.insert(labels[i + label_offset]); } + + // Label values must be {0, 1} or {-1, 1} + if (labelValSet != {0, 1} && labelValSet != {-1, 1}) + throw Exception( + std::string(NameArrayAUC::name) + "lable values must be {0, 1} or {-1, 1}", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + // Order pairs of score and lable by score ascending std::sort(pairs.begin(), pairs.end(), [](const auto & lhs, const auto & rhs) { return lhs.score < rhs.score; }); // Calculate AUC size_t curr_cnt = 0; size_t curr_pos_cnt = 0; - size_t curr_sum = 0; + Int64 curr_sum = 0; ResultType last_score = -1; ResultType rank_sum = 0; for (size_t i = 0; i < pairs.size(); ++i) diff --git a/docs/en/query_language/functions/array_functions.md b/docs/en/query_language/functions/array_functions.md index 35aa9c98dfb..3574f37a509 100644 --- a/docs/en/query_language/functions/array_functions.md +++ b/docs/en/query_language/functions/array_functions.md @@ -896,11 +896,33 @@ Result: │ [('a','d'),('b','e'),('c','f')] │ └────────────────────────────────────────────┘ ``` -## arrayAUC(arr_scores, arr_labels) +## arrayAUC {#arrayauc} +Calculate AUC(Area Under the Curve, which is a concept in machine learning, see more details: https://en.wikipedia.org/wiki/Receiver_operating_characteristic#Area_under_the_curve). -Returns AUC (Area Under the Curve, which is a concept in machine learning, see more details: https://developers.google.com/machine-learning/crash-course/classification/roc-and-auc). +**Syntax** +```sql +arrayAUC(arr_scores, arr_labels) +``` -`arr_scores` represents scores prediction model gives, while `arr_labels` represents labels of samples, usually 1 for positive sample and 0 for negtive sample. +**Parameters** +- `arr_scores` — scores prediction model gives. +- `arr_labels` — labels of samples, usually 1 for positive sample and 0 for negtive sample. +**Returned value** +return AUC value with type Float64. + +**Example** +Query: +```sql +select auc([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1]) +``` + +Result: + +```text +┌─arrayAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1])─┐ +│ 0.75 │ +└────────────────────────────────────────-----──┘ +``` [Original article](https://clickhouse.yandex/docs/en/query_language/functions/array_functions/) From f0f0768e81504e9af675337a0d3d39d1b0b7351f Mon Sep 17 00:00:00 2001 From: liyang Date: Sun, 9 Feb 2020 10:45:58 +0800 Subject: [PATCH 010/712] fix bugs --- dbms/src/Functions/array/arrayScalarProduct.h | 6 ++--- dbms/src/Functions/array/array_auc.cpp | 26 ++++++++++--------- .../0_stateless/01064_array_auc.reference | 15 +++++++++++ .../queries/0_stateless/01064_array_auc.sql | 17 +++++++++++- 4 files changed, 47 insertions(+), 17 deletions(-) diff --git a/dbms/src/Functions/array/arrayScalarProduct.h b/dbms/src/Functions/array/arrayScalarProduct.h index 87f618b5b4f..1352038195b 100644 --- a/dbms/src/Functions/array/arrayScalarProduct.h +++ b/dbms/src/Functions/array/arrayScalarProduct.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -112,12 +113,9 @@ public: throw Exception("All arguments for function " + getName() + " must be an array.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); auto & nested_type = array_type->getNestedType(); - bool is_number = isNativeNumber(nested_type) - if (!is_number) - { + if (!isNativeNumber(nested_type) && !isEnum(nested_type)) throw Exception( getName() + " cannot process values of type " + nested_type->getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - } nested_types[i] = nested_type; } diff --git a/dbms/src/Functions/array/array_auc.cpp b/dbms/src/Functions/array/array_auc.cpp index 861b617e3ea..3ed9cd275d3 100644 --- a/dbms/src/Functions/array/array_auc.cpp +++ b/dbms/src/Functions/array/array_auc.cpp @@ -14,6 +14,10 @@ class ArrayAUCImpl { public: using ResultType = Float64; + using LabelValueSet = std::set; + using LabelValueSets = std::vector; + + inline static const LabelValueSets expect_label_value_sets = {{0, 1}, {-1, 1}}; struct ScoreLabel { @@ -27,23 +31,22 @@ public: // Labels values are either {0, 1} or {-1, 1}, and its type must be one of (Enum8, UInt8, Int8) if (!which.isUInt8() && !which.isEnum8() && !which.isInt8()) { - throw Exception(std::string(NameArrayAUC::name) + "lable type must be UInt8", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception( + std::string(NameArrayAUC::name) + "lable type must be UInt8, Enum8 or Int8", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } if (which.isEnum8()) { auto type8 = checkAndGetDataType(label_type.get()); - if (type8) + if (!type8) throw Exception(std::string(NameArrayAUC::name) + "lable type not valid Enum8", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - std::set valSet; + LabelValueSet value_set; const auto & values = type8->getValues(); for (const auto & value : values) - { - valSet.insert(value.second); - } + value_set.insert(value.second); - if (valSet != {0, 1} || valSet != {-1, 1}) + if (std::find(expect_label_value_sets.begin(), expect_label_value_sets.end(), value_set) == expect_label_value_sets.end()) throw Exception( std::string(NameArrayAUC::name) + "lable values must be {0, 1} or {-1, 1}", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } @@ -67,7 +70,7 @@ public: // Calculate positive and negative label number and restore scores and labels in vector size_t num_pos = 0; size_t num_neg = 0; - std::set labelValSet; + LabelValueSet label_value_set; std::vector pairs(score_len); for (size_t i = 0; i < score_len; ++i) { @@ -78,11 +81,11 @@ public: else ++num_neg; - labelValSet.insert(labels[i + label_offset]); + label_value_set.insert(labels[i + label_offset]); } // Label values must be {0, 1} or {-1, 1} - if (labelValSet != {0, 1} && labelValSet != {-1, 1}) + if (std::find(expect_label_value_sets.begin(), expect_label_value_sets.end(), label_value_set) == expect_label_value_sets.end()) throw Exception( std::string(NameArrayAUC::name) + "lable values must be {0, 1} or {-1, 1}", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); @@ -119,6 +122,7 @@ public: } }; + /// auc(array_score, array_label) - Calculate AUC with array of score and label using FunctionArrayAUC = FunctionArrayScalarProduct; @@ -126,6 +130,4 @@ void registerFunctionArrayAUC(FunctionFactory & factory) { factory.registerFunction(); } - - } diff --git a/dbms/tests/queries/0_stateless/01064_array_auc.reference b/dbms/tests/queries/0_stateless/01064_array_auc.reference index 39c64a9f26e..8c17bba359a 100644 --- a/dbms/tests/queries/0_stateless/01064_array_auc.reference +++ b/dbms/tests/queries/0_stateless/01064_array_auc.reference @@ -1 +1,16 @@ 0.75 +0.75 +0.75 +0.75 +0.75 +0.75 +0.75 +0.75 +0.75 +0.25 +0.25 +0.25 +0.25 +0.25 +0.125 +0.25 diff --git a/dbms/tests/queries/0_stateless/01064_array_auc.sql b/dbms/tests/queries/0_stateless/01064_array_auc.sql index ca270937f63..de05c47c51b 100644 --- a/dbms/tests/queries/0_stateless/01064_array_auc.sql +++ b/dbms/tests/queries/0_stateless/01064_array_auc.sql @@ -1 +1,16 @@ -select arrayAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1]) +select arrayAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1]); +select arrayAUC([0.1, 0.4, 0.35, 0.8], cast([0, 0, 1, 1] as Array(Int8))); +select arrayAUC([0.1, 0.4, 0.35, 0.8], cast([-1, -1, 1, 1] as Array(Int8))); +select arrayAUC([0.1, 0.4, 0.35, 0.8], cast(['false', 'false', 'true', 'true'] as Array(Enum8('false' = 0, 'true' = 1)))); +select arrayAUC([0.1, 0.4, 0.35, 0.8], cast(['false', 'false', 'true', 'true'] as Array(Enum8('false' = -1, 'true' = 1)))); +select arrayAUC(cast([10, 40, 35, 80] as Array(UInt8)), [0, 0, 1, 1]); +select arrayAUC(cast([10, 40, 35, 80] as Array(UInt16)), [0, 0, 1, 1]); +select arrayAUC(cast([10, 40, 35, 80] as Array(UInt32)), [0, 0, 1, 1]); +select arrayAUC(cast([10, 40, 35, 80] as Array(UInt64)), [0, 0, 1, 1]); +select arrayAUC(cast([-10, -40, -35, -80] as Array(Int8)), [0, 0, 1, 1]); +select arrayAUC(cast([-10, -40, -35, -80] as Array(Int16)), [0, 0, 1, 1]); +select arrayAUC(cast([-10, -40, -35, -80] as Array(Int32)), [0, 0, 1, 1]); +select arrayAUC(cast([-10, -40, -35, -80] as Array(Int64)), [0, 0, 1, 1]); +select arrayAUC(cast([-0.1, -0.4, -0.35, -0.8] as Array(Float32)) , [0, 0, 1, 1]); +select arrayAUC([0, 3, 5, 6, 7.5, 8], [1, 0, 1, 0, 0, 0]); +select arrayAUC([0.1, 0.35, 0.4, 0.8], [1, 0, 1, 0]); \ No newline at end of file From fd87a5bb2d72ce1bf6df1ae7d7f760e936199954 Mon Sep 17 00:00:00 2001 From: liyang Date: Mon, 10 Feb 2020 09:45:30 +0800 Subject: [PATCH 011/712] fix error --- docs/en/query_language/functions/array_functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/query_language/functions/array_functions.md b/docs/en/query_language/functions/array_functions.md index 3574f37a509..101ba8a8b7f 100644 --- a/docs/en/query_language/functions/array_functions.md +++ b/docs/en/query_language/functions/array_functions.md @@ -914,7 +914,7 @@ return AUC value with type Float64. **Example** Query: ```sql -select auc([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1]) +select arrayAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1]) ``` Result: From 93dba03e8115635712136766a98a30374b364134 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 30 Jan 2020 15:51:47 +0300 Subject: [PATCH 012/712] SHOW CREATE for DatabaseMemory --- dbms/src/Databases/DatabaseMemory.cpp | 24 +++++++++++++-- dbms/src/Databases/DatabaseMemory.h | 3 ++ dbms/src/Databases/DatabasesCommon.cpp | 30 ++++++++++++------- dbms/src/Databases/DatabasesCommon.h | 3 ++ .../01069_database_memory.reference | 1 + .../0_stateless/01069_database_memory.sql | 3 ++ 6 files changed, 50 insertions(+), 14 deletions(-) diff --git a/dbms/src/Databases/DatabaseMemory.cpp b/dbms/src/Databases/DatabaseMemory.cpp index 28a6a8a7af0..c54d1abfb8f 100644 --- a/dbms/src/Databases/DatabaseMemory.cpp +++ b/dbms/src/Databases/DatabaseMemory.cpp @@ -7,6 +7,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int UNKNOWN_TABLE; +} + DatabaseMemory::DatabaseMemory(const String & name_) : DatabaseWithOwnTablesBase(name_, "DatabaseMemory(" + name_ + ")") , data_path("data/" + escapeForFileName(database_name) + "/") @@ -16,16 +21,20 @@ void DatabaseMemory::createTable( const Context & /*context*/, const String & table_name, const StoragePtr & table, - const ASTPtr & /*query*/) + const ASTPtr & query) { - attachTable(table_name, table); + std::lock_guard lock{mutex}; + attachTableUnlocked(table_name, table); + create_queries.emplace(table_name, query); } void DatabaseMemory::removeTable( const Context & /*context*/, const String & table_name) { - detachTable(table_name); + std::lock_guard lock{mutex}; + detachTableUnlocked(table_name); + create_queries.erase(table_name); } ASTPtr DatabaseMemory::getCreateDatabaseQuery(const Context & /*context*/) const @@ -37,4 +46,13 @@ ASTPtr DatabaseMemory::getCreateDatabaseQuery(const Context & /*context*/) const return create_query; } +ASTPtr DatabaseMemory::getCreateTableQueryImpl(const Context &, const String & table_name, bool throw_on_error) const +{ + std::lock_guard lock{mutex}; + auto it = create_queries.find(table_name); + if (it == create_queries.end() && throw_on_error) + throw Exception("No table " + table_name + " in database " + database_name, ErrorCodes::UNKNOWN_TABLE); + return it->second; +} + } diff --git a/dbms/src/Databases/DatabaseMemory.h b/dbms/src/Databases/DatabaseMemory.h index 01789c11901..63fa896fefa 100644 --- a/dbms/src/Databases/DatabaseMemory.h +++ b/dbms/src/Databases/DatabaseMemory.h @@ -33,6 +33,7 @@ public: const Context & context, const String & table_name) override; + ASTPtr getCreateTableQueryImpl(const Context & /*context*/, const String & name, bool throw_on_error) const override; ASTPtr getCreateDatabaseQuery(const Context & /*context*/) const override; /// DatabaseMemory allows to create tables, which store data on disk. @@ -44,6 +45,8 @@ public: private: String data_path; + using NameToASTCreate = std::unordered_map; + NameToASTCreate create_queries; }; } diff --git a/dbms/src/Databases/DatabasesCommon.cpp b/dbms/src/Databases/DatabasesCommon.cpp index becdad672a1..756019eccf1 100644 --- a/dbms/src/Databases/DatabasesCommon.cpp +++ b/dbms/src/Databases/DatabasesCommon.cpp @@ -66,18 +66,21 @@ bool DatabaseWithOwnTablesBase::empty(const Context & /*context*/) const StoragePtr DatabaseWithOwnTablesBase::detachTable(const String & table_name) { - StoragePtr res; - { - std::lock_guard lock(mutex); - if (dictionaries.count(table_name)) - throw Exception("Cannot detach dictionary " + database_name + "." + table_name + " as table, use DETACH DICTIONARY query.", ErrorCodes::UNKNOWN_TABLE); + std::lock_guard lock(mutex); + return detachTableUnlocked(table_name); +} - auto it = tables.find(table_name); - if (it == tables.end()) - throw Exception("Table " + backQuote(database_name) + "." + backQuote(table_name) + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE); - res = it->second; - tables.erase(it); - } +StoragePtr DatabaseWithOwnTablesBase::detachTableUnlocked(const String & table_name) +{ + StoragePtr res; + if (dictionaries.count(table_name)) + throw Exception("Cannot detach dictionary " + database_name + "." + table_name + " as table, use DETACH DICTIONARY query.", ErrorCodes::UNKNOWN_TABLE); + + auto it = tables.find(table_name); + if (it == tables.end()) + throw Exception("Table " + backQuote(database_name) + "." + backQuote(table_name) + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE); + res = it->second; + tables.erase(it); return res; } @@ -85,6 +88,11 @@ StoragePtr DatabaseWithOwnTablesBase::detachTable(const String & table_name) void DatabaseWithOwnTablesBase::attachTable(const String & table_name, const StoragePtr & table) { std::lock_guard lock(mutex); + attachTableUnlocked(table_name, table); +} + +void DatabaseWithOwnTablesBase::attachTableUnlocked(const String & table_name, const StoragePtr & table) +{ if (!tables.emplace(table_name, table).second) throw Exception("Table " + database_name + "." + table_name + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS); } diff --git a/dbms/src/Databases/DatabasesCommon.h b/dbms/src/Databases/DatabasesCommon.h index 291c0e56e8f..1cefb8949bc 100644 --- a/dbms/src/Databases/DatabasesCommon.h +++ b/dbms/src/Databases/DatabasesCommon.h @@ -46,6 +46,9 @@ protected: Poco::Logger * log; DatabaseWithOwnTablesBase(const String & name_, const String & logger); + + void attachTableUnlocked(const String & table_name, const StoragePtr & table); + StoragePtr detachTableUnlocked(const String & table_name); }; } diff --git a/dbms/tests/queries/0_stateless/01069_database_memory.reference b/dbms/tests/queries/0_stateless/01069_database_memory.reference index b2fa2bd204c..35175dd4b62 100644 --- a/dbms/tests/queries/0_stateless/01069_database_memory.reference +++ b/dbms/tests/queries/0_stateless/01069_database_memory.reference @@ -5,3 +5,4 @@ CREATE DATABASE memory_01069 ENGINE = Memory() 4 3 4 +CREATE TABLE memory_01069.file (`n` UInt8) ENGINE = File(CSV) diff --git a/dbms/tests/queries/0_stateless/01069_database_memory.sql b/dbms/tests/queries/0_stateless/01069_database_memory.sql index 645790e3f5e..ed006a5ce04 100644 --- a/dbms/tests/queries/0_stateless/01069_database_memory.sql +++ b/dbms/tests/queries/0_stateless/01069_database_memory.sql @@ -15,4 +15,7 @@ DROP TABLE memory_01069.mt; SELECT * FROM memory_01069.mt ORDER BY n; -- { serverError 60 } SELECT * FROM memory_01069.file ORDER BY n; +SHOW CREATE TABLE memory_01069.mt; -- { serverError 60 } +SHOW CREATE TABLE memory_01069.file; + DROP DATABASE memory_01069; From a508c25d05688cf79fc4dbf6f8fb02b4daafc4f4 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 30 Jan 2020 22:00:51 +0300 Subject: [PATCH 013/712] store temporary tables in DatabaseMemory --- dbms/src/Databases/DatabaseMemory.cpp | 2 +- dbms/src/Databases/DatabaseMemory.h | 3 +- dbms/src/Interpreters/Context.cpp | 115 ++++++++++++++---- dbms/src/Interpreters/Context.h | 9 +- .../Interpreters/InterpreterCreateQuery.cpp | 2 +- .../src/Interpreters/InterpreterDropQuery.cpp | 2 +- .../Interpreters/InterpreterSelectQuery.cpp | 2 +- 7 files changed, 105 insertions(+), 30 deletions(-) diff --git a/dbms/src/Databases/DatabaseMemory.cpp b/dbms/src/Databases/DatabaseMemory.cpp index c54d1abfb8f..c83c91d253c 100644 --- a/dbms/src/Databases/DatabaseMemory.cpp +++ b/dbms/src/Databases/DatabaseMemory.cpp @@ -51,7 +51,7 @@ ASTPtr DatabaseMemory::getCreateTableQueryImpl(const Context &, const String & t std::lock_guard lock{mutex}; auto it = create_queries.find(table_name); if (it == create_queries.end() && throw_on_error) - throw Exception("No table " + table_name + " in database " + database_name, ErrorCodes::UNKNOWN_TABLE); + throw Exception("There is no metadata of table " + table_name + " in database " + database_name, ErrorCodes::UNKNOWN_TABLE); return it->second; } diff --git a/dbms/src/Databases/DatabaseMemory.h b/dbms/src/Databases/DatabaseMemory.h index 63fa896fefa..6a0752ce778 100644 --- a/dbms/src/Databases/DatabaseMemory.h +++ b/dbms/src/Databases/DatabaseMemory.h @@ -19,7 +19,8 @@ namespace DB class DatabaseMemory : public DatabaseWithOwnTablesBase { public: - DatabaseMemory(const String & name_); + //FIXME default name + DatabaseMemory(const String & name_ = "_temporary_and_external_tables"); String getEngineName() const override { return "Memory"; } diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index d290f8dc4f4..7a9013b106b 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -58,6 +58,8 @@ #include #include +#include + namespace ProfileEvents { extern const Event ContextLock; @@ -99,6 +101,57 @@ namespace ErrorCodes extern const int ACCESS_DENIED; } +struct TemporaryTableHolder : boost::noncopyable +{ + static constexpr const char * database_name = "_temporary_and_external_tables"; + + TemporaryTableHolder(const Context & context_, + DatabaseMemory & external_tables_, + const StoragePtr & table, + const ASTPtr & query = {}) + : context(context_), external_tables(external_tables_) + { + if (query) + { + ASTCreateQuery & create = dynamic_cast(*query); + if (create.uuid == UUIDHelpers::Nil) + create.uuid = UUIDHelpers::generateV4(); + id = create.uuid; + } + else + id = UUIDHelpers::generateV4(); + external_tables.createTable(context, "_data_" + toString(id), table, query); + } + + TemporaryTableHolder(TemporaryTableHolder && other) + : context(other.context), external_tables(other.external_tables), id(other.id) + { + other.id = UUIDHelpers::Nil; + } + + ~TemporaryTableHolder() + { + external_tables.removeTable(context, "_data_" + toString(id)); + } + + StorageID getGlobalTableID() const + { + return StorageID{database_name, "_data_" + toString(id), id}; + } + + StoragePtr getTable() const + { + auto table = external_tables.tryGetTable(context, "_data_" + toString(id)); + if (!table) + throw Exception("Temporary table " + getGlobalTableID().getNameForLogs() + " not found", ErrorCodes::LOGICAL_ERROR); + return table; + } + + const Context & context; + DatabaseMemory & external_tables; + UUID id; +}; + /** Set of known objects (environment), that could be used in query. * Shared (global) part. Order of members (especially, order of destruction) is very important. @@ -134,6 +187,8 @@ struct ContextShared mutable VolumePtr tmp_volume; /// Volume for the the temporary files that occur when processing the request. Databases databases; /// List of databases and tables in them. + DatabaseMemory temporary_and_external_tables; + mutable std::optional embedded_dictionaries; /// Metrica's dictionaries. Have lazy initialization. mutable std::optional external_dictionaries_loader; mutable std::optional external_models_loader; @@ -785,9 +840,8 @@ void Context::removeDependency(const StorageID & from, const StorageID & where) Dependencies Context::getDependencies(const StorageID & from) const { auto lock = getLock(); - - String db = resolveDatabase(from.database_name, current_database); - ViewDependencies::const_iterator iter = shared->view_dependencies.find(StorageID(db, from.table_name, from.uuid)); + StorageID resolved = resolveStorageIDUnlocked(from); + ViewDependencies::const_iterator iter = shared->view_dependencies.find(resolved); if (iter == shared->view_dependencies.end()) return {}; @@ -820,7 +874,7 @@ bool Context::isDatabaseExist(const String & database_name) const bool Context::isExternalTableExist(const String & table_name) const { - return external_tables.end() != external_tables.find(table_name); + return external_tables_mapping.count(table_name); } @@ -872,8 +926,8 @@ Tables Context::getExternalTables() const auto lock = getLock(); Tables res; - for (auto & table : external_tables) - res[table.first] = table.second.first; + for (auto & table : external_tables_mapping) + res[table.first] = table.second->getTable(); if (session_context && session_context != this) { @@ -891,11 +945,11 @@ Tables Context::getExternalTables() const StoragePtr Context::tryGetExternalTable(const String & table_name) const { - TableAndCreateASTs::const_iterator jt = external_tables.find(table_name); - if (external_tables.end() == jt) + auto it = external_tables_mapping.find(table_name); + if (external_tables_mapping.end() == it) return StoragePtr(); - return jt->second.first; + return it->second->getTable(); } StoragePtr Context::getTable(const String & database_name, const String & table_name) const @@ -965,10 +1019,11 @@ StoragePtr Context::getTableImpl(const StorageID & table_id, std::optional(*this, shared->temporary_and_external_tables, storage, ast)); } @@ -984,16 +1039,9 @@ bool Context::hasScalar(const String & name) const } -StoragePtr Context::tryRemoveExternalTable(const String & table_name) +bool Context::removeExternalTable(const String & table_name) { - TableAndCreateASTs::const_iterator it = external_tables.find(table_name); - - if (external_tables.end() == it) - return StoragePtr(); - - auto storage = it->second.first; - external_tables.erase(it); - return storage; + return external_tables_mapping.erase(table_name); } @@ -1080,11 +1128,11 @@ DatabasePtr Context::detachDatabase(const String & database_name) ASTPtr Context::getCreateExternalTableQuery(const String & table_name) const { - TableAndCreateASTs::const_iterator jt = external_tables.find(table_name); - if (external_tables.end() == jt) + auto it = external_tables_mapping.find(table_name); + if (external_tables_mapping.end() == it) throw Exception("Temporary table " + backQuoteIfNeed(table_name) + " doesn't exist", ErrorCodes::UNKNOWN_TABLE); - return jt->second.second; + return shared->temporary_and_external_tables.getCreateTableQuery(*this, it->second->getGlobalTableID().table_name); } Settings Context::getSettings() const @@ -2163,6 +2211,27 @@ void Context::resetInputCallbacks() input_blocks_reader = {}; } +StorageID Context::resolveStorageIDUnlocked(StorageID storage_id) const +{ + if (storage_id.uuid != UUIDHelpers::Nil) + { + //TODO maybe update table and db name? + //TODO add flag `resolved` to StorageID and check access rights if it was not previously resolved + return storage_id; + } + if (storage_id.database_name.empty()) + { + auto it = external_tables_mapping.find(storage_id.getTableName()); + if (it != external_tables_mapping.end()) + return it->second->getGlobalTableID(); /// Do not check access rights for session-local table + if (current_database.empty()) + throw Exception("Default database is not selected", ErrorCodes::UNKNOWN_DATABASE); + storage_id.database_name = current_database; + } + checkDatabaseAccessRightsImpl(storage_id.database_name); + return storage_id; +} + SessionCleaner::~SessionCleaner() { diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index 07fa6b06c1f..3e0997db151 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -143,6 +143,8 @@ struct SubscriptionForUserChange SubscriptionForUserChange & operator =(const SubscriptionForUserChange &) { subscription = {}; return *this; } }; +struct TemporaryTableHolder; + /** A set of known objects that can be used in the query. * Consists of a shared part (always common to all sessions and queries) * and copied part (which can be its own for each session or query). @@ -178,7 +180,9 @@ private: String default_format; /// Format, used when server formats data by itself and if query does not have FORMAT specification. /// Thus, used in HTTP interface. If not specified - then some globally default format is used. // TODO maybe replace with DatabaseMemory? - TableAndCreateASTs external_tables; /// Temporary tables. + //TableAndCreateASTs external_tables; /// Temporary tables. + using TemporaryTablesMapping = std::map>; + TemporaryTablesMapping external_tables_mapping; Scalars scalars; StoragePtr view_source; /// Temporary StorageValues used to generate alias columns for materialized views Tables table_function_results; /// Temporary tables obtained by execution of table functions. Keyed by AST tree id. @@ -316,7 +320,8 @@ public: void addExternalTable(const String & table_name, const StoragePtr & storage, const ASTPtr & ast = {}); void addScalar(const String & name, const Block & block); bool hasScalar(const String & name) const; - StoragePtr tryRemoveExternalTable(const String & table_name); + bool removeExternalTable(const String & table_name); + StorageID resolveStorageIDUnlocked(StorageID storage_id) const; StoragePtr executeTableFunction(const ASTPtr & table_expression); diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp index 471f38cfee8..4ca606d5a98 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp @@ -616,7 +616,7 @@ bool InterpreterCreateQuery::doCreateTable(const ASTCreateQuery & create, throw Exception("Table " + create.database + "." + table_name + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS); } } - else if (context.tryGetExternalTable(table_name) && create.if_not_exists) + else if (context.isExternalTableExist(table_name) && create.if_not_exists) return false; StoragePtr res; diff --git a/dbms/src/Interpreters/InterpreterDropQuery.cpp b/dbms/src/Interpreters/InterpreterDropQuery.cpp index c51365ad2ba..94d077abe8c 100644 --- a/dbms/src/Interpreters/InterpreterDropQuery.cpp +++ b/dbms/src/Interpreters/InterpreterDropQuery.cpp @@ -221,7 +221,7 @@ BlockIO InterpreterDropQuery::executeToTemporaryTable(const String & table_name, } else if (kind == ASTDropQuery::Kind::Drop) { - context_handle.tryRemoveExternalTable(table_name); + context_handle.removeExternalTable(table_name); table->shutdown(); /// If table was already dropped by anyone, an exception will be thrown auto table_lock = table->lockExclusively(context.getCurrentQueryId()); diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index ac7ea12d898..29c25115469 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -357,7 +357,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( /// Save the new temporary tables in the query context for (const auto & it : query_analyzer->getExternalTables()) - if (!context->tryGetExternalTable(it.first)) + if (!context->isExternalTableExist(it.first)) context->addExternalTable(it.first, it.second); } From b7197ba531e2249637866ba7d1890be28de30156 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 10 Feb 2020 16:40:42 +0300 Subject: [PATCH 014/712] fix build --- dbms/src/Core/UUID.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/dbms/src/Core/UUID.h b/dbms/src/Core/UUID.h index 4f8fdced814..d5a6232a72b 100644 --- a/dbms/src/Core/UUID.h +++ b/dbms/src/Core/UUID.h @@ -2,10 +2,24 @@ #include #include +#include namespace DB { STRONG_TYPEDEF(UInt128, UUID) +namespace UUIDHelpers +{ + inline UUID generateV4() + { + UInt128 res{thread_local_rng(), thread_local_rng()}; + res.low = (res.low & 0xffffffffffff0fffull) | 0x0000000000004000ull; + res.high = (res.high & 0x3fffffffffffffffull) | 0x8000000000000000ull; + return UUID{res}; + } + + const UUID Nil = UUID(UInt128(0, 0)); +} + } From 4ff7bf78aba768561f444e1aa1bff4309f5995de Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 3 Feb 2020 15:54:36 +0300 Subject: [PATCH 015/712] add DatabaseCatalog --- dbms/programs/local/LocalServer.cpp | 2 + dbms/programs/server/Server.cpp | 2 + dbms/src/Databases/DatabaseMemory.h | 3 +- dbms/src/Interpreters/Context.cpp | 232 ++++++------------ dbms/src/Interpreters/Context.h | 27 +- dbms/src/Interpreters/DatabaseCatalog.cpp | 228 +++++++++++++++++ dbms/src/Interpreters/DatabaseCatalog.h | 91 +++++++ .../InterpreterShowTablesQuery.cpp | 3 +- 8 files changed, 411 insertions(+), 177 deletions(-) create mode 100644 dbms/src/Interpreters/DatabaseCatalog.cpp create mode 100644 dbms/src/Interpreters/DatabaseCatalog.h diff --git a/dbms/programs/local/LocalServer.cpp b/dbms/programs/local/LocalServer.cpp index 5cfceaeb592..e9fca03d7e8 100644 --- a/dbms/programs/local/LocalServer.cpp +++ b/dbms/programs/local/LocalServer.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -200,6 +201,7 @@ try loadMetadataSystem(*context); attachSystemTables(); loadMetadata(*context); + context->getDatabaseCatalog().loadDatabases(); LOG_DEBUG(log, "Loaded metadata."); } else diff --git a/dbms/programs/server/Server.cpp b/dbms/programs/server/Server.cpp index 0cd357bbd94..fc53bf1c462 100644 --- a/dbms/programs/server/Server.cpp +++ b/dbms/programs/server/Server.cpp @@ -41,6 +41,7 @@ #include #include #include +#include #include #include #include @@ -551,6 +552,7 @@ int Server::main(const std::vector & /*args*/) attachSystemTablesServer(*global_context->getDatabase("system"), has_zookeeper); /// Then, load remaining databases loadMetadata(*global_context); + global_context->getDatabaseCatalog().loadDatabases(); } catch (...) { diff --git a/dbms/src/Databases/DatabaseMemory.h b/dbms/src/Databases/DatabaseMemory.h index 6a0752ce778..63fa896fefa 100644 --- a/dbms/src/Databases/DatabaseMemory.h +++ b/dbms/src/Databases/DatabaseMemory.h @@ -19,8 +19,7 @@ namespace DB class DatabaseMemory : public DatabaseWithOwnTablesBase { public: - //FIXME default name - DatabaseMemory(const String & name_ = "_temporary_and_external_tables"); + DatabaseMemory(const String & name_); String getEngineName() const override { return "Memory"; } diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index 7a9013b106b..07d6c001074 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -59,6 +59,7 @@ #include #include +#include namespace ProfileEvents { @@ -103,10 +104,8 @@ namespace ErrorCodes struct TemporaryTableHolder : boost::noncopyable { - static constexpr const char * database_name = "_temporary_and_external_tables"; - TemporaryTableHolder(const Context & context_, - DatabaseMemory & external_tables_, + IDatabase & external_tables_, const StoragePtr & table, const ASTPtr & query = {}) : context(context_), external_tables(external_tables_) @@ -136,7 +135,7 @@ struct TemporaryTableHolder : boost::noncopyable StorageID getGlobalTableID() const { - return StorageID{database_name, "_data_" + toString(id), id}; + return StorageID{DatabaseCatalog::TEMPORARY_DATABASE, "_data_" + toString(id), id}; } StoragePtr getTable() const @@ -148,7 +147,7 @@ struct TemporaryTableHolder : boost::noncopyable } const Context & context; - DatabaseMemory & external_tables; + IDatabase & external_tables; UUID id; }; @@ -186,8 +185,7 @@ struct ContextShared String tmp_path; /// Path to the temporary files that occur when processing the request. mutable VolumePtr tmp_volume; /// Volume for the the temporary files that occur when processing the request. - Databases databases; /// List of databases and tables in them. - DatabaseMemory temporary_and_external_tables; + std::shared_ptr database_catalog; /// Manages databases and tables in them. mutable std::optional embedded_dictionaries; /// Metrica's dictionaries. Have lazy initialization. mutable std::optional external_dictionaries_loader; @@ -319,28 +317,7 @@ struct ContextShared if (system_logs) system_logs->shutdown(); - /** At this point, some tables may have threads that block our mutex. - * To shutdown them correctly, we will copy the current list of tables, - * and ask them all to finish their work. - * Then delete all objects with tables. - */ - - Databases current_databases; - - { - std::lock_guard lock(mutex); - current_databases = databases; - } - - /// We still hold "databases" in Context (instead of std::move) for Buffer tables to flush data correctly. - - for (auto & database : current_databases) - database.second->shutdown(); - - { - std::lock_guard lock(mutex); - databases.clear(); - } + database_catalog->shutdown(); /// Preemptive destruction is important, because these objects may have a refcount to ContextShared (cyclic reference). /// TODO: Get rid of this. @@ -385,6 +362,7 @@ Context Context::createGlobal() res.row_policy = std::make_shared(); res.access_rights = std::make_shared(); res.shared = std::make_shared(); + res.shared->database_catalog = std::make_shared(res); return res; } @@ -406,19 +384,6 @@ MergeList & Context::getMergeList() { return shared->merge_list; } const MergeList & Context::getMergeList() const { return shared->merge_list; } -const Databases Context::getDatabases() const -{ - auto lock = getLock(); - return shared->databases; -} - -Databases Context::getDatabases() -{ - auto lock = getLock(); - return shared->databases; -} - - Context::SessionKey Context::getSessionKey(const String & session_id) const { auto & user_name = client_info.current_user; @@ -526,50 +491,66 @@ std::chrono::steady_clock::duration Context::closeSessions() const return shared->close_interval; } - -static String resolveDatabase(const String & database_name, const String & current_database) +DatabaseCatalog & Context::getDatabaseCatalog() const { + return *shared->database_catalog; +} + +Databases Context::getDatabases() const +{ + return shared->database_catalog->getDatabases(); +} + +String Context::resolveDatabase(const String & database_name) const +{ + String res = database_name.empty() ? getCurrentDatabase() : database_name; + if (res.empty()) + throw Exception("Default database is not selected", ErrorCodes::UNKNOWN_DATABASE); + return res; +} + +String Context::resolveDatabaseAndCheckAccess(const String & database_name) const +{ + auto lock = getLock(); String res = database_name.empty() ? current_database : database_name; if (res.empty()) throw Exception("Default database is not selected", ErrorCodes::UNKNOWN_DATABASE); return res; } +//StorageID Context::resolveDatabase(StorageID table_id) const +//{ +// table_id.database_name = resolveDatabase(table_id.database_name); +// return table_id; +//} -const DatabasePtr Context::getDatabase(const String & database_name) const +DatabasePtr Context::getDatabase(const String & database_name) const { - auto lock = getLock(); - String db = resolveDatabase(database_name, current_database); - assertDatabaseExists(db); - return shared->databases[db]; + auto db = resolveDatabaseAndCheckAccess(database_name); + return shared->database_catalog->getDatabase(db, *this); } -DatabasePtr Context::getDatabase(const String & database_name) +DatabasePtr Context::tryGetDatabase(const String & database_name) const { - auto lock = getLock(); - String db = resolveDatabase(database_name, current_database); - assertDatabaseExists(db); - return shared->databases[db]; + String db = resolveDatabase(database_name); + return shared->database_catalog->tryGetDatabase(database_name, *this); } -const DatabasePtr Context::tryGetDatabase(const String & database_name) const +bool Context::isDatabaseExist(const String & database_name) const { - auto lock = getLock(); - String db = resolveDatabase(database_name, current_database); - auto it = shared->databases.find(db); - if (it == shared->databases.end()) - return {}; - return it->second; + String db = resolveDatabaseAndCheckAccess(database_name); + return shared->database_catalog->isDatabaseExist(database_name); } -DatabasePtr Context::tryGetDatabase(const String & database_name) +void Context::addDatabase(const String & database_name, const DatabasePtr & database) { - auto lock = getLock(); - String db = resolveDatabase(database_name, current_database); - auto it = shared->databases.find(db); - if (it == shared->databases.end()) - return {}; - return it->second; + shared->database_catalog->attachDatabase(database_name, database, *this); +} + + +DatabasePtr Context::detachDatabase(const String & database_name) +{ + return shared->database_catalog->detachDatabase(database_name, *this); } String Context::getPath() const @@ -850,26 +831,17 @@ Dependencies Context::getDependencies(const StorageID & from) const bool Context::isTableExist(const String & database_name, const String & table_name) const { - auto lock = getLock(); - String db = resolveDatabase(database_name, current_database); - Databases::const_iterator it = shared->databases.find(db); - return shared->databases.end() != it - && it->second->isTableExist(*this, table_name); + //FIXME do we need resolve temporary tables here? + auto table_id = resolveStorageID({database_name, table_name}); + return shared->database_catalog->isTableExist(table_id, *this); } bool Context::isDictionaryExists(const String & database_name, const String & dictionary_name) const { auto lock = getLock(); - String db = resolveDatabase(database_name, current_database); - Databases::const_iterator it = shared->databases.find(db); - return shared->databases.end() != it && it->second->isDictionaryExist(*this, dictionary_name); -} - -bool Context::isDatabaseExist(const String & database_name) const -{ - auto lock = getLock(); - String db = resolveDatabase(database_name, current_database); - return shared->databases.end() != shared->databases.find(db); + String db = resolveDatabaseAndCheckAccess(database_name); + auto db_ptr = shared->database_catalog->tryGetDatabase(database_name, *this); + return db_ptr && db_ptr->isDictionaryExist(*this, dictionary_name); } bool Context::isExternalTableExist(const String & table_name) const @@ -880,32 +852,18 @@ bool Context::isExternalTableExist(const String & table_name) const void Context::assertTableDoesntExist(const String & database_name, const String & table_name) const { - auto lock = getLock(); - String db = resolveDatabase(database_name, current_database); - Databases::const_iterator it = shared->databases.find(db); - if (shared->databases.end() != it && it->second->isTableExist(*this, table_name)) - throw Exception("Table " + backQuoteIfNeed(db) + "." + backQuoteIfNeed(table_name) + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS); + //FIXME do we need resolve temporary tables here? (and do we need this method?) + auto table_id = resolveStorageID({database_name, table_name}); + shared->database_catalog->assertTableDoesntExist(table_id, *this); } void Context::assertDatabaseExists(const String & database_name) const { - auto lock = getLock(); - String db = resolveDatabase(database_name, current_database); - if (shared->databases.end() == shared->databases.find(db)) - throw Exception("Database " + backQuoteIfNeed(db) + " doesn't exist", ErrorCodes::UNKNOWN_DATABASE); + String db = resolveDatabaseAndCheckAccess(database_name); + shared->database_catalog->assertDatabaseExists(db); } - -void Context::assertDatabaseDoesntExist(const String & database_name) const -{ - auto lock = getLock(); - String db = resolveDatabase(database_name, current_database); - if (shared->databases.end() != shared->databases.find(db)) - throw Exception("Database " + backQuoteIfNeed(db) + " already exists.", ErrorCodes::DATABASE_ALREADY_EXISTS); -} - - const Scalars & Context::getScalars() const { return scalars; @@ -979,41 +937,8 @@ StoragePtr Context::tryGetTable(const StorageID & table_id) const StoragePtr Context::getTableImpl(const StorageID & table_id, std::optional * exception) const { - String db; - DatabasePtr database; - - { - auto lock = getLock(); - - if (table_id.database_name.empty()) - { - StoragePtr res = tryGetExternalTable(table_id.table_name); - if (res) - return res; - } - - db = resolveDatabase(table_id.database_name, current_database); - - Databases::const_iterator it = shared->databases.find(db); - if (shared->databases.end() == it) - { - if (exception) - exception->emplace("Database " + backQuoteIfNeed(db) + " doesn't exist", ErrorCodes::UNKNOWN_DATABASE); - return {}; - } - - database = it->second; - } - - auto table = database->tryGetTable(*this, table_id.table_name); - if (!table) - { - if (exception) - exception->emplace("Table " + table_id.getNameForLogs() + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE); - return {}; - } - - return table; + auto resolved_id = resolveStorageID(table_id); + return shared->database_catalog->getTable(resolved_id, *this, exception); } @@ -1023,7 +948,8 @@ void Context::addExternalTable(const String & table_name, const StoragePtr & sto if (external_tables_mapping.end() != external_tables_mapping.find(table_name)) throw Exception("Temporary table " + backQuoteIfNeed(table_name) + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS); - external_tables_mapping.emplace(table_name, std::make_shared(*this, shared->temporary_and_external_tables, storage, ast)); + auto holder = std::make_shared(*this, *shared->database_catalog->getDatabaseForTemporaryTables(), storage, ast); + external_tables_mapping.emplace(table_name, std::move(holder)); } @@ -1106,33 +1032,13 @@ std::unique_ptr Context::getDDLGuard(const String & database, const St return std::make_unique(shared->ddl_guards[database], std::move(lock), table); } - -void Context::addDatabase(const String & database_name, const DatabasePtr & database) -{ - auto lock = getLock(); - - assertDatabaseDoesntExist(database_name); - shared->databases[database_name] = database; -} - - -DatabasePtr Context::detachDatabase(const String & database_name) -{ - auto lock = getLock(); - auto res = getDatabase(database_name); - shared->databases.erase(database_name); - - return res; -} - - ASTPtr Context::getCreateExternalTableQuery(const String & table_name) const { auto it = external_tables_mapping.find(table_name); if (external_tables_mapping.end() == it) throw Exception("Temporary table " + backQuoteIfNeed(table_name) + " doesn't exist", ErrorCodes::UNKNOWN_TABLE); - return shared->temporary_and_external_tables.getCreateTableQuery(*this, it->second->getGlobalTableID().table_name); + return shared->database_catalog->getDatabaseForTemporaryTables()->getCreateTableQuery(*this, it->second->getGlobalTableID().table_name); } Settings Context::getSettings() const @@ -1215,6 +1121,7 @@ void Context::checkSettingsConstraints(const SettingsChanges & changes) String Context::getCurrentDatabase() const { + auto lock = getLock(); return current_database; } @@ -2211,6 +2118,12 @@ void Context::resetInputCallbacks() input_blocks_reader = {}; } +StorageID Context::resolveStorageID(StorageID storage_id) const +{ + auto lock = getLock(); + return resolveStorageIDUnlocked(std::move(storage_id)); +} + StorageID Context::resolveStorageIDUnlocked(StorageID storage_id) const { if (storage_id.uuid != UUIDHelpers::Nil) @@ -2228,7 +2141,6 @@ StorageID Context::resolveStorageIDUnlocked(StorageID storage_id) const throw Exception("Default database is not selected", ErrorCodes::UNKNOWN_DATABASE); storage_id.database_name = current_database; } - checkDatabaseAccessRightsImpl(storage_id.database_name); return storage_id; } diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index 3e0997db151..da316dc4e4a 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -144,6 +144,8 @@ struct SubscriptionForUserChange }; struct TemporaryTableHolder; +class DatabaseCatalog; +using DatabaseCatalogPtr = std::shared_ptr; /** A set of known objects that can be used in the query. * Consists of a shared part (always common to all sessions and queries) @@ -300,6 +302,8 @@ public: void addDependencyUnsafe(const StorageID & from, const StorageID & where); void removeDependencyUnsafe(const StorageID & from, const StorageID & where); + DatabaseCatalog & getDatabaseCatalog() const; + /// Checking the existence of the table/database. Database can be empty - in this case the current database is used. bool isTableExist(const String & database_name, const String & table_name) const; bool isDatabaseExist(const String & database_name) const; @@ -307,7 +311,12 @@ public: bool isExternalTableExist(const String & table_name) const; void assertTableDoesntExist(const String & database_name, const String & table_name) const; void assertDatabaseExists(const String & database_name) const; - void assertDatabaseDoesntExist(const String & database_name) const; + + String resolveDatabase(const String & database_name) const; + String resolveDatabaseAndCheckAccess(const String & database_name) const; + //StorageID resolveDatabase(StorageID table_id) const; + StorageID resolveStorageID(StorageID storage_id) const; + StorageID resolveStorageIDUnlocked(StorageID storage_id) const; const Scalars & getScalars() const; const Block & getScalar(const String & name) const; @@ -321,7 +330,6 @@ public: void addScalar(const String & name, const Block & block); bool hasScalar(const String & name) const; bool removeExternalTable(const String & table_name); - StorageID resolveStorageIDUnlocked(StorageID storage_id) const; StoragePtr executeTableFunction(const ASTPtr & table_expression); @@ -410,13 +418,10 @@ public: /// Get query for the CREATE table. ASTPtr getCreateExternalTableQuery(const String & table_name) const; - const DatabasePtr getDatabase(const String & database_name) const; - DatabasePtr getDatabase(const String & database_name); - const DatabasePtr tryGetDatabase(const String & database_name) const; - DatabasePtr tryGetDatabase(const String & database_name); + DatabasePtr getDatabase(const String & database_name) const; + DatabasePtr tryGetDatabase(const String & database_name) const; - const Databases getDatabases() const; - Databases getDatabases(); + Databases getDatabases() const; std::shared_ptr acquireSession(const String & session_id, std::chrono::steady_clock::duration timeout, bool session_check) const; void releaseSession(const String & session_id, std::chrono::steady_clock::duration timeout); @@ -617,12 +622,6 @@ private: void calculateUserSettings(); void calculateAccessRights(); - /** Check if the current client has access to the specified database. - * If access is denied, throw an exception. - * NOTE: This method should always be called when the `shared->mutex` mutex is acquired. - */ - void checkDatabaseAccessRightsImpl(const std::string & database_name) const; - template void checkAccessImpl(const Args &... args) const; diff --git a/dbms/src/Interpreters/DatabaseCatalog.cpp b/dbms/src/Interpreters/DatabaseCatalog.cpp new file mode 100644 index 00000000000..e53a3298e36 --- /dev/null +++ b/dbms/src/Interpreters/DatabaseCatalog.cpp @@ -0,0 +1,228 @@ +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int UNKNOWN_DATABASE; + extern const int UNKNOWN_TABLE; + extern const int TABLE_ALREADY_EXISTS; + extern const int DATABASE_ALREADY_EXISTS; +} + +void DatabaseCatalog::loadDatabases() +{ + + auto db_for_temporary_and_external_tables = std::make_shared(TEMPORARY_DATABASE); + attachDatabase(TEMPORARY_DATABASE, db_for_temporary_and_external_tables, global_context); +} + +void DatabaseCatalog::shutdown() +{ + /** At this point, some tables may have threads that block our mutex. + * To shutdown them correctly, we will copy the current list of tables, + * and ask them all to finish their work. + * Then delete all objects with tables. + */ + + Databases current_databases; + { + std::lock_guard lock(databases_mutex); + current_databases = databases; + } + + /// We still hold "databases" (instead of std::move) for Buffer tables to flush data correctly. + + for (auto & database : current_databases) + database.second->shutdown(); + + + std::lock_guard lock(databases_mutex); + databases.clear(); + for (auto & elem : uuid_map) + { + std::lock_guard map_lock(elem.mutex); + elem.map.clear(); + } +} + +DatabaseAndTable DatabaseCatalog::tryGetByUUID(const UUID & uuid) const +{ + assert(uuid != UUIDHelpers::Nil && 0 <= getFirstLevelIdx(uuid) && getFirstLevelIdx(uuid) < uuid_map.size()); + const UUIDToStorageMapPart & map_part = uuid_map[getFirstLevelIdx(uuid)]; + std::lock_guard lock{map_part.mutex}; + auto it = map_part.map.find(uuid); + if (it == map_part.map.end()) + return {}; + return it->second; +} + +//String DatabaseCatalog::resolveDatabase(const String & database_name, const String & current_database) +//{ +// String res = database_name.empty() ? current_database : database_name; +// if (res.empty()) +// throw Exception("Default database is not selected", ErrorCodes::UNKNOWN_DATABASE); +// return res; +//} + +StoragePtr DatabaseCatalog::getTable(const StorageID & table_id, const Context & local_context, std::optional * exception) const +{ + //if (table_id.hasUUID()) + //{ + // auto db_and_table = tryGetByUUID(table_id.uuid); + // if (!db_and_table.first || !db_and_table.second) + // { + // assert(!db_and_table.first && !db_and_table.second); + // if (exception) + // exception->emplace("Table " + table_id.getNameForLogs() + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE); + // return {}; +// + // } + // return db_and_table.second; + //} + + std::lock_guard _lock{databases_mutex}; + + auto it = databases.find(table_id.getDatabaseName()); + if (databases.end() == it) + { + if (exception) + exception->emplace("Database " + backQuoteIfNeed(table_id.getDatabaseName()) + " doesn't exist", ErrorCodes::UNKNOWN_DATABASE); + return {}; + } + + auto database = it->second; + auto table = database->tryGetTable(local_context, table_id.table_name); + if (!table && exception) + exception->emplace("Table " + table_id.getNameForLogs() + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE); + + return table; +} + +void DatabaseCatalog::assertDatabaseExists(const String & database_name) const +{ + std::lock_guard lock{databases_mutex}; + assertDatabaseExistsUnlocked(database_name); +} + +void DatabaseCatalog::assertDatabaseDoesntExist(const String & database_name) const +{ + std::lock_guard lock{databases_mutex}; + assertDatabaseDoesntExistUnlocked(database_name); +} + +void DatabaseCatalog::assertDatabaseExistsUnlocked(const String & database_name) const +{ + if (databases.end() == databases.find(database_name)) + throw Exception("Database " + backQuoteIfNeed(database_name) + " doesn't exist", ErrorCodes::UNKNOWN_DATABASE); +} + + +void DatabaseCatalog::assertDatabaseDoesntExistUnlocked(const String & database_name) const +{ + if (databases.end() != databases.find(database_name)) + throw Exception("Database " + backQuoteIfNeed(database_name) + " already exists.", ErrorCodes::DATABASE_ALREADY_EXISTS); +} + +void DatabaseCatalog::attachDatabase(const String & database_name, const DatabasePtr & database, const Context & /*local_context*/) +{ + //local_context.checkDatabaseAccessRights(database_name); + std::lock_guard lock{databases_mutex}; + assertDatabaseDoesntExistUnlocked(database_name); + databases[database_name] = database; +} + + +DatabasePtr DatabaseCatalog::detachDatabase(const String & database_name, const Context & local_context) +{ + //local_context.checkDatabaseAccessRights(database_name); + std::lock_guard lock{databases_mutex}; + auto res = getDatabase(database_name, local_context); //FIXME locks order + databases.erase(database_name); + return res; +} + +DatabasePtr DatabaseCatalog::getDatabase(const String & database_name, const Context & /*local_context*/) const +{ + //String db = local_context.resolveDatabase(database_name); + //local_context.checkDatabaseAccessRights(db); //FIXME non-atomic + std::lock_guard lock{databases_mutex}; + assertDatabaseExistsUnlocked(database_name); + return databases.find(database_name)->second; +} + +DatabasePtr DatabaseCatalog::tryGetDatabase(const String & database_name, const Context & /*local_context*/) const +{ + //String db = local_context.resolveDatabase(database_name); + std::lock_guard lock{databases_mutex}; + auto it = databases.find(database_name); + if (it == databases.end()) + return {}; + return it->second; +} + +bool DatabaseCatalog::isDatabaseExist(const String & database_name) const +{ + std::lock_guard lock{databases_mutex}; + return databases.end() != databases.find(database_name); +} + +Databases DatabaseCatalog::getDatabases() const +{ + std::lock_guard lock{databases_mutex}; + return databases; +} + +bool DatabaseCatalog::isTableExist(const DB::StorageID & table_id, const DB::Context & context) const +{ + //if (table_id.hasUUID()) + // return tryGetByUUID(table_id.uuid).second != nullptr; + //else + //{ + std::lock_guard lock{databases_mutex}; + auto db = databases.find(table_id.database_name); + return db != databases.end() && db->second->isTableExist(context, table_id.table_name); + //} +} + +void DatabaseCatalog::assertTableDoesntExist(const StorageID & table_id, const Context & context) const +{ + if (!isTableExist(table_id, context)) + throw Exception("Table " + table_id.getNameForLogs() + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS); + +} + +DatabasePtr DatabaseCatalog::getDatabaseForTemporaryTables() const +{ + return getDatabase(TEMPORARY_DATABASE, global_context); +} + +void DatabaseCatalog::addUUIDMapping(const UUID & uuid, DatabasePtr database, StoragePtr table) +{ + assert(uuid != UUIDHelpers::Nil && 0 <= getFirstLevelIdx(uuid) && getFirstLevelIdx(uuid) < uuid_map.size()); + UUIDToStorageMapPart & map_part = uuid_map[getFirstLevelIdx(uuid)]; + std::lock_guard lock{map_part.mutex}; + auto [_, inserted] = map_part.map.try_emplace(uuid, std::move(database), std::move(table)); + if (!inserted) + throw Exception("Mapping for table with UUID=" + toString(uuid) + " already exists", ErrorCodes::LOGICAL_ERROR); +} + +void DatabaseCatalog::removeUUIDMapping(const UUID & uuid) +{ + assert(uuid != UUIDHelpers::Nil && 0 <= getFirstLevelIdx(uuid) && getFirstLevelIdx(uuid) < uuid_map.size()); + UUIDToStorageMapPart & map_part = uuid_map[getFirstLevelIdx(uuid)]; + std::lock_guard lock{map_part.mutex}; + if (!map_part.map.erase(uuid)) + throw Exception("Mapping for table with UUID=" + toString(uuid) + " doesn't exist", ErrorCodes::LOGICAL_ERROR); +} + +} + + diff --git a/dbms/src/Interpreters/DatabaseCatalog.h b/dbms/src/Interpreters/DatabaseCatalog.h new file mode 100644 index 00000000000..beba4f404b0 --- /dev/null +++ b/dbms/src/Interpreters/DatabaseCatalog.h @@ -0,0 +1,91 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +class Context; +class IDatabase; +struct StorageID; +class Exception; +using DatabasePtr = std::shared_ptr; +using DatabaseAndTable = std::pair; + +//TODO make singleton? +class DatabaseCatalog : boost::noncopyable +{ +public: + using Databases = std::map>; + static constexpr const char * TEMPORARY_DATABASE = "_temporary_and_external_tables"; + static constexpr const char * SYSTEM_DATABASE = "system"; + + + DatabaseCatalog(Context & global_context_/*, String default_database_*/) + : global_context(global_context_)/*, default_database(std::move(default_database_))*/ {} + + void loadDatabases(); + void shutdown(); + + + //static String resolveDatabase(const String & database_name, const String & current_database); + void assertDatabaseExists(const String & database_name) const; + void assertDatabaseDoesntExist(const String & database_name) const; + + DatabasePtr getDatabaseForTemporaryTables() const; + + void attachDatabase(const String & database_name, const DatabasePtr & database, const Context & local_context); // ca, a + DatabasePtr detachDatabase(const String & database_name, const Context & local_context); // (sr), ca, a + + DatabasePtr getDatabase(const String & database_name, const Context & local_context) const; // sr, ca, a + DatabasePtr tryGetDatabase(const String & database_name, const Context & local_context) const; // sr + bool isDatabaseExist(const String & database_name) const; // sr, ca + Databases getDatabases() const; + + DatabaseAndTable tryGetByUUID(const UUID & uuid) const; + + void assertTableDoesntExist(const StorageID & table_id, const Context & context) const; // sr, ca + bool isTableExist(const StorageID & table_id, const Context & context) const; // sr, ca + + void addUUIDMapping(const UUID & uuid, DatabasePtr database, StoragePtr table); + void removeUUIDMapping(const UUID & uuid); + + StoragePtr getTable(const StorageID & table_id, const Context & local_context, std::optional * exception) const; + +private: + void assertDatabaseExistsUnlocked(const String & database_name) const; + void assertDatabaseDoesntExistUnlocked(const String & database_name) const; + + struct UUIDToStorageMapPart + { + std::unordered_map map; + mutable std::mutex mutex; + }; + + static constexpr UInt64 bits_for_first_level = 8; + using UUIDToStorageMap = std::array; + + inline size_t getFirstLevelIdx(const UUID & uuid) const + { + return uuid.toUnderType().low >> (64 - bits_for_first_level); + } + +private: + [[maybe_unused]] Context & global_context; + mutable std::mutex databases_mutex; + //const String default_database; + Databases databases; + UUIDToStorageMap uuid_map; + + +}; + + +} diff --git a/dbms/src/Interpreters/InterpreterShowTablesQuery.cpp b/dbms/src/Interpreters/InterpreterShowTablesQuery.cpp index 28d06358327..4e5a84d30d2 100644 --- a/dbms/src/Interpreters/InterpreterShowTablesQuery.cpp +++ b/dbms/src/Interpreters/InterpreterShowTablesQuery.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -36,7 +37,7 @@ String InterpreterShowTablesQuery::getRewrittenQuery() throw Exception("The `FROM` and `TEMPORARY` cannot be used together in `SHOW TABLES`", ErrorCodes::SYNTAX_ERROR); String database = query.from.empty() ? context.getCurrentDatabase() : query.from; - context.assertDatabaseExists(database); + context.getDatabaseCatalog().assertDatabaseExists(database); std::stringstream rewritten_query; rewritten_query << "SELECT name FROM system."; From 869e20d2078738089f9abb887ed66094c30d0381 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 10 Feb 2020 16:10:17 +0300 Subject: [PATCH 016/712] move databases from Context to DatabaseCatalog --- dbms/programs/copier/ClusterCopier.cpp | 2 +- dbms/programs/local/LocalServer.cpp | 10 +-- .../programs/server/ReplicasStatusHandler.cpp | 2 +- dbms/programs/server/Server.cpp | 6 +- dbms/programs/server/TCPHandler.cpp | 2 +- dbms/src/Interpreters/ActionLocksManager.cpp | 2 +- dbms/src/Interpreters/AsynchronousMetrics.cpp | 3 +- dbms/src/Interpreters/Context.cpp | 90 ++----------------- dbms/src/Interpreters/Context.h | 16 +--- dbms/src/Interpreters/DatabaseCatalog.cpp | 35 ++++++-- dbms/src/Interpreters/DatabaseCatalog.h | 18 ++-- .../Interpreters/InterpreterCreateQuery.cpp | 25 +++--- .../src/Interpreters/InterpreterDropQuery.cpp | 14 +-- .../Interpreters/InterpreterRenameQuery.cpp | 7 +- .../InterpreterShowCreateQuery.cpp | 6 +- .../InterpreterShowTablesQuery.cpp | 4 +- .../Interpreters/InterpreterSystemQuery.cpp | 7 +- dbms/src/Interpreters/loadMetadata.cpp | 2 +- dbms/src/Interpreters/tests/create_query.cpp | 2 +- .../tests/expression_analyzer.cpp | 2 +- .../tests/in_join_subqueries_preprocessor.cpp | 6 +- dbms/src/Interpreters/tests/select_query.cpp | 4 +- dbms/src/Storages/IStorage.cpp | 2 +- dbms/src/Storages/StorageBuffer.cpp | 2 +- dbms/src/Storages/StorageDistributed.cpp | 2 +- dbms/src/Storages/StorageMaterializedView.cpp | 2 +- dbms/src/Storages/StorageMerge.cpp | 4 +- dbms/src/Storages/StorageMergeTree.cpp | 4 +- dbms/src/Storages/StorageNull.cpp | 2 +- .../Storages/StorageReplicatedMergeTree.cpp | 6 +- .../Storages/System/StorageSystemColumns.cpp | 2 +- .../System/StorageSystemDatabases.cpp | 2 +- .../Storages/System/StorageSystemGraphite.cpp | 2 +- .../System/StorageSystemMutations.cpp | 2 +- .../System/StorageSystemPartsBase.cpp | 2 +- .../Storages/System/StorageSystemReplicas.cpp | 2 +- .../System/StorageSystemReplicationQueue.cpp | 2 +- .../Storages/System/StorageSystemTables.cpp | 5 +- ..._transform_query_for_external_database.cpp | 3 +- .../src/TableFunctions/TableFunctionMerge.cpp | 2 +- 40 files changed, 129 insertions(+), 184 deletions(-) diff --git a/dbms/programs/copier/ClusterCopier.cpp b/dbms/programs/copier/ClusterCopier.cpp index 2c6b16a7ae4..4d5a7788e92 100644 --- a/dbms/programs/copier/ClusterCopier.cpp +++ b/dbms/programs/copier/ClusterCopier.cpp @@ -2467,7 +2467,7 @@ void ClusterCopierApp::mainImpl() registerDisks(); static const std::string default_database = "_local"; - context->addDatabase(default_database, std::make_shared(default_database)); + DatabaseCatalog::instance().attachDatabase(default_database, std::make_shared(default_database)); context->setCurrentDatabase(default_database); /// Initialize query scope just in case. diff --git a/dbms/programs/local/LocalServer.cpp b/dbms/programs/local/LocalServer.cpp index e9fca03d7e8..d5e08382424 100644 --- a/dbms/programs/local/LocalServer.cpp +++ b/dbms/programs/local/LocalServer.cpp @@ -188,7 +188,7 @@ try * if such tables will not be dropped, clickhouse-server will not be able to load them due to security reasons. */ std::string default_database = config().getString("default_database", "_local"); - context->addDatabase(default_database, std::make_shared(default_database)); + DatabaseCatalog::instance().attachDatabase(default_database, std::make_shared(default_database)); context->setCurrentDatabase(default_database); applyCmdOptions(); @@ -201,7 +201,7 @@ try loadMetadataSystem(*context); attachSystemTables(); loadMetadata(*context); - context->getDatabaseCatalog().loadDatabases(); + DatabaseCatalog::instance().loadDatabases(); LOG_DEBUG(log, "Loaded metadata."); } else @@ -250,12 +250,12 @@ std::string LocalServer::getInitialCreateTableQuery() void LocalServer::attachSystemTables() { - DatabasePtr system_database = context->tryGetDatabase("system"); + DatabasePtr system_database = DatabaseCatalog::instance().tryGetDatabase(DatabaseCatalog::SYSTEM_DATABASE); if (!system_database) { /// TODO: add attachTableDelayed into DatabaseMemory to speedup loading - system_database = std::make_shared("system"); - context->addDatabase("system", system_database); + system_database = std::make_shared(DatabaseCatalog::SYSTEM_DATABASE); + DatabaseCatalog::instance().attachDatabase(DatabaseCatalog::SYSTEM_DATABASE, system_database); } attachSystemTablesLocal(*system_database); diff --git a/dbms/programs/server/ReplicasStatusHandler.cpp b/dbms/programs/server/ReplicasStatusHandler.cpp index 4d72c6da3cf..512b0889869 100644 --- a/dbms/programs/server/ReplicasStatusHandler.cpp +++ b/dbms/programs/server/ReplicasStatusHandler.cpp @@ -35,7 +35,7 @@ void ReplicasStatusHandler::handleRequest(Poco::Net::HTTPServerRequest & request bool ok = true; std::stringstream message; - auto databases = context.getDatabases(); + auto databases = DatabaseCatalog::instance().getDatabases(); /// Iterate through all the replicated tables. for (const auto & db : databases) diff --git a/dbms/programs/server/Server.cpp b/dbms/programs/server/Server.cpp index fc53bf1c462..bde1d5c3cd3 100644 --- a/dbms/programs/server/Server.cpp +++ b/dbms/programs/server/Server.cpp @@ -549,10 +549,10 @@ int Server::main(const std::vector & /*args*/) /// After attaching system databases we can initialize system log. global_context->initializeSystemLogs(); /// After the system database is created, attach virtual system tables (in addition to query_log and part_log) - attachSystemTablesServer(*global_context->getDatabase("system"), has_zookeeper); + attachSystemTablesServer(*DatabaseCatalog::instance().getSystemDatabase(), has_zookeeper); /// Then, load remaining databases loadMetadata(*global_context); - global_context->getDatabaseCatalog().loadDatabases(); + DatabaseCatalog::instance().loadDatabases(); } catch (...) { @@ -693,7 +693,7 @@ int Server::main(const std::vector & /*args*/) /// This object will periodically calculate some metrics. AsynchronousMetrics async_metrics(*global_context); - attachSystemTablesAsync(*global_context->getDatabase("system"), async_metrics); + attachSystemTablesAsync(*DatabaseCatalog::instance().getSystemDatabase(), async_metrics); for (const auto & listen_host : listen_hosts) { diff --git a/dbms/programs/server/TCPHandler.cpp b/dbms/programs/server/TCPHandler.cpp index b645118494b..3cb00320b10 100644 --- a/dbms/programs/server/TCPHandler.cpp +++ b/dbms/programs/server/TCPHandler.cpp @@ -108,7 +108,7 @@ void TCPHandler::runImpl() /// When connecting, the default database can be specified. if (!default_database.empty()) { - if (!connection_context.isDatabaseExist(default_database)) + if (!DatabaseCatalog::instance().isDatabaseExist(default_database)) { Exception e("Database " + backQuote(default_database) + " doesn't exist", ErrorCodes::UNKNOWN_DATABASE); LOG_ERROR(log, "Code: " << e.code() << ", e.displayText() = " << e.displayText() diff --git a/dbms/src/Interpreters/ActionLocksManager.cpp b/dbms/src/Interpreters/ActionLocksManager.cpp index cc5d01863a2..023129f7cdc 100644 --- a/dbms/src/Interpreters/ActionLocksManager.cpp +++ b/dbms/src/Interpreters/ActionLocksManager.cpp @@ -22,7 +22,7 @@ namespace ActionLocks template inline void forEachTable(Context & context, F && f) { - for (auto & elem : context.getDatabases()) + for (auto & elem : DatabaseCatalog::instance().getDatabases()) for (auto iterator = elem.second->getTablesIterator(context); iterator->isValid(); iterator->next()) f(iterator->table()); diff --git a/dbms/src/Interpreters/AsynchronousMetrics.cpp b/dbms/src/Interpreters/AsynchronousMetrics.cpp index 661325b22c2..16d00036c24 100644 --- a/dbms/src/Interpreters/AsynchronousMetrics.cpp +++ b/dbms/src/Interpreters/AsynchronousMetrics.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -131,7 +132,7 @@ void AsynchronousMetrics::update() set("Uptime", context.getUptimeSeconds()); { - auto databases = context.getDatabases(); + auto databases = DatabaseCatalog::instance().getDatabases(); size_t max_queue_size = 0; size_t max_inserts_in_queue = 0; diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index 07d6c001074..ba7de8f877d 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -185,8 +185,6 @@ struct ContextShared String tmp_path; /// Path to the temporary files that occur when processing the request. mutable VolumePtr tmp_volume; /// Volume for the the temporary files that occur when processing the request. - std::shared_ptr database_catalog; /// Manages databases and tables in them. - mutable std::optional embedded_dictionaries; /// Metrica's dictionaries. Have lazy initialization. mutable std::optional external_dictionaries_loader; mutable std::optional external_models_loader; @@ -317,7 +315,7 @@ struct ContextShared if (system_logs) system_logs->shutdown(); - database_catalog->shutdown(); + DatabaseCatalog::instance().shutdown(); /// Preemptive destruction is important, because these objects may have a refcount to ContextShared (cyclic reference). /// TODO: Get rid of this. @@ -362,7 +360,6 @@ Context Context::createGlobal() res.row_policy = std::make_shared(); res.access_rights = std::make_shared(); res.shared = std::make_shared(); - res.shared->database_catalog = std::make_shared(res); return res; } @@ -491,16 +488,6 @@ std::chrono::steady_clock::duration Context::closeSessions() const return shared->close_interval; } -DatabaseCatalog & Context::getDatabaseCatalog() const -{ - return *shared->database_catalog; -} - -Databases Context::getDatabases() const -{ - return shared->database_catalog->getDatabases(); -} - String Context::resolveDatabase(const String & database_name) const { String res = database_name.empty() ? getCurrentDatabase() : database_name; @@ -509,50 +496,6 @@ String Context::resolveDatabase(const String & database_name) const return res; } -String Context::resolveDatabaseAndCheckAccess(const String & database_name) const -{ - auto lock = getLock(); - String res = database_name.empty() ? current_database : database_name; - if (res.empty()) - throw Exception("Default database is not selected", ErrorCodes::UNKNOWN_DATABASE); - return res; -} - -//StorageID Context::resolveDatabase(StorageID table_id) const -//{ -// table_id.database_name = resolveDatabase(table_id.database_name); -// return table_id; -//} - -DatabasePtr Context::getDatabase(const String & database_name) const -{ - auto db = resolveDatabaseAndCheckAccess(database_name); - return shared->database_catalog->getDatabase(db, *this); -} - -DatabasePtr Context::tryGetDatabase(const String & database_name) const -{ - String db = resolveDatabase(database_name); - return shared->database_catalog->tryGetDatabase(database_name, *this); -} - -bool Context::isDatabaseExist(const String & database_name) const -{ - String db = resolveDatabaseAndCheckAccess(database_name); - return shared->database_catalog->isDatabaseExist(database_name); -} - -void Context::addDatabase(const String & database_name, const DatabasePtr & database) -{ - shared->database_catalog->attachDatabase(database_name, database, *this); -} - - -DatabasePtr Context::detachDatabase(const String & database_name) -{ - return shared->database_catalog->detachDatabase(database_name, *this); -} - String Context::getPath() const { auto lock = getLock(); @@ -833,14 +776,14 @@ bool Context::isTableExist(const String & database_name, const String & table_na { //FIXME do we need resolve temporary tables here? auto table_id = resolveStorageID({database_name, table_name}); - return shared->database_catalog->isTableExist(table_id, *this); + return DatabaseCatalog::instance().isTableExist(table_id, *this); } bool Context::isDictionaryExists(const String & database_name, const String & dictionary_name) const { auto lock = getLock(); - String db = resolveDatabaseAndCheckAccess(database_name); - auto db_ptr = shared->database_catalog->tryGetDatabase(database_name, *this); + String db = resolveDatabase(database_name); + auto db_ptr = DatabaseCatalog::instance().tryGetDatabase(database_name); return db_ptr && db_ptr->isDictionaryExist(*this, dictionary_name); } @@ -849,21 +792,6 @@ bool Context::isExternalTableExist(const String & table_name) const return external_tables_mapping.count(table_name); } - -void Context::assertTableDoesntExist(const String & database_name, const String & table_name) const -{ - //FIXME do we need resolve temporary tables here? (and do we need this method?) - auto table_id = resolveStorageID({database_name, table_name}); - shared->database_catalog->assertTableDoesntExist(table_id, *this); -} - - -void Context::assertDatabaseExists(const String & database_name) const -{ - String db = resolveDatabaseAndCheckAccess(database_name); - shared->database_catalog->assertDatabaseExists(db); -} - const Scalars & Context::getScalars() const { return scalars; @@ -938,7 +866,7 @@ StoragePtr Context::tryGetTable(const StorageID & table_id) const StoragePtr Context::getTableImpl(const StorageID & table_id, std::optional * exception) const { auto resolved_id = resolveStorageID(table_id); - return shared->database_catalog->getTable(resolved_id, *this, exception); + return DatabaseCatalog::instance().getTable(resolved_id, *this, exception); } @@ -948,7 +876,7 @@ void Context::addExternalTable(const String & table_name, const StoragePtr & sto if (external_tables_mapping.end() != external_tables_mapping.find(table_name)) throw Exception("Temporary table " + backQuoteIfNeed(table_name) + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS); - auto holder = std::make_shared(*this, *shared->database_catalog->getDatabaseForTemporaryTables(), storage, ast); + auto holder = std::make_shared(*this, *DatabaseCatalog::instance().getDatabaseForTemporaryTables(), storage, ast); external_tables_mapping.emplace(table_name, std::move(holder)); } @@ -1038,7 +966,7 @@ ASTPtr Context::getCreateExternalTableQuery(const String & table_name) const if (external_tables_mapping.end() == it) throw Exception("Temporary table " + backQuoteIfNeed(table_name) + " doesn't exist", ErrorCodes::UNKNOWN_TABLE); - return shared->database_catalog->getDatabaseForTemporaryTables()->getCreateTableQuery(*this, it->second->getGlobalTableID().table_name); + return DatabaseCatalog::instance().getDatabaseForTemporaryTables()->getCreateTableQuery(*this, it->second->getGlobalTableID().table_name); } Settings Context::getSettings() const @@ -1140,10 +1068,10 @@ String Context::getInitialQueryId() const void Context::setCurrentDatabase(const String & name) { + DatabaseCatalog::instance().assertDatabaseExists(name); auto lock = getLock(); - assertDatabaseExists(name); - current_database = name; calculateAccessRights(); + current_database = name; } diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index da316dc4e4a..3ee96829e53 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -181,8 +182,6 @@ private: String default_format; /// Format, used when server formats data by itself and if query does not have FORMAT specification. /// Thus, used in HTTP interface. If not specified - then some globally default format is used. - // TODO maybe replace with DatabaseMemory? - //TableAndCreateASTs external_tables; /// Temporary tables. using TemporaryTablesMapping = std::map>; TemporaryTablesMapping external_tables_mapping; Scalars scalars; @@ -302,15 +301,10 @@ public: void addDependencyUnsafe(const StorageID & from, const StorageID & where); void removeDependencyUnsafe(const StorageID & from, const StorageID & where); - DatabaseCatalog & getDatabaseCatalog() const; - /// Checking the existence of the table/database. Database can be empty - in this case the current database is used. bool isTableExist(const String & database_name, const String & table_name) const; - bool isDatabaseExist(const String & database_name) const; bool isDictionaryExists(const String & database_name, const String & dictionary_name) const; bool isExternalTableExist(const String & table_name) const; - void assertTableDoesntExist(const String & database_name, const String & table_name) const; - void assertDatabaseExists(const String & database_name) const; String resolveDatabase(const String & database_name) const; String resolveDatabaseAndCheckAccess(const String & database_name) const; @@ -336,9 +330,6 @@ public: void addViewSource(const StoragePtr & storage); StoragePtr getViewSource(); - void addDatabase(const String & database_name, const DatabasePtr & database); - DatabasePtr detachDatabase(const String & database_name); - /// Get an object that protects the table from concurrently executing multiple DDL operations. std::unique_ptr getDDLGuard(const String & database, const String & table) const; @@ -418,11 +409,6 @@ public: /// Get query for the CREATE table. ASTPtr getCreateExternalTableQuery(const String & table_name) const; - DatabasePtr getDatabase(const String & database_name) const; - DatabasePtr tryGetDatabase(const String & database_name) const; - - Databases getDatabases() const; - std::shared_ptr acquireSession(const String & session_id, std::chrono::steady_clock::duration timeout, bool session_check) const; void releaseSession(const String & session_id, std::chrono::steady_clock::duration timeout); diff --git a/dbms/src/Interpreters/DatabaseCatalog.cpp b/dbms/src/Interpreters/DatabaseCatalog.cpp index e53a3298e36..59e4368c381 100644 --- a/dbms/src/Interpreters/DatabaseCatalog.cpp +++ b/dbms/src/Interpreters/DatabaseCatalog.cpp @@ -17,11 +17,12 @@ namespace ErrorCodes extern const int DATABASE_ALREADY_EXISTS; } + void DatabaseCatalog::loadDatabases() { auto db_for_temporary_and_external_tables = std::make_shared(TEMPORARY_DATABASE); - attachDatabase(TEMPORARY_DATABASE, db_for_temporary_and_external_tables, global_context); + attachDatabase(TEMPORARY_DATABASE, db_for_temporary_and_external_tables); } void DatabaseCatalog::shutdown() @@ -131,7 +132,7 @@ void DatabaseCatalog::assertDatabaseDoesntExistUnlocked(const String & database_ throw Exception("Database " + backQuoteIfNeed(database_name) + " already exists.", ErrorCodes::DATABASE_ALREADY_EXISTS); } -void DatabaseCatalog::attachDatabase(const String & database_name, const DatabasePtr & database, const Context & /*local_context*/) +void DatabaseCatalog::attachDatabase(const String & database_name, const DatabasePtr & database) { //local_context.checkDatabaseAccessRights(database_name); std::lock_guard lock{databases_mutex}; @@ -140,17 +141,18 @@ void DatabaseCatalog::attachDatabase(const String & database_name, const Databas } -DatabasePtr DatabaseCatalog::detachDatabase(const String & database_name, const Context & local_context) +DatabasePtr DatabaseCatalog::detachDatabase(const String & database_name) { //local_context.checkDatabaseAccessRights(database_name); std::lock_guard lock{databases_mutex}; - auto res = getDatabase(database_name, local_context); //FIXME locks order + auto res = getDatabase(database_name); //FIXME locks order databases.erase(database_name); return res; } -DatabasePtr DatabaseCatalog::getDatabase(const String & database_name, const Context & /*local_context*/) const +DatabasePtr DatabaseCatalog::getDatabase(const String & database_name) const { + assert(!database_name.empty()); //String db = local_context.resolveDatabase(database_name); //local_context.checkDatabaseAccessRights(db); //FIXME non-atomic std::lock_guard lock{databases_mutex}; @@ -158,8 +160,9 @@ DatabasePtr DatabaseCatalog::getDatabase(const String & database_name, const Con return databases.find(database_name)->second; } -DatabasePtr DatabaseCatalog::tryGetDatabase(const String & database_name, const Context & /*local_context*/) const +DatabasePtr DatabaseCatalog::tryGetDatabase(const String & database_name) const { + assert(!database_name.empty()); //String db = local_context.resolveDatabase(database_name); std::lock_guard lock{databases_mutex}; auto it = databases.find(database_name); @@ -170,6 +173,7 @@ DatabasePtr DatabaseCatalog::tryGetDatabase(const String & database_name, const bool DatabaseCatalog::isDatabaseExist(const String & database_name) const { + assert(!database_name.empty()); std::lock_guard lock{databases_mutex}; return databases.end() != databases.find(database_name); } @@ -201,7 +205,12 @@ void DatabaseCatalog::assertTableDoesntExist(const StorageID & table_id, const C DatabasePtr DatabaseCatalog::getDatabaseForTemporaryTables() const { - return getDatabase(TEMPORARY_DATABASE, global_context); + return getDatabase(TEMPORARY_DATABASE); +} + +DatabasePtr DatabaseCatalog::getSystemDatabase() const +{ + return getDatabase(SYSTEM_DATABASE); } void DatabaseCatalog::addUUIDMapping(const UUID & uuid, DatabasePtr database, StoragePtr table) @@ -223,6 +232,18 @@ void DatabaseCatalog::removeUUIDMapping(const UUID & uuid) throw Exception("Mapping for table with UUID=" + toString(uuid) + " doesn't exist", ErrorCodes::LOGICAL_ERROR); } +DatabaseCatalog & DatabaseCatalog::instance() +{ + static DatabaseCatalog database_catalog; + return database_catalog; +} + +DatabasePtr DatabaseCatalog::getDatabase(const String & database_name, const Context & local_context) const +{ + String resolved_database = local_context.resolveDatabase(database_name); + return getDatabase(resolved_database); +} + } diff --git a/dbms/src/Interpreters/DatabaseCatalog.h b/dbms/src/Interpreters/DatabaseCatalog.h index beba4f404b0..b5cae14e699 100644 --- a/dbms/src/Interpreters/DatabaseCatalog.h +++ b/dbms/src/Interpreters/DatabaseCatalog.h @@ -27,9 +27,10 @@ public: static constexpr const char * TEMPORARY_DATABASE = "_temporary_and_external_tables"; static constexpr const char * SYSTEM_DATABASE = "system"; + static DatabaseCatalog & instance(); - DatabaseCatalog(Context & global_context_/*, String default_database_*/) - : global_context(global_context_)/*, default_database(std::move(default_database_))*/ {} + //DatabaseCatalog(/*Context & global_context_, String default_database_*/) {} + //: global_context(global_context_)/*, default_database(std::move(default_database_))*/ {} void loadDatabases(); void shutdown(); @@ -40,12 +41,14 @@ public: void assertDatabaseDoesntExist(const String & database_name) const; DatabasePtr getDatabaseForTemporaryTables() const; + DatabasePtr getSystemDatabase() const; - void attachDatabase(const String & database_name, const DatabasePtr & database, const Context & local_context); // ca, a - DatabasePtr detachDatabase(const String & database_name, const Context & local_context); // (sr), ca, a + void attachDatabase(const String & database_name, const DatabasePtr & database); // ca, a + DatabasePtr detachDatabase(const String & database_name); // (sr), ca, a - DatabasePtr getDatabase(const String & database_name, const Context & local_context) const; // sr, ca, a - DatabasePtr tryGetDatabase(const String & database_name, const Context & local_context) const; // sr + DatabasePtr getDatabase(const String & database_name, const Context & local_context) const; + DatabasePtr getDatabase(const String & database_name) const; // sr, ca, a + DatabasePtr tryGetDatabase(const String & database_name) const; // sr bool isDatabaseExist(const String & database_name) const; // sr, ca Databases getDatabases() const; @@ -60,6 +63,7 @@ public: StoragePtr getTable(const StorageID & table_id, const Context & local_context, std::optional * exception) const; private: + DatabaseCatalog() = default; void assertDatabaseExistsUnlocked(const String & database_name) const; void assertDatabaseDoesntExistUnlocked(const String & database_name) const; @@ -78,7 +82,7 @@ private: } private: - [[maybe_unused]] Context & global_context; + //[[maybe_unused]] Context & global_context; mutable std::mutex databases_mutex; //const String default_database; Databases databases; diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp index 4ca606d5a98..ab828fc7306 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp @@ -86,7 +86,7 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) auto guard = context.getDDLGuard(database_name, ""); /// Database can be created before or it can be created concurrently in another thread, while we were waiting in DDLGuard - if (context.isDatabaseExist(database_name)) + if (DatabaseCatalog::instance().isDatabaseExist(database_name)) { if (create.if_not_exists) return {}; @@ -147,7 +147,7 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) bool renamed = false; try { - context.addDatabase(database_name, database); + DatabaseCatalog::instance().attachDatabase(database_name, database); added = true; if (need_write_metadata) @@ -163,7 +163,7 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) if (renamed) Poco::File(metadata_file_tmp_path).remove(); if (added) - context.detachDatabase(database_name); + DatabaseCatalog::instance().detachDatabase(database_name); throw; } @@ -507,10 +507,10 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const { /// NOTE Getting the structure from the table specified in the AS is done not atomically with the creation of the table. - String as_database_name = create.as_database.empty() ? context.getCurrentDatabase() : create.as_database; + String as_database_name = context.resolveDatabase(create.as_database); String as_table_name = create.as_table; - ASTPtr as_create_ptr = context.getDatabase(as_database_name)->getCreateTableQuery(context, as_table_name); + ASTPtr as_create_ptr = DatabaseCatalog::instance().getDatabase(as_database_name)->getCreateTableQuery(context, as_table_name); const auto & as_create = as_create_ptr->as(); const String qualified_name = backQuoteIfNeed(as_database_name) + "." + backQuoteIfNeed(as_table_name); @@ -542,18 +542,20 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) throw Exception("Temporary tables cannot be inside a database. You should not specify a database for a temporary table.", ErrorCodes::BAD_DATABASE_FOR_TEMPORARY_TABLE); + String current_database = context.getCurrentDatabase(); + // If this is a stub ATTACH query, read the query definition from the database if (create.attach && !create.storage && !create.columns_list) { bool if_not_exists = create.if_not_exists; // Table SQL definition is available even if the table is detached - auto query = context.getDatabase(create.database)->getCreateTableQuery(context, create.table); + auto database_name = create.database.empty() ? current_database : create.database; + auto query = DatabaseCatalog::instance().getDatabase(database_name)->getCreateTableQuery(context, create.table); create = query->as(); // Copy the saved create query, but use ATTACH instead of CREATE create.attach = true; create.if_not_exists = if_not_exists; } - String current_database = context.getCurrentDatabase(); if (!create.temporary && create.database.empty()) create.database = current_database; if (!create.to_table.empty() && create.to_database.empty()) @@ -588,7 +590,7 @@ bool InterpreterCreateQuery::doCreateTable(const ASTCreateQuery & create, bool need_add_to_database = !create.temporary || create.is_live_view; if (need_add_to_database) { - database = context.getDatabase(create.database); + database = DatabaseCatalog::instance().getDatabase(create.database); /** If the request specifies IF NOT EXISTS, we allow concurrent CREATE queries (which do nothing). * If table doesnt exist, one thread is creating table, while others wait in DDLGuard. @@ -684,12 +686,11 @@ BlockIO InterpreterCreateQuery::createDictionary(ASTCreateQuery & create) { String dictionary_name = create.table; - if (create.database.empty()) - create.database = context.getCurrentDatabase(); + create.database = context.resolveDatabase(create.database); const String & database_name = create.database; auto guard = context.getDDLGuard(database_name, dictionary_name); - DatabasePtr database = context.getDatabase(database_name); + DatabasePtr database = DatabaseCatalog::instance().getDatabase(database_name); if (database->isDictionaryExist(context, dictionary_name)) { @@ -703,7 +704,7 @@ BlockIO InterpreterCreateQuery::createDictionary(ASTCreateQuery & create) if (create.attach) { - auto query = context.getDatabase(database_name)->getCreateDictionaryQuery(context, dictionary_name); + auto query = DatabaseCatalog::instance().getDatabase(database_name)->getCreateDictionaryQuery(context, dictionary_name); create = query->as(); create.attach = true; } diff --git a/dbms/src/Interpreters/InterpreterDropQuery.cpp b/dbms/src/Interpreters/InterpreterDropQuery.cpp index 94d077abe8c..9f7ea306f10 100644 --- a/dbms/src/Interpreters/InterpreterDropQuery.cpp +++ b/dbms/src/Interpreters/InterpreterDropQuery.cpp @@ -76,7 +76,7 @@ BlockIO InterpreterDropQuery::executeToTable( ErrorCodes::UNKNOWN_TABLE); } - String database_name = database_name_.empty() ? context.getCurrentDatabase() : database_name_; + String database_name = context.resolveDatabase(database_name_); auto ddl_guard = (!no_ddl_lock ? context.getDDLGuard(database_name, table_name) : nullptr); @@ -168,7 +168,7 @@ BlockIO InterpreterDropQuery::executeToDictionary( if (is_temporary) throw Exception("Temporary dictionaries are not possible.", ErrorCodes::SYNTAX_ERROR); - String database_name = database_name_.empty() ? context.getCurrentDatabase() : database_name_; + String database_name = context.resolveDatabase(database_name_); auto ddl_guard = (!no_ddl_lock ? context.getDDLGuard(database_name, dictionary_name) : nullptr); @@ -248,7 +248,7 @@ BlockIO InterpreterDropQuery::executeToDatabase(const String & database_name, AS else if (kind == ASTDropQuery::Kind::Detach) { context.checkAccess(AccessType::DETACH_DATABASE, database_name); - context.detachDatabase(database_name); + DatabaseCatalog::instance().detachDatabase(database_name); database->shutdown(); } else if (kind == ASTDropQuery::Kind::Drop) @@ -270,14 +270,14 @@ BlockIO InterpreterDropQuery::executeToDatabase(const String & database_name, AS auto context_lock = context.getLock(); /// Someone could have time to delete the database before us. - context.assertDatabaseExists(database_name); + DatabaseCatalog::instance().assertDatabaseExists(database_name); /// Someone could have time to create a table in the database to be deleted while we deleted the tables without the context lock. - if (!context.getDatabase(database_name)->empty(context)) + if (!DatabaseCatalog::instance().getDatabase(database_name)->empty(context)) throw Exception("New table appeared in database being dropped. Try dropping it again.", ErrorCodes::DATABASE_NOT_EMPTY); /// Delete database information from the RAM - context.detachDatabase(database_name); + DatabaseCatalog::instance().detachDatabase(database_name); database->shutdown(); @@ -296,7 +296,7 @@ BlockIO InterpreterDropQuery::executeToDatabase(const String & database_name, AS DatabasePtr InterpreterDropQuery::tryGetDatabase(const String & database_name, bool if_exists) { - return if_exists ? context.tryGetDatabase(database_name) : context.getDatabase(database_name); + return if_exists ? DatabaseCatalog::instance().tryGetDatabase(database_name) : DatabaseCatalog::instance().getDatabase(database_name); } DatabaseAndTable InterpreterDropQuery::tryGetDatabaseAndTable(const String & database_name, const String & table_name, bool if_exists) diff --git a/dbms/src/Interpreters/InterpreterRenameQuery.cpp b/dbms/src/Interpreters/InterpreterRenameQuery.cpp index 7965291daaa..150fca16b53 100644 --- a/dbms/src/Interpreters/InterpreterRenameQuery.cpp +++ b/dbms/src/Interpreters/InterpreterRenameQuery.cpp @@ -88,16 +88,17 @@ BlockIO InterpreterRenameQuery::execute() for (auto & table_guard : table_guards) table_guard.second = context.getDDLGuard(table_guard.first.database_name, table_guard.first.table_name); + auto & database_catalog = DatabaseCatalog::instance(); for (auto & elem : descriptions) { - context.assertTableDoesntExist(elem.to_database_name, elem.to_table_name); + database_catalog.assertTableDoesntExist(StorageID(elem.to_database_name, elem.to_table_name), context); auto from_table = context.getTable(elem.from_database_name, elem.from_table_name); auto from_table_lock = from_table->lockExclusively(context.getCurrentQueryId()); - context.getDatabase(elem.from_database_name)->renameTable( + database_catalog.getDatabase(elem.from_database_name)->renameTable( context, elem.from_table_name, - *context.getDatabase(elem.to_database_name), + *database_catalog.getDatabase(elem.to_database_name), elem.to_table_name, from_table_lock); } diff --git a/dbms/src/Interpreters/InterpreterShowCreateQuery.cpp b/dbms/src/Interpreters/InterpreterShowCreateQuery.cpp index 8d5e03b4a7d..4f59b6665d5 100644 --- a/dbms/src/Interpreters/InterpreterShowCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterShowCreateQuery.cpp @@ -52,7 +52,7 @@ BlockInputStreamPtr InterpreterShowCreateQuery::executeImpl() else { context.checkAccess(AccessType::SHOW, show_query->database, show_query->table); - create_query = context.getDatabase(show_query->database)->getCreateTableQuery(context, show_query->table); + create_query = DatabaseCatalog::instance().getDatabase(show_query->database, context)->getCreateTableQuery(context, show_query->table); } } else if ((show_query = query_ptr->as())) @@ -60,14 +60,14 @@ BlockInputStreamPtr InterpreterShowCreateQuery::executeImpl() if (show_query->temporary) throw Exception("Temporary databases are not possible.", ErrorCodes::SYNTAX_ERROR); context.checkAccess(AccessType::SHOW, show_query->database); - create_query = context.getDatabase(show_query->database)->getCreateDatabaseQuery(context); + create_query = DatabaseCatalog::instance().getDatabase(show_query->database)->getCreateDatabaseQuery(context); } else if ((show_query = query_ptr->as())) { if (show_query->temporary) throw Exception("Temporary dictionaries are not possible.", ErrorCodes::SYNTAX_ERROR); context.checkAccess(AccessType::SHOW, show_query->database, show_query->table); - create_query = context.getDatabase(show_query->database)->getCreateDictionaryQuery(context, show_query->table); + create_query = DatabaseCatalog::instance().getDatabase(show_query->database, context)->getCreateDictionaryQuery(context, show_query->table); } if (!create_query && show_query && show_query->temporary) diff --git a/dbms/src/Interpreters/InterpreterShowTablesQuery.cpp b/dbms/src/Interpreters/InterpreterShowTablesQuery.cpp index 4e5a84d30d2..23d386a53f4 100644 --- a/dbms/src/Interpreters/InterpreterShowTablesQuery.cpp +++ b/dbms/src/Interpreters/InterpreterShowTablesQuery.cpp @@ -36,8 +36,8 @@ String InterpreterShowTablesQuery::getRewrittenQuery() if (query.temporary && !query.from.empty()) throw Exception("The `FROM` and `TEMPORARY` cannot be used together in `SHOW TABLES`", ErrorCodes::SYNTAX_ERROR); - String database = query.from.empty() ? context.getCurrentDatabase() : query.from; - context.getDatabaseCatalog().assertDatabaseExists(database); + String database = context.resolveDatabase(query.from); + DatabaseCatalog::instance().assertDatabaseExists(database); std::stringstream rewritten_query; rewritten_query << "SELECT name FROM system."; diff --git a/dbms/src/Interpreters/InterpreterSystemQuery.cpp b/dbms/src/Interpreters/InterpreterSystemQuery.cpp index 7c5043154af..6ac9da8f3d2 100644 --- a/dbms/src/Interpreters/InterpreterSystemQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSystemQuery.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -135,7 +136,7 @@ void startStopAction(Context & context, Poco::Logger * log, ASTSystemQuery & que } else { - for (auto & elem : context.getDatabases()) + for (auto & elem : DatabaseCatalog::instance().getDatabases()) { for (auto iterator = elem.second->getTablesIterator(context); iterator->isValid(); iterator->next()) { @@ -312,7 +313,7 @@ BlockIO InterpreterSystemQuery::execute() StoragePtr InterpreterSystemQuery::tryRestartReplica(const String & database_name, const String & table_name, Context & system_context) { context.checkAccess(AccessType::RESTART_REPLICA, database_name, table_name); - auto database = system_context.getDatabase(database_name); + auto database = DatabaseCatalog::instance().getDatabase(database_name, system_context); auto table_ddl_guard = system_context.getDDLGuard(database_name, table_name); ASTPtr create_ast; @@ -361,7 +362,7 @@ void InterpreterSystemQuery::restartReplicas(Context & system_context) { std::vector> replica_names; - for (auto & elem : system_context.getDatabases()) + for (auto & elem : DatabaseCatalog::instance().getDatabases()) { DatabasePtr & database = elem.second; const String & database_name = elem.first; diff --git a/dbms/src/Interpreters/loadMetadata.cpp b/dbms/src/Interpreters/loadMetadata.cpp index 241385f97ec..53954faa2c0 100644 --- a/dbms/src/Interpreters/loadMetadata.cpp +++ b/dbms/src/Interpreters/loadMetadata.cpp @@ -140,7 +140,7 @@ void loadMetadataSystem(Context & context) Poco::File(global_path + "metadata/" SYSTEM_DATABASE).createDirectories(); auto system_database = std::make_shared(SYSTEM_DATABASE, global_path + "metadata/" SYSTEM_DATABASE "/", context); - context.addDatabase(SYSTEM_DATABASE, system_database); + DatabaseCatalog::instance().attachDatabase(SYSTEM_DATABASE, system_database); } } diff --git a/dbms/src/Interpreters/tests/create_query.cpp b/dbms/src/Interpreters/tests/create_query.cpp index e159afced58..20a0bfcb062 100644 --- a/dbms/src/Interpreters/tests/create_query.cpp +++ b/dbms/src/Interpreters/tests/create_query.cpp @@ -83,7 +83,7 @@ try context.setPath("./"); auto database = std::make_shared("test", "./metadata/test/", context); - context.addDatabase("test", database); + DatabaseCatalog::instance().attachDatabase("test", database); database->loadStoredObjects(context, false); context.setCurrentDatabase("test"); diff --git a/dbms/src/Interpreters/tests/expression_analyzer.cpp b/dbms/src/Interpreters/tests/expression_analyzer.cpp index 079a22620bc..ff7579506f4 100644 --- a/dbms/src/Interpreters/tests/expression_analyzer.cpp +++ b/dbms/src/Interpreters/tests/expression_analyzer.cpp @@ -100,7 +100,7 @@ int main() context.makeGlobalContext(); auto system_database = std::make_shared("system"); - context.addDatabase("system", system_database); + DatabaseCatalog::instance().attachDatabase("system", system_database); //context.setCurrentDatabase("system"); system_database->attachTable("one", StorageSystemOne::create("one")); system_database->attachTable("numbers", StorageSystemNumbers::create("numbers", false)); diff --git a/dbms/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp b/dbms/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp index a46afb2c8c3..017ca2e1739 100644 --- a/dbms/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp +++ b/dbms/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp @@ -1166,7 +1166,7 @@ TestResult check(const TestEntry & entry) auto storage_distributed_hits = StorageDistributedFake::create("distant_db", "distant_hits", entry.shard_count); DB::DatabasePtr database = std::make_shared("test", "./metadata/test/", context); - context.addDatabase("test", database); + DB::DatabaseCatalog::instance().attachDatabase("test", database); database->attachTable("visits_all", storage_distributed_visits); database->attachTable("hits_all", storage_distributed_hits); context.setCurrentDatabase("test"); @@ -1209,12 +1209,12 @@ TestResult check(const TestEntry & entry) bool res = equals(ast_input, ast_expected); std::string output = DB::queryToString(ast_input); - context.detachDatabase("test"); + DB::DatabaseCatalog::instance().detachDatabase("test"); return TestResult(res, output); } catch (DB::Exception & e) { - context.detachDatabase("test"); + DB::DatabaseCatalog::instance().detachDatabase("test"); return TestResult(false, e.displayText()); } } diff --git a/dbms/src/Interpreters/tests/select_query.cpp b/dbms/src/Interpreters/tests/select_query.cpp index 197d1c55faf..2eb9ee92da2 100644 --- a/dbms/src/Interpreters/tests/select_query.cpp +++ b/dbms/src/Interpreters/tests/select_query.cpp @@ -38,9 +38,9 @@ try loadMetadata(context); DatabasePtr system = std::make_shared("system", "./metadata/system/", context); - context.addDatabase("system", system); + DatabaseCatalog::instance().attachDatabase("system", system); system->loadStoredObjects(context, false); - attachSystemTablesLocal(*context.getDatabase("system")); + attachSystemTablesLocal(*DatabaseCatalog::instance().getSystemDatabase()); context.setCurrentDatabase("default"); ReadBufferFromFileDescriptor in(STDIN_FILENO); diff --git a/dbms/src/Storages/IStorage.cpp b/dbms/src/Storages/IStorage.cpp index 332ecbc8681..4c833601ca9 100644 --- a/dbms/src/Storages/IStorage.cpp +++ b/dbms/src/Storages/IStorage.cpp @@ -387,7 +387,7 @@ void IStorage::alter( auto table_id = getStorageID(); StorageInMemoryMetadata metadata = getInMemoryMetadata(); params.apply(metadata); - context.getDatabase(table_id.database_name)->alterTable(context, table_id.table_name, metadata); + DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id.table_name, metadata); setColumns(std::move(metadata.columns)); } diff --git a/dbms/src/Storages/StorageBuffer.cpp b/dbms/src/Storages/StorageBuffer.cpp index 2f1b633413d..58cd66e4ee3 100644 --- a/dbms/src/Storages/StorageBuffer.cpp +++ b/dbms/src/Storages/StorageBuffer.cpp @@ -765,7 +765,7 @@ void StorageBuffer::alter(const AlterCommands & params, const Context & context, StorageInMemoryMetadata metadata = getInMemoryMetadata(); params.apply(metadata); - context.getDatabase(table_id.database_name)->alterTable(context, table_id.table_name, metadata); + DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id.table_name, metadata); setColumns(std::move(metadata.columns)); } diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp index 3dabc4aa701..0782176f158 100644 --- a/dbms/src/Storages/StorageDistributed.cpp +++ b/dbms/src/Storages/StorageDistributed.cpp @@ -469,7 +469,7 @@ void StorageDistributed::alter(const AlterCommands & params, const Context & con checkAlterIsPossible(params, context.getSettingsRef()); StorageInMemoryMetadata metadata = getInMemoryMetadata(); params.apply(metadata); - context.getDatabase(table_id.database_name)->alterTable(context, table_id.table_name, metadata); + DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id.table_name, metadata); setColumns(std::move(metadata.columns)); } diff --git a/dbms/src/Storages/StorageMaterializedView.cpp b/dbms/src/Storages/StorageMaterializedView.cpp index 2ace03e154b..4c47a1d4691 100644 --- a/dbms/src/Storages/StorageMaterializedView.cpp +++ b/dbms/src/Storages/StorageMaterializedView.cpp @@ -294,7 +294,7 @@ void StorageMaterializedView::alter( } /// end modify query - context.getDatabase(table_id.database_name)->alterTable(context, table_id.table_name, metadata); + DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id.table_name, metadata); setColumns(std::move(metadata.columns)); } diff --git a/dbms/src/Storages/StorageMerge.cpp b/dbms/src/Storages/StorageMerge.cpp index 1455556f366..15c90106824 100644 --- a/dbms/src/Storages/StorageMerge.cpp +++ b/dbms/src/Storages/StorageMerge.cpp @@ -399,7 +399,7 @@ StorageMerge::StorageListWithLocks StorageMerge::getSelectedTables(const ASTPtr DatabaseTablesIteratorPtr StorageMerge::getDatabaseIterator(const Context & context) const { checkStackSize(); - auto database = context.getDatabase(source_database); + auto database = DatabaseCatalog::instance().getDatabase(source_database, context); auto table_name_match = [this](const String & table_name_) { return table_name_regexp.match(table_name_); }; return database->getTablesIterator(global_context, table_name_match); } @@ -425,7 +425,7 @@ void StorageMerge::alter( StorageInMemoryMetadata storage_metadata = getInMemoryMetadata(); params.apply(storage_metadata); - context.getDatabase(table_id.database_name)->alterTable(context, table_id.table_name, storage_metadata); + DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id.table_name, storage_metadata); setColumns(storage_metadata.columns); } diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index d8b25627a7e..f9131ac3a23 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -272,7 +272,7 @@ void StorageMergeTree::alter( { lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); - context.getDatabase(table_id.database_name)->alterTable(context, table_id.table_name, metadata); + DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id.table_name, metadata); update_metadata(); } @@ -289,7 +289,7 @@ void StorageMergeTree::alter( lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); - context.getDatabase(table_id.database_name)->alterTable(context, table_id.table_name, metadata); + DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id.table_name, metadata); update_metadata(); diff --git a/dbms/src/Storages/StorageNull.cpp b/dbms/src/Storages/StorageNull.cpp index bbb620132c8..3de86079b9b 100644 --- a/dbms/src/Storages/StorageNull.cpp +++ b/dbms/src/Storages/StorageNull.cpp @@ -51,7 +51,7 @@ void StorageNull::alter( StorageInMemoryMetadata metadata = getInMemoryMetadata(); params.apply(metadata); - context.getDatabase(table_id.database_name)->alterTable(context, table_id.table_name, metadata); + DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id.table_name, metadata); setColumns(std::move(metadata.columns)); } diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index e3a38f54219..50041982943 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -541,7 +541,7 @@ void StorageReplicatedMergeTree::setTableStructure(ColumnsDescription new_column } auto table_id = getStorageID(); - global_context.getDatabase(table_id.database_name)->alterTable(global_context, table_id.table_name, metadata); + DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(global_context, table_id.table_name, metadata); /// Even if the primary/sorting keys didn't change we must reinitialize it /// because primary key column types might have changed. @@ -3231,7 +3231,7 @@ void StorageReplicatedMergeTree::alter( changeSettings(metadata.settings_ast, table_lock_holder); - global_context.getDatabase(table_id.database_name)->alterTable(query_context, table_id.table_name, metadata); + DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(query_context, table_id.table_name, metadata); return; } @@ -3309,7 +3309,7 @@ void StorageReplicatedMergeTree::alter( auto old_metadata = getInMemoryMetadata(); old_metadata.settings_ast = metadata.settings_ast; changeSettings(metadata.settings_ast, table_lock_holder); - global_context.getDatabase(table_id.database_name)->alterTable(query_context, table_id.table_name, old_metadata); + DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(query_context, table_id.table_name, old_metadata); } /// Modify shared metadata nodes in ZooKeeper. diff --git a/dbms/src/Storages/System/StorageSystemColumns.cpp b/dbms/src/Storages/System/StorageSystemColumns.cpp index 0fc85898264..6c568e1cba3 100644 --- a/dbms/src/Storages/System/StorageSystemColumns.cpp +++ b/dbms/src/Storages/System/StorageSystemColumns.cpp @@ -267,7 +267,7 @@ Pipes StorageSystemColumns::readWithProcessors( Pipes pipes; { - Databases databases = context.getDatabases(); + Databases databases = DatabaseCatalog::instance().getDatabases(); /// Add `database` column. MutableColumnPtr database_column_mut = ColumnString::create(); diff --git a/dbms/src/Storages/System/StorageSystemDatabases.cpp b/dbms/src/Storages/System/StorageSystemDatabases.cpp index ee193548921..4588fd28482 100644 --- a/dbms/src/Storages/System/StorageSystemDatabases.cpp +++ b/dbms/src/Storages/System/StorageSystemDatabases.cpp @@ -23,7 +23,7 @@ void StorageSystemDatabases::fillData(MutableColumns & res_columns, const Contex const auto access_rights = context.getAccessRights(); const bool check_access_for_databases = !access_rights->isGranted(AccessType::SHOW); - auto databases = context.getDatabases(); + auto databases = DatabaseCatalog::instance().getDatabases(); for (const auto & database : databases) { if (check_access_for_databases && !access_rights->isGranted(AccessType::SHOW, database.first)) diff --git a/dbms/src/Storages/System/StorageSystemGraphite.cpp b/dbms/src/Storages/System/StorageSystemGraphite.cpp index b58762b3860..b5023cff7c3 100644 --- a/dbms/src/Storages/System/StorageSystemGraphite.cpp +++ b/dbms/src/Storages/System/StorageSystemGraphite.cpp @@ -27,7 +27,7 @@ NamesAndTypesList StorageSystemGraphite::getNamesAndTypes() */ StorageSystemGraphite::Configs StorageSystemGraphite::getConfigs(const Context & context) const { - const Databases databases = context.getDatabases(); + const Databases databases = DatabaseCatalog::instance().getDatabases(); Configs graphite_configs; for (const auto & db : databases) diff --git a/dbms/src/Storages/System/StorageSystemMutations.cpp b/dbms/src/Storages/System/StorageSystemMutations.cpp index 279a553d5f9..51c5bd47c6d 100644 --- a/dbms/src/Storages/System/StorageSystemMutations.cpp +++ b/dbms/src/Storages/System/StorageSystemMutations.cpp @@ -42,7 +42,7 @@ void StorageSystemMutations::fillData(MutableColumns & res_columns, const Contex /// Collect a set of *MergeTree tables. std::map> merge_tree_tables; - for (const auto & db : context.getDatabases()) + for (const auto & db : DatabaseCatalog::instance().getDatabases()) { /// Lazy database can not contain MergeTree tables if (db.second->getEngineName() == "Lazy") diff --git a/dbms/src/Storages/System/StorageSystemPartsBase.cpp b/dbms/src/Storages/System/StorageSystemPartsBase.cpp index 2fdb28d62e1..a393d862b54 100644 --- a/dbms/src/Storages/System/StorageSystemPartsBase.cpp +++ b/dbms/src/Storages/System/StorageSystemPartsBase.cpp @@ -76,7 +76,7 @@ StoragesInfoStream::StoragesInfoStream(const SelectQueryInfo & query_info, const const bool check_access_for_tables = !access_rights->isGranted(AccessType::SHOW); { - Databases databases = context.getDatabases(); + Databases databases = DatabaseCatalog::instance().getDatabases(); /// Add column 'database'. MutableColumnPtr database_column_mut = ColumnString::create(); diff --git a/dbms/src/Storages/System/StorageSystemReplicas.cpp b/dbms/src/Storages/System/StorageSystemReplicas.cpp index fb3f117cfca..5e86dbc4e1d 100644 --- a/dbms/src/Storages/System/StorageSystemReplicas.cpp +++ b/dbms/src/Storages/System/StorageSystemReplicas.cpp @@ -69,7 +69,7 @@ Pipes StorageSystemReplicas::readWithProcessors( /// We collect a set of replicated tables. std::map> replicated_tables; - for (const auto & db : context.getDatabases()) + for (const auto & db : DatabaseCatalog::instance().getDatabases()) { /// Lazy database can not contain replicated tables if (db.second->getEngineName() == "Lazy") diff --git a/dbms/src/Storages/System/StorageSystemReplicationQueue.cpp b/dbms/src/Storages/System/StorageSystemReplicationQueue.cpp index 68627987bf4..50bc9f8cb62 100644 --- a/dbms/src/Storages/System/StorageSystemReplicationQueue.cpp +++ b/dbms/src/Storages/System/StorageSystemReplicationQueue.cpp @@ -53,7 +53,7 @@ void StorageSystemReplicationQueue::fillData(MutableColumns & res_columns, const const bool check_access_for_databases = !access_rights->isGranted(AccessType::SHOW); std::map> replicated_tables; - for (const auto & db : context.getDatabases()) + for (const auto & db : DatabaseCatalog::instance().getDatabases()) { /// Lazy database can not contain replicated tables if (db.second->getEngineName() == "Lazy") diff --git a/dbms/src/Storages/System/StorageSystemTables.cpp b/dbms/src/Storages/System/StorageSystemTables.cpp index 5b39a0f0bf7..5b2eca1ec01 100644 --- a/dbms/src/Storages/System/StorageSystemTables.cpp +++ b/dbms/src/Storages/System/StorageSystemTables.cpp @@ -57,7 +57,7 @@ StorageSystemTables::StorageSystemTables(const std::string & name_) static ColumnPtr getFilteredDatabases(const ASTPtr & query, const Context & context) { MutableColumnPtr column = ColumnString::create(); - for (const auto & db : context.getDatabases()) + for (const auto & db : DatabaseCatalog::instance().getDatabases()) column->insert(db.first); Block block { ColumnWithTypeAndName(std::move(column), std::make_shared(), "database") }; @@ -118,7 +118,8 @@ protected: while (database_idx < databases->size() && (!tables_it || !tables_it->isValid())) { database_name = databases->getDataAt(database_idx).toString(); - database = context.tryGetDatabase(database_name); + //FIXME access is not checked + database = DatabaseCatalog::instance().tryGetDatabase(database_name); if (!database) { diff --git a/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp b/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp index 57729218a40..856de3d89ca 100644 --- a/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp +++ b/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp @@ -32,7 +32,8 @@ struct State registerFunctions(); DatabasePtr database = std::make_shared("test"); database->attachTable("table", StorageMemory::create(StorageID("test", "table"), ColumnsDescription{columns}, ConstraintsDescription{})); - context.addDatabase("test", database); + context.makeGlobalContext(); + DatabaseCatalog::instance().attachDatabase("test", database); context.setCurrentDatabase("test"); } }; diff --git a/dbms/src/TableFunctions/TableFunctionMerge.cpp b/dbms/src/TableFunctions/TableFunctionMerge.cpp index b0c22c96117..eedc6dda10a 100644 --- a/dbms/src/TableFunctions/TableFunctionMerge.cpp +++ b/dbms/src/TableFunctions/TableFunctionMerge.cpp @@ -31,7 +31,7 @@ static NamesAndTypesList chooseColumns(const String & source_database, const Str StoragePtr any_table; { - auto database = context.getDatabase(source_database); + auto database = DatabaseCatalog::instance().getDatabase(source_database); auto iterator = database->getTablesIterator(context, table_name_match); if (iterator->isValid()) From 45338ad9ca8fab5bad1e92de6d9717822cb4feb9 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 10 Feb 2020 21:19:35 +0300 Subject: [PATCH 017/712] fixes --- dbms/src/Interpreters/Context.cpp | 39 ++++++++++++------- dbms/src/Interpreters/Context.h | 21 +++++----- dbms/src/Interpreters/DatabaseCatalog.cpp | 20 +++------- dbms/src/Interpreters/DatabaseCatalog.h | 3 +- .../src/Interpreters/InterpreterDropQuery.cpp | 1 + 5 files changed, 45 insertions(+), 39 deletions(-) diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index ba7de8f877d..b57442d19e5 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -95,7 +95,6 @@ namespace ErrorCodes extern const int PARTITION_SIZE_EXCEEDS_MAX_DROP_SIZE_LIMIT; extern const int SESSION_NOT_FOUND; extern const int SESSION_IS_LOCKED; - extern const int CANNOT_GET_CREATE_TABLE_QUERY; extern const int LOGICAL_ERROR; extern const int SCALAR_ALREADY_EXISTS; extern const int UNKNOWN_SCALAR; @@ -765,7 +764,7 @@ Dependencies Context::getDependencies(const StorageID & from) const { auto lock = getLock(); StorageID resolved = resolveStorageIDUnlocked(from); - ViewDependencies::const_iterator iter = shared->view_dependencies.find(resolved); + auto iter = shared->view_dependencies.find(resolved); if (iter == shared->view_dependencies.end()) return {}; @@ -774,8 +773,7 @@ Dependencies Context::getDependencies(const StorageID & from) const bool Context::isTableExist(const String & database_name, const String & table_name) const { - //FIXME do we need resolve temporary tables here? - auto table_id = resolveStorageID({database_name, table_name}); + auto table_id = resolveStorageID({database_name, table_name}, StorageNamespace::Ordinary); return DatabaseCatalog::instance().isTableExist(table_id, *this); } @@ -2046,30 +2044,45 @@ void Context::resetInputCallbacks() input_blocks_reader = {}; } -StorageID Context::resolveStorageID(StorageID storage_id) const +StorageID Context::resolveStorageID(StorageID storage_id, StorageNamespace where) const { auto lock = getLock(); - return resolveStorageIDUnlocked(std::move(storage_id)); + return resolveStorageIDUnlocked(std::move(storage_id), where); } -StorageID Context::resolveStorageIDUnlocked(StorageID storage_id) const +StorageID Context::resolveStorageIDUnlocked(StorageID storage_id, StorageNamespace where) const { if (storage_id.uuid != UUIDHelpers::Nil) - { - //TODO maybe update table and db name? - //TODO add flag `resolved` to StorageID and check access rights if it was not previously resolved return storage_id; + + bool look_for_external_table = where & StorageNamespace::External; + bool in_current_database = where & StorageNamespace::CurrentDatabase; + bool in_specified_database = where & StorageNamespace::Global; + + if (!storage_id.database_name.empty()) + { + if (in_specified_database) + return storage_id; + throw Exception("External and temporary tables have no database, but " + + storage_id.database_name + " is specified", ErrorCodes::UNKNOWN_TABLE); } - if (storage_id.database_name.empty()) + + if (look_for_external_table) { auto it = external_tables_mapping.find(storage_id.getTableName()); if (it != external_tables_mapping.end()) - return it->second->getGlobalTableID(); /// Do not check access rights for session-local table + return it->second->getGlobalTableID(); + } + + if (in_current_database) + { if (current_database.empty()) throw Exception("Default database is not selected", ErrorCodes::UNKNOWN_DATABASE); storage_id.database_name = current_database; + return storage_id; } - return storage_id; + + throw Exception("Cannot resolve database name for table " + storage_id.getNameForLogs(), ErrorCodes::UNKNOWN_TABLE); } diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index 3ee96829e53..5d4a37bfdb6 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -145,8 +145,6 @@ struct SubscriptionForUserChange }; struct TemporaryTableHolder; -class DatabaseCatalog; -using DatabaseCatalogPtr = std::shared_ptr; /** A set of known objects that can be used in the query. * Consists of a shared part (always common to all sessions and queries) @@ -197,9 +195,6 @@ private: using SampleBlockCache = std::unordered_map; mutable SampleBlockCache sample_block_cache; - using DatabasePtr = std::shared_ptr; - using Databases = std::map>; - NameToNameMap query_parameters; /// Dictionary with query parameters for prepared statements. /// (key=name, value) @@ -306,11 +301,19 @@ public: bool isDictionaryExists(const String & database_name, const String & dictionary_name) const; bool isExternalTableExist(const String & table_name) const; + enum StorageNamespace + { + Global = 1u, /// Database name must be specified + CurrentDatabase = 2u, /// Use current database + Ordinary = Global | CurrentDatabase, /// If database name is not specified, use current database + External = 4u, /// Try get external table + All = External | Ordinary /// If database name is not specified, try get external table, + /// if external table not found use current database. + }; + String resolveDatabase(const String & database_name) const; - String resolveDatabaseAndCheckAccess(const String & database_name) const; - //StorageID resolveDatabase(StorageID table_id) const; - StorageID resolveStorageID(StorageID storage_id) const; - StorageID resolveStorageIDUnlocked(StorageID storage_id) const; + StorageID resolveStorageID(StorageID storage_id, StorageNamespace where = StorageNamespace::All) const; + StorageID resolveStorageIDUnlocked(StorageID storage_id, StorageNamespace where = StorageNamespace::All) const; const Scalars & getScalars() const; const Block & getScalar(const String & name) const; diff --git a/dbms/src/Interpreters/DatabaseCatalog.cpp b/dbms/src/Interpreters/DatabaseCatalog.cpp index 59e4368c381..e1127efb965 100644 --- a/dbms/src/Interpreters/DatabaseCatalog.cpp +++ b/dbms/src/Interpreters/DatabaseCatalog.cpp @@ -65,13 +65,6 @@ DatabaseAndTable DatabaseCatalog::tryGetByUUID(const UUID & uuid) const return it->second; } -//String DatabaseCatalog::resolveDatabase(const String & database_name, const String & current_database) -//{ -// String res = database_name.empty() ? current_database : database_name; -// if (res.empty()) -// throw Exception("Default database is not selected", ErrorCodes::UNKNOWN_DATABASE); -// return res; -//} StoragePtr DatabaseCatalog::getTable(const StorageID & table_id, const Context & local_context, std::optional * exception) const { @@ -121,6 +114,7 @@ void DatabaseCatalog::assertDatabaseDoesntExist(const String & database_name) co void DatabaseCatalog::assertDatabaseExistsUnlocked(const String & database_name) const { + assert(!database_name.empty()); if (databases.end() == databases.find(database_name)) throw Exception("Database " + backQuoteIfNeed(database_name) + " doesn't exist", ErrorCodes::UNKNOWN_DATABASE); } @@ -128,13 +122,13 @@ void DatabaseCatalog::assertDatabaseExistsUnlocked(const String & database_name) void DatabaseCatalog::assertDatabaseDoesntExistUnlocked(const String & database_name) const { + assert(!database_name.empty()); if (databases.end() != databases.find(database_name)) throw Exception("Database " + backQuoteIfNeed(database_name) + " already exists.", ErrorCodes::DATABASE_ALREADY_EXISTS); } void DatabaseCatalog::attachDatabase(const String & database_name, const DatabasePtr & database) { - //local_context.checkDatabaseAccessRights(database_name); std::lock_guard lock{databases_mutex}; assertDatabaseDoesntExistUnlocked(database_name); databases[database_name] = database; @@ -143,18 +137,15 @@ void DatabaseCatalog::attachDatabase(const String & database_name, const Databas DatabasePtr DatabaseCatalog::detachDatabase(const String & database_name) { - //local_context.checkDatabaseAccessRights(database_name); std::lock_guard lock{databases_mutex}; - auto res = getDatabase(database_name); //FIXME locks order + assertDatabaseExistsUnlocked(database_name); + auto res = databases.find(database_name)->second; databases.erase(database_name); return res; } DatabasePtr DatabaseCatalog::getDatabase(const String & database_name) const { - assert(!database_name.empty()); - //String db = local_context.resolveDatabase(database_name); - //local_context.checkDatabaseAccessRights(db); //FIXME non-atomic std::lock_guard lock{databases_mutex}; assertDatabaseExistsUnlocked(database_name); return databases.find(database_name)->second; @@ -163,7 +154,6 @@ DatabasePtr DatabaseCatalog::getDatabase(const String & database_name) const DatabasePtr DatabaseCatalog::tryGetDatabase(const String & database_name) const { assert(!database_name.empty()); - //String db = local_context.resolveDatabase(database_name); std::lock_guard lock{databases_mutex}; auto it = databases.find(database_name); if (it == databases.end()) @@ -198,7 +188,7 @@ bool DatabaseCatalog::isTableExist(const DB::StorageID & table_id, const DB::Con void DatabaseCatalog::assertTableDoesntExist(const StorageID & table_id, const Context & context) const { - if (!isTableExist(table_id, context)) + if (isTableExist(table_id, context)) throw Exception("Table " + table_id.getNameForLogs() + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS); } diff --git a/dbms/src/Interpreters/DatabaseCatalog.h b/dbms/src/Interpreters/DatabaseCatalog.h index b5cae14e699..70cbb6b8050 100644 --- a/dbms/src/Interpreters/DatabaseCatalog.h +++ b/dbms/src/Interpreters/DatabaseCatalog.h @@ -18,12 +18,11 @@ struct StorageID; class Exception; using DatabasePtr = std::shared_ptr; using DatabaseAndTable = std::pair; +using Databases = std::map>; -//TODO make singleton? class DatabaseCatalog : boost::noncopyable { public: - using Databases = std::map>; static constexpr const char * TEMPORARY_DATABASE = "_temporary_and_external_tables"; static constexpr const char * SYSTEM_DATABASE = "system"; diff --git a/dbms/src/Interpreters/InterpreterDropQuery.cpp b/dbms/src/Interpreters/InterpreterDropQuery.cpp index 9f7ea306f10..9a92c0eee47 100644 --- a/dbms/src/Interpreters/InterpreterDropQuery.cpp +++ b/dbms/src/Interpreters/InterpreterDropQuery.cpp @@ -250,6 +250,7 @@ BlockIO InterpreterDropQuery::executeToDatabase(const String & database_name, AS context.checkAccess(AccessType::DETACH_DATABASE, database_name); DatabaseCatalog::instance().detachDatabase(database_name); database->shutdown(); + //FIXME someone may still use tables from database } else if (kind == ASTDropQuery::Kind::Drop) { From 18dd0c8f8cc7fe774e222234419edc364664ec86 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 10 Feb 2020 21:31:52 +0300 Subject: [PATCH 018/712] move DDLGuard to DatabaseCatalog --- dbms/src/Interpreters/Context.cpp | 38 ----------------- dbms/src/Interpreters/Context.h | 32 --------------- dbms/src/Interpreters/DatabaseCatalog.cpp | 28 +++++++++++++ dbms/src/Interpreters/DatabaseCatalog.h | 41 ++++++++++++++++++- .../Interpreters/InterpreterCreateQuery.cpp | 6 +-- .../src/Interpreters/InterpreterDropQuery.cpp | 6 +-- .../Interpreters/InterpreterRenameQuery.cpp | 5 ++- .../Interpreters/InterpreterSystemQuery.cpp | 2 +- 8 files changed, 78 insertions(+), 80 deletions(-) diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index b57442d19e5..80108fcbc15 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -90,7 +90,6 @@ namespace ErrorCodes extern const int THERE_IS_NO_SESSION; extern const int THERE_IS_NO_QUERY; extern const int NO_ELEMENTS_IN_CONFIG; - extern const int DDL_GUARD_IS_ACTIVE; extern const int TABLE_SIZE_EXCEEDS_MAX_DROP_SIZE_LIMIT; extern const int PARTITION_SIZE_EXCEEDS_MAX_DROP_SIZE_LIMIT; extern const int SESSION_NOT_FOUND; @@ -253,16 +252,6 @@ struct ContextShared bool shutdown_called = false; - /// Do not allow simultaneous execution of DDL requests on the same table. - /// database -> object -> (mutex, counter), counter: how many threads are running a query on the table at the same time - /// For the duration of the operation, an element is placed here, and an object is returned, - /// which deletes the element in the destructor when counter becomes zero. - /// In case the element already exists, waits, when query will be executed in other thread. See class DDLGuard below. - using DDLGuards = std::unordered_map; - DDLGuards ddl_guards; - /// If you capture mutex and ddl_guards_mutex, then you need to grab them strictly in this order. - mutable std::mutex ddl_guards_mutex; - Stopwatch uptime_watch; Context::ApplicationType application_type = Context::ApplicationType::SERVER; @@ -931,33 +920,6 @@ StoragePtr Context::getViewSource() return view_source; } - -DDLGuard::DDLGuard(Map & map_, std::unique_lock guards_lock_, const String & elem) - : map(map_), guards_lock(std::move(guards_lock_)) -{ - it = map.emplace(elem, Entry{std::make_unique(), 0}).first; - ++it->second.counter; - guards_lock.unlock(); - table_lock = std::unique_lock(*it->second.mutex); -} - -DDLGuard::~DDLGuard() -{ - guards_lock.lock(); - --it->second.counter; - if (!it->second.counter) - { - table_lock.unlock(); - map.erase(it); - } -} - -std::unique_ptr Context::getDDLGuard(const String & database, const String & table) const -{ - std::unique_lock lock(shared->ddl_guards_mutex); - return std::make_unique(shared->ddl_guards[database], std::move(lock), table); -} - ASTPtr Context::getCreateExternalTableQuery(const String & table_name) const { auto it = external_tables_mapping.find(table_name); diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index 5d4a37bfdb6..ceea7ad267b 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -76,7 +76,6 @@ class TraceLog; class MetricLog; struct MergeTreeSettings; class IDatabase; -class DDLGuard; class DDLWorker; class ITableFunction; class Block; @@ -333,9 +332,6 @@ public: void addViewSource(const StoragePtr & storage); StoragePtr getViewSource(); - /// Get an object that protects the table from concurrently executing multiple DDL operations. - std::unique_ptr getDDLGuard(const String & database, const String & table) const; - String getCurrentDatabase() const; String getCurrentQueryId() const; @@ -629,34 +625,6 @@ private: }; -/// Allows executing DDL query only in one thread. -/// Puts an element into the map, locks tables's mutex, counts how much threads run parallel query on the table, -/// when counter is 0 erases element in the destructor. -/// If the element already exists in the map, waits, when ddl query will be finished in other thread. -class DDLGuard -{ -public: - struct Entry - { - std::unique_ptr mutex; - UInt32 counter; - }; - - /// Element name -> (mutex, counter). - /// NOTE: using std::map here (and not std::unordered_map) to avoid iterator invalidation on insertion. - using Map = std::map; - - DDLGuard(Map & map_, std::unique_lock guards_lock_, const String & elem); - ~DDLGuard(); - -private: - Map & map; - Map::iterator it; - std::unique_lock guards_lock; - std::unique_lock table_lock; -}; - - class SessionCleaner { public: diff --git a/dbms/src/Interpreters/DatabaseCatalog.cpp b/dbms/src/Interpreters/DatabaseCatalog.cpp index e1127efb965..98f7145c392 100644 --- a/dbms/src/Interpreters/DatabaseCatalog.cpp +++ b/dbms/src/Interpreters/DatabaseCatalog.cpp @@ -15,6 +15,7 @@ namespace ErrorCodes extern const int UNKNOWN_TABLE; extern const int TABLE_ALREADY_EXISTS; extern const int DATABASE_ALREADY_EXISTS; + extern const int DDL_GUARD_IS_ACTIVE; } @@ -234,6 +235,33 @@ DatabasePtr DatabaseCatalog::getDatabase(const String & database_name, const Con return getDatabase(resolved_database); } +std::unique_ptr DatabaseCatalog::getDDLGuard(const String & database, const String & table) +{ + std::unique_lock lock(ddl_guards_mutex); + return std::make_unique(ddl_guards[database], std::move(lock), table); +} + + +DDLGuard::DDLGuard(Map & map_, std::unique_lock guards_lock_, const String & elem) + : map(map_), guards_lock(std::move(guards_lock_)) +{ + it = map.emplace(elem, Entry{std::make_unique(), 0}).first; + ++it->second.counter; + guards_lock.unlock(); + table_lock = std::unique_lock(*it->second.mutex); +} + +DDLGuard::~DDLGuard() +{ + guards_lock.lock(); + --it->second.counter; + if (!it->second.counter) + { + table_lock.unlock(); + map.erase(it); + } +} + } diff --git a/dbms/src/Interpreters/DatabaseCatalog.h b/dbms/src/Interpreters/DatabaseCatalog.h index 70cbb6b8050..1a25800737a 100644 --- a/dbms/src/Interpreters/DatabaseCatalog.h +++ b/dbms/src/Interpreters/DatabaseCatalog.h @@ -20,6 +20,35 @@ using DatabasePtr = std::shared_ptr; using DatabaseAndTable = std::pair; using Databases = std::map>; + +/// Allows executing DDL query only in one thread. +/// Puts an element into the map, locks tables's mutex, counts how much threads run parallel query on the table, +/// when counter is 0 erases element in the destructor. +/// If the element already exists in the map, waits, when ddl query will be finished in other thread. +class DDLGuard +{ +public: + struct Entry + { + std::unique_ptr mutex; + UInt32 counter; + }; + + /// Element name -> (mutex, counter). + /// NOTE: using std::map here (and not std::unordered_map) to avoid iterator invalidation on insertion. + using Map = std::map; + + DDLGuard(Map & map_, std::unique_lock guards_lock_, const String & elem); + ~DDLGuard(); + +private: + Map & map; + Map::iterator it; + std::unique_lock guards_lock; + std::unique_lock table_lock; +}; + + class DatabaseCatalog : boost::noncopyable { public: @@ -34,6 +63,8 @@ public: void loadDatabases(); void shutdown(); + /// Get an object that protects the table from concurrently executing multiple DDL operations. + std::unique_ptr getDDLGuard(const String & database, const String & table); //static String resolveDatabase(const String & database_name, const String & current_database); void assertDatabaseExists(const String & database_name) const; @@ -88,7 +119,15 @@ private: UUIDToStorageMap uuid_map; + /// Do not allow simultaneous execution of DDL requests on the same table. + /// database -> object -> (mutex, counter), counter: how many threads are running a query on the table at the same time + /// For the duration of the operation, an element is placed here, and an object is returned, + /// which deletes the element in the destructor when counter becomes zero. + /// In case the element already exists, waits, when query will be executed in other thread. See class DDLGuard below. + using DDLGuards = std::unordered_map; + DDLGuards ddl_guards; + /// If you capture mutex and ddl_guards_mutex, then you need to grab them strictly in this order. + mutable std::mutex ddl_guards_mutex; }; - } diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp index ab828fc7306..4aee64b0c42 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp @@ -83,7 +83,7 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) { String database_name = create.database; - auto guard = context.getDDLGuard(database_name, ""); + auto guard = DatabaseCatalog::instance().getDDLGuard(database_name, ""); /// Database can be created before or it can be created concurrently in another thread, while we were waiting in DDLGuard if (DatabaseCatalog::instance().isDatabaseExist(database_name)) @@ -595,7 +595,7 @@ bool InterpreterCreateQuery::doCreateTable(const ASTCreateQuery & create, /** If the request specifies IF NOT EXISTS, we allow concurrent CREATE queries (which do nothing). * If table doesnt exist, one thread is creating table, while others wait in DDLGuard. */ - guard = context.getDDLGuard(create.database, table_name); + guard = DatabaseCatalog::instance().getDDLGuard(create.database, table_name); /// Table can be created before or it can be created concurrently in another thread, while we were waiting in DDLGuard. if (database->isTableExist(context, table_name)) @@ -689,7 +689,7 @@ BlockIO InterpreterCreateQuery::createDictionary(ASTCreateQuery & create) create.database = context.resolveDatabase(create.database); const String & database_name = create.database; - auto guard = context.getDDLGuard(database_name, dictionary_name); + auto guard = DatabaseCatalog::instance().getDDLGuard(database_name, dictionary_name); DatabasePtr database = DatabaseCatalog::instance().getDatabase(database_name); if (database->isDictionaryExist(context, dictionary_name)) diff --git a/dbms/src/Interpreters/InterpreterDropQuery.cpp b/dbms/src/Interpreters/InterpreterDropQuery.cpp index 9a92c0eee47..af32089c7a9 100644 --- a/dbms/src/Interpreters/InterpreterDropQuery.cpp +++ b/dbms/src/Interpreters/InterpreterDropQuery.cpp @@ -78,7 +78,7 @@ BlockIO InterpreterDropQuery::executeToTable( String database_name = context.resolveDatabase(database_name_); - auto ddl_guard = (!no_ddl_lock ? context.getDDLGuard(database_name, table_name) : nullptr); + auto ddl_guard = (!no_ddl_lock ? DatabaseCatalog::instance().getDDLGuard(database_name, table_name) : nullptr); auto [database, table] = tryGetDatabaseAndTable(database_name, table_name, if_exists); @@ -170,7 +170,7 @@ BlockIO InterpreterDropQuery::executeToDictionary( String database_name = context.resolveDatabase(database_name_); - auto ddl_guard = (!no_ddl_lock ? context.getDDLGuard(database_name, dictionary_name) : nullptr); + auto ddl_guard = (!no_ddl_lock ? DatabaseCatalog::instance().getDDLGuard(database_name, dictionary_name) : nullptr); DatabasePtr database = tryGetDatabase(database_name, if_exists); @@ -237,7 +237,7 @@ BlockIO InterpreterDropQuery::executeToTemporaryTable(const String & table_name, BlockIO InterpreterDropQuery::executeToDatabase(const String & database_name, ASTDropQuery::Kind kind, bool if_exists) { - auto ddl_guard = context.getDDLGuard(database_name, ""); + auto ddl_guard = DatabaseCatalog::instance().getDDLGuard(database_name, ""); if (auto database = tryGetDatabase(database_name, if_exists)) { diff --git a/dbms/src/Interpreters/InterpreterRenameQuery.cpp b/dbms/src/Interpreters/InterpreterRenameQuery.cpp index 150fca16b53..820676fa2d8 100644 --- a/dbms/src/Interpreters/InterpreterRenameQuery.cpp +++ b/dbms/src/Interpreters/InterpreterRenameQuery.cpp @@ -84,11 +84,12 @@ BlockIO InterpreterRenameQuery::execute() table_guards[to]; } + auto & database_catalog = DatabaseCatalog::instance(); + /// Must do it in consistent order. for (auto & table_guard : table_guards) - table_guard.second = context.getDDLGuard(table_guard.first.database_name, table_guard.first.table_name); + table_guard.second = database_catalog.getDDLGuard(table_guard.first.database_name, table_guard.first.table_name); - auto & database_catalog = DatabaseCatalog::instance(); for (auto & elem : descriptions) { database_catalog.assertTableDoesntExist(StorageID(elem.to_database_name, elem.to_table_name), context); diff --git a/dbms/src/Interpreters/InterpreterSystemQuery.cpp b/dbms/src/Interpreters/InterpreterSystemQuery.cpp index 6ac9da8f3d2..ca6de13d0fd 100644 --- a/dbms/src/Interpreters/InterpreterSystemQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSystemQuery.cpp @@ -315,7 +315,7 @@ StoragePtr InterpreterSystemQuery::tryRestartReplica(const String & database_nam context.checkAccess(AccessType::RESTART_REPLICA, database_name, table_name); auto database = DatabaseCatalog::instance().getDatabase(database_name, system_context); - auto table_ddl_guard = system_context.getDDLGuard(database_name, table_name); + auto table_ddl_guard = DatabaseCatalog::instance().getDDLGuard(database_name, table_name); ASTPtr create_ast; /// Detach actions From 3c5d6cf6f1e1af3c9c19ffa1b57bbad8efa9b6e1 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 11 Feb 2020 20:25:26 +0300 Subject: [PATCH 019/712] fix --- dbms/src/Interpreters/Context.cpp | 2 +- dbms/src/Interpreters/DatabaseCatalog.cpp | 19 +++++++++++-------- dbms/src/Interpreters/DatabaseCatalog.h | 2 +- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index 80108fcbc15..7be0ecbccbc 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -770,7 +770,7 @@ bool Context::isDictionaryExists(const String & database_name, const String & di { auto lock = getLock(); String db = resolveDatabase(database_name); - auto db_ptr = DatabaseCatalog::instance().tryGetDatabase(database_name); + auto db_ptr = DatabaseCatalog::instance().tryGetDatabase(db); return db_ptr && db_ptr->isDictionaryExist(*this, dictionary_name); } diff --git a/dbms/src/Interpreters/DatabaseCatalog.cpp b/dbms/src/Interpreters/DatabaseCatalog.cpp index 98f7145c392..c83e8dfcf54 100644 --- a/dbms/src/Interpreters/DatabaseCatalog.cpp +++ b/dbms/src/Interpreters/DatabaseCatalog.cpp @@ -83,17 +83,20 @@ StoragePtr DatabaseCatalog::getTable(const StorageID & table_id, const Context & // return db_and_table.second; //} - std::lock_guard _lock{databases_mutex}; - - auto it = databases.find(table_id.getDatabaseName()); - if (databases.end() == it) + DatabasePtr database; { - if (exception) - exception->emplace("Database " + backQuoteIfNeed(table_id.getDatabaseName()) + " doesn't exist", ErrorCodes::UNKNOWN_DATABASE); - return {}; + std::lock_guard _lock{databases_mutex}; + auto it = databases.find(table_id.getDatabaseName()); + if (databases.end() == it) + { + if (exception) + exception->emplace("Database " + backQuoteIfNeed(table_id.getDatabaseName()) + " doesn't exist", + ErrorCodes::UNKNOWN_DATABASE); + return {}; + } + database = it->second; } - auto database = it->second; auto table = database->tryGetTable(local_context, table_id.table_name); if (!table && exception) exception->emplace("Table " + table_id.getNameForLogs() + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE); diff --git a/dbms/src/Interpreters/DatabaseCatalog.h b/dbms/src/Interpreters/DatabaseCatalog.h index 1a25800737a..871cd5a0ddd 100644 --- a/dbms/src/Interpreters/DatabaseCatalog.h +++ b/dbms/src/Interpreters/DatabaseCatalog.h @@ -113,7 +113,7 @@ private: private: //[[maybe_unused]] Context & global_context; - mutable std::mutex databases_mutex; + mutable std::recursive_mutex databases_mutex; //const String default_database; Databases databases; UUIDToStorageMap uuid_map; From 2e6796e7d7d0bae8f4f5268e12654bf062807c15 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 12 Feb 2020 19:54:26 +0300 Subject: [PATCH 020/712] move ViewDependencies to DatabaseCatalog --- .../PushingToViewsBlockOutputStream.cpp | 2 +- dbms/src/Interpreters/Context.cpp | 44 ------------------- dbms/src/Interpreters/Context.h | 15 ------- dbms/src/Interpreters/DatabaseCatalog.cpp | 37 +++++++++++++++- dbms/src/Interpreters/DatabaseCatalog.h | 22 +++++++++- dbms/src/Storages/IStorage.h | 3 -- dbms/src/Storages/Kafka/StorageKafka.cpp | 29 ++++++------ dbms/src/Storages/Kafka/StorageKafka.h | 2 - .../src/Storages/LiveView/StorageLiveView.cpp | 10 ++--- dbms/src/Storages/StorageMaterializedView.cpp | 30 ++++--------- .../Storages/System/StorageSystemTables.cpp | 2 +- 11 files changed, 85 insertions(+), 111 deletions(-) diff --git a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp index 260ecf6bdb9..3efa098423a 100644 --- a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp +++ b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp @@ -32,7 +32,7 @@ PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream( bool disable_deduplication_for_children = !no_destination && storage->supportsDeduplication(); auto table_id = storage->getStorageID(); - Dependencies dependencies = context.getDependencies(table_id); + Dependencies dependencies = DatabaseCatalog::instance().getDependencies(table_id); /// We need special context for materialized views insertions if (!dependencies.empty()) diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index 7be0ecbccbc..eac208c7086 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -193,7 +193,6 @@ struct ContextShared mutable MarkCachePtr mark_cache; /// Cache of marks in compressed files. ProcessList process_list; /// Executing queries at the moment. MergeList merge_list; /// The list of executable merge (for (Replicated)?MergeTree) - ViewDependencies view_dependencies; /// Current dependencies ConfigurationPtr users_config; /// Config with the users, profiles and quotas sections. InterserverIOHandler interserver_io_handler; /// Handler for interserver communication. std::optional background_pool; /// The thread pool for the background work performed by the tables. @@ -717,49 +716,6 @@ void Context::setUser(const String & name, const String & password, const Poco:: calculateAccessRights(); } -void Context::addDependencyUnsafe(const StorageID & from, const StorageID & where) -{ - shared->view_dependencies[from].insert(where); - - // Notify table of dependencies change - auto table = tryGetTable(from); - if (table != nullptr) - table->updateDependencies(); -} - -void Context::addDependency(const StorageID & from, const StorageID & where) -{ - auto lock = getLock(); - addDependencyUnsafe(from, where); -} - -void Context::removeDependencyUnsafe(const StorageID & from, const StorageID & where) -{ - shared->view_dependencies[from].erase(where); - - // Notify table of dependencies change - auto table = tryGetTable(from); - if (table != nullptr) - table->updateDependencies(); -} - -void Context::removeDependency(const StorageID & from, const StorageID & where) -{ - auto lock = getLock(); - removeDependencyUnsafe(from, where); -} - -Dependencies Context::getDependencies(const StorageID & from) const -{ - auto lock = getLock(); - StorageID resolved = resolveStorageIDUnlocked(from); - auto iter = shared->view_dependencies.find(resolved); - if (iter == shared->view_dependencies.end()) - return {}; - - return Dependencies(iter->second.begin(), iter->second.end()); -} - bool Context::isTableExist(const String & database_name, const String & table_name) const { auto table_id = resolveStorageID({database_name, table_name}, StorageNamespace::Ordinary); diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index ceea7ad267b..a4cfdf7b70d 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -106,13 +106,6 @@ class CompiledExpressionCache; #endif -/// Table -> set of table-views that make SELECT from it. -using ViewDependencies = std::map>; -using Dependencies = std::vector; - -using TableAndCreateAST = std::pair; -using TableAndCreateASTs = std::map; - /// Callback for external tables initializer using ExternalTablesInitializer = std::function; @@ -287,14 +280,6 @@ public: ClientInfo & getClientInfo() { return client_info; } const ClientInfo & getClientInfo() const { return client_info; } - void addDependency(const StorageID & from, const StorageID & where); - void removeDependency(const StorageID & from, const StorageID & where); - Dependencies getDependencies(const StorageID & from) const; - - /// Functions where we can lock the context manually - void addDependencyUnsafe(const StorageID & from, const StorageID & where); - void removeDependencyUnsafe(const StorageID & from, const StorageID & where); - /// Checking the existence of the table/database. Database can be empty - in this case the current database is used. bool isTableExist(const String & database_name, const String & table_name) const; bool isDictionaryExists(const String & database_name, const String & dictionary_name) const; diff --git a/dbms/src/Interpreters/DatabaseCatalog.cpp b/dbms/src/Interpreters/DatabaseCatalog.cpp index c83e8dfcf54..d3a33190e1b 100644 --- a/dbms/src/Interpreters/DatabaseCatalog.cpp +++ b/dbms/src/Interpreters/DatabaseCatalog.cpp @@ -47,12 +47,13 @@ void DatabaseCatalog::shutdown() std::lock_guard lock(databases_mutex); - databases.clear(); for (auto & elem : uuid_map) { std::lock_guard map_lock(elem.mutex); elem.map.clear(); } + databases.clear(); + view_dependencies.clear(); } DatabaseAndTable DatabaseCatalog::tryGetByUUID(const UUID & uuid) const @@ -238,6 +239,40 @@ DatabasePtr DatabaseCatalog::getDatabase(const String & database_name, const Con return getDatabase(resolved_database); } + +void DatabaseCatalog::addDependency(const StorageID & from, const StorageID & where) +{ + std::lock_guard lock{databases_mutex}; + view_dependencies[from].insert(where); + +} + +void DatabaseCatalog::removeDependency(const StorageID & from, const StorageID & where) +{ + std::lock_guard lock{databases_mutex}; + view_dependencies[from].erase(where); +} + +Dependencies DatabaseCatalog::getDependencies(const StorageID & from) const +{ + std::lock_guard lock{databases_mutex}; + auto iter = view_dependencies.find(from); + if (iter == view_dependencies.end()) + return {}; + return Dependencies(iter->second.begin(), iter->second.end()); +} + +void +DatabaseCatalog::updateDependency(const StorageID & old_from, const StorageID & old_where, const StorageID & new_from, + const StorageID & new_where) +{ + std::lock_guard lock{databases_mutex}; + if (!old_from.empty()) + view_dependencies[old_from].erase(old_where); + if (!new_from.empty()) + view_dependencies[new_from].insert(new_where); +} + std::unique_ptr DatabaseCatalog::getDDLGuard(const String & database, const String & table) { std::unique_lock lock(ddl_guards_mutex); diff --git a/dbms/src/Interpreters/DatabaseCatalog.h b/dbms/src/Interpreters/DatabaseCatalog.h index 871cd5a0ddd..b3c3963c047 100644 --- a/dbms/src/Interpreters/DatabaseCatalog.h +++ b/dbms/src/Interpreters/DatabaseCatalog.h @@ -1,9 +1,12 @@ #pragma once #include +#include #include +#include #include #include #include +#include #include #include #include @@ -14,12 +17,15 @@ namespace DB class Context; class IDatabase; -struct StorageID; class Exception; + using DatabasePtr = std::shared_ptr; using DatabaseAndTable = std::pair; using Databases = std::map>; +/// Table -> set of table-views that make SELECT from it. +using ViewDependencies = std::map>; +using Dependencies = std::vector; /// Allows executing DDL query only in one thread. /// Puts an element into the map, locks tables's mutex, counts how much threads run parallel query on the table, @@ -92,8 +98,16 @@ public: StoragePtr getTable(const StorageID & table_id, const Context & local_context, std::optional * exception) const; + + void addDependency(const StorageID & from, const StorageID & where); + void removeDependency(const StorageID & from, const StorageID & where); + Dependencies getDependencies(const StorageID & from) const; + + /// For Materialized and Live View + void updateDependency(const StorageID & old_from, const StorageID & old_where,const StorageID & new_from, const StorageID & new_where); + private: - DatabaseCatalog() = default; + DatabaseCatalog() : log(&Poco::Logger::get("DatabaseCatalog")) {} void assertDatabaseExistsUnlocked(const String & database_name) const; void assertDatabaseDoesntExistUnlocked(const String & database_name) const; @@ -114,10 +128,14 @@ private: private: //[[maybe_unused]] Context & global_context; mutable std::recursive_mutex databases_mutex; + + ViewDependencies view_dependencies; /// Current dependencies + //const String default_database; Databases databases; UUIDToStorageMap uuid_map; + Poco::Logger * log; /// Do not allow simultaneous execution of DDL requests on the same table. /// database -> object -> (mutex, counter), counter: how many threads are running a query on the table at the same time diff --git a/dbms/src/Storages/IStorage.h b/dbms/src/Storages/IStorage.h index 29cdc0661cd..458ac074e4b 100644 --- a/dbms/src/Storages/IStorage.h +++ b/dbms/src/Storages/IStorage.h @@ -413,9 +413,6 @@ public: /// We do not use mutex because it is not very important that the size could change during the operation. virtual void checkPartitionCanBeDropped(const ASTPtr & /*partition*/) {} - /** Notify engine about updated dependencies for this storage. */ - virtual void updateDependencies() {} - /// Returns data paths if storage supports it, empty vector otherwise. virtual Strings getDataPaths() const { return {}; } diff --git a/dbms/src/Storages/Kafka/StorageKafka.cpp b/dbms/src/Storages/Kafka/StorageKafka.cpp index 6b0bab72bb0..bb8f295fd77 100644 --- a/dbms/src/Storages/Kafka/StorageKafka.cpp +++ b/dbms/src/Storages/Kafka/StorageKafka.cpp @@ -192,11 +192,6 @@ void StorageKafka::shutdown() task->deactivate(); } -void StorageKafka::updateDependencies() -{ - task->activateAndSchedule(); -} - void StorageKafka::pushReadBuffer(ConsumerBufferPtr buffer) { @@ -295,7 +290,7 @@ void StorageKafka::updateConfiguration(cppkafka::Configuration & conf) bool StorageKafka::checkDependencies(const StorageID & table_id) { // Check if all dependencies are attached - auto dependencies = global_context.getDependencies(table_id); + auto dependencies = DatabaseCatalog::instance().getDependencies(table_id); if (dependencies.size() == 0) return true; @@ -325,19 +320,21 @@ void StorageKafka::threadFunc() { auto table_id = getStorageID(); // Check if at least one direct dependency is attached - auto dependencies = global_context.getDependencies(table_id); - - // Keep streaming as long as there are attached views and streaming is not cancelled - while (!stream_cancelled && num_created_consumers > 0 && dependencies.size() > 0) + size_t dependencies_count = DatabaseCatalog::instance().getDependencies(table_id).size(); + if (dependencies_count) { - if (!checkDependencies(table_id)) - break; + // Keep streaming as long as there are attached views and streaming is not cancelled + while (!stream_cancelled && num_created_consumers > 0) + { + if (!checkDependencies(table_id)) + break; - LOG_DEBUG(log, "Started streaming to " << dependencies.size() << " attached views"); + LOG_DEBUG(log, "Started streaming to " << dependencies_count << " attached views"); - // Reschedule if not limited - if (!streamToViews()) - break; + // Reschedule if not limited + if (!streamToViews()) + break; + } } } catch (...) diff --git a/dbms/src/Storages/Kafka/StorageKafka.h b/dbms/src/Storages/Kafka/StorageKafka.h index bf710f58202..f37f94ce8e0 100644 --- a/dbms/src/Storages/Kafka/StorageKafka.h +++ b/dbms/src/Storages/Kafka/StorageKafka.h @@ -47,8 +47,6 @@ public: const ASTPtr & query, const Context & context) override; - void updateDependencies() override; - void pushReadBuffer(ConsumerBufferPtr buf); ConsumerBufferPtr popReadBuffer(); ConsumerBufferPtr popReadBuffer(std::chrono::milliseconds timeout); diff --git a/dbms/src/Storages/LiveView/StorageLiveView.cpp b/dbms/src/Storages/LiveView/StorageLiveView.cpp index 20c1678995d..9824db87416 100644 --- a/dbms/src/Storages/LiveView/StorageLiveView.cpp +++ b/dbms/src/Storages/LiveView/StorageLiveView.cpp @@ -260,7 +260,7 @@ StorageLiveView::StorageLiveView( auto inner_query_tmp = inner_query->clone(); select_table_id = extractDependentTable(inner_query_tmp, global_context, table_id_.table_name, inner_subquery); - global_context.addDependency(select_table_id, table_id_); + DatabaseCatalog::instance().addDependency(select_table_id, table_id_); is_temporary = query.temporary; temporary_live_view_timeout = local_context.getSettingsRef().temporary_live_view_timeout.totalSeconds(); @@ -372,7 +372,7 @@ bool StorageLiveView::getNewBlocks() void StorageLiveView::checkTableCanBeDropped() const { auto table_id = getStorageID(); - Dependencies dependencies = global_context.getDependencies(table_id); + Dependencies dependencies = DatabaseCatalog::instance().getDependencies(table_id); if (!dependencies.empty()) { StorageID dependent_table_id = dependencies.front(); @@ -399,7 +399,7 @@ void StorageLiveView::noUsersThread(std::shared_ptr storage, co return; if (storage->hasUsers()) return; - if (!storage->global_context.getDependencies(table_id).empty()) + if (!DatabaseCatalog::instance().getDependencies(table_id).empty()) continue; drop_table = true; } @@ -471,7 +471,7 @@ void StorageLiveView::startup() void StorageLiveView::shutdown() { - global_context.removeDependency(select_table_id, getStorageID()); + DatabaseCatalog::instance().removeDependency(select_table_id, getStorageID()); bool expected = false; if (!shutdown_called.compare_exchange_strong(expected, true)) return; @@ -503,7 +503,7 @@ StorageLiveView::~StorageLiveView() void StorageLiveView::drop(TableStructureWriteLockHolder &) { auto table_id = getStorageID(); - global_context.removeDependency(select_table_id, table_id); + DatabaseCatalog::instance().removeDependency(select_table_id, table_id); std::lock_guard lock(mutex); is_dropped = true; diff --git a/dbms/src/Storages/StorageMaterializedView.cpp b/dbms/src/Storages/StorageMaterializedView.cpp index 4c47a1d4691..f81a02e2675 100644 --- a/dbms/src/Storages/StorageMaterializedView.cpp +++ b/dbms/src/Storages/StorageMaterializedView.cpp @@ -150,7 +150,7 @@ StorageMaterializedView::StorageMaterializedView( } if (!select_table_id.empty()) - global_context.addDependency(select_table_id, getStorageID()); + DatabaseCatalog::instance().addDependency(select_table_id, getStorageID()); } NameAndTypePair StorageMaterializedView::getColumn(const String & column_name) const @@ -228,7 +228,7 @@ void StorageMaterializedView::drop(TableStructureWriteLockHolder &) { auto table_id = getStorageID(); if (!select_table_id.empty()) - global_context.removeDependency(select_table_id, table_id); + DatabaseCatalog::instance().removeDependency(select_table_id, table_id); if (has_inner_table && tryGetTargetTable()) executeDropQuery(ASTDropQuery::Kind::Drop, global_context, target_table_id); @@ -277,20 +277,11 @@ void StorageMaterializedView::alter( checkAllowedQueries(select_query); auto new_select_table_id = extractDependentTableFromSelectQuery(select_query, context); + DatabaseCatalog::instance().updateDependency(select_table_id, table_id, new_select_table_id, table_id); - { - auto context_lock = global_context.getLock(); - - if (!select_table_id.empty()) - global_context.removeDependency(select_table_id, getStorageID()); - - if (!new_select_table_id.empty()) - global_context.addDependency(new_select_table_id, getStorageID()); - - select_table_id = new_select_table_id; - select = metadata.select; - inner_query = new_inner_query; - } + select_table_id = new_select_table_id; + select = metadata.select; + inner_query = new_inner_query; } /// end modify query @@ -360,19 +351,16 @@ void StorageMaterializedView::rename( target_table_id.table_name = new_target_table_name; } - auto lock = global_context.getLock(); - if (!select_table_id.empty()) - global_context.removeDependencyUnsafe(select_table_id, getStorageID()); + auto old_table_id = getStorageID(); IStorage::renameInMemory(new_database_name, new_table_name); - if (!select_table_id.empty()) - global_context.addDependencyUnsafe(select_table_id, getStorageID()); + DatabaseCatalog::instance().updateDependency(select_table_id, old_table_id, select_table_id, getStorageID()); } void StorageMaterializedView::shutdown() { /// Make sure the dependency is removed after DETACH TABLE if (!select_table_id.empty()) - global_context.removeDependency(select_table_id, getStorageID()); + DatabaseCatalog::instance().removeDependency(select_table_id, getStorageID()); } StoragePtr StorageMaterializedView::getTargetTable() const diff --git a/dbms/src/Storages/System/StorageSystemTables.cpp b/dbms/src/Storages/System/StorageSystemTables.cpp index 5b2eca1ec01..4e3c51d5772 100644 --- a/dbms/src/Storages/System/StorageSystemTables.cpp +++ b/dbms/src/Storages/System/StorageSystemTables.cpp @@ -275,7 +275,7 @@ protected: Array dependencies_database_name_array; if (columns_mask[src_index] || columns_mask[src_index + 1]) { - const auto dependencies = context.getDependencies(StorageID(database_name, table_name)); + const auto dependencies = DatabaseCatalog::instance().getDependencies(StorageID(database_name, table_name)); dependencies_table_name_array.reserve(dependencies.size()); dependencies_database_name_array.reserve(dependencies.size()); From b6039f8c50a13077b76ee6a6516292c8c9419ccf Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 12 Feb 2020 21:14:12 +0300 Subject: [PATCH 021/712] remove tryGetExternalTable --- dbms/programs/server/TCPHandler.cpp | 16 +++--- dbms/src/Interpreters/Context.cpp | 52 ++++++++++++------- dbms/src/Interpreters/Context.h | 18 +++---- dbms/src/Interpreters/DatabaseCatalog.cpp | 12 +++-- dbms/src/Interpreters/DatabaseCatalog.h | 2 +- dbms/src/Interpreters/ExternalTablesVisitor.h | 4 +- .../src/Interpreters/InterpreterDropQuery.cpp | 5 +- dbms/src/Storages/StorageID.h | 5 ++ 8 files changed, 70 insertions(+), 44 deletions(-) diff --git a/dbms/programs/server/TCPHandler.cpp b/dbms/programs/server/TCPHandler.cpp index 3cb00320b10..1174f3c465e 100644 --- a/dbms/programs/server/TCPHandler.cpp +++ b/dbms/programs/server/TCPHandler.cpp @@ -960,8 +960,8 @@ bool TCPHandler::receiveData(bool scalar) initBlockInput(); /// The name of the temporary table for writing data, default to empty string - String name; - readStringBinary(name, *in); + auto temporary_id = StorageID::createEmpty(); + readStringBinary(temporary_id.table_name, *in); /// Read one block from the network and write it down Block block = state.block_in->read(); @@ -969,22 +969,24 @@ bool TCPHandler::receiveData(bool scalar) if (block) { if (scalar) - query_context->addScalar(name, block); + query_context->addScalar(temporary_id.table_name, block); else { /// If there is an insert request, then the data should be written directly to `state.io.out`. /// Otherwise, we write the blocks in the temporary `external_table_name` table. if (!state.need_receive_data_for_insert && !state.need_receive_data_for_input) { + auto resolved = query_context->tryResolveStorageID(temporary_id, Context::ResolveExternal); StoragePtr storage; /// If such a table does not exist, create it. - if (!(storage = query_context->tryGetExternalTable(name))) + if (temporary_id.empty()) { NamesAndTypesList columns = block.getNamesAndTypesList(); - storage = StorageMemory::create(StorageID("_external", name), ColumnsDescription{columns}, ConstraintsDescription{}); + storage = StorageMemory::create(temporary_id, ColumnsDescription{columns}, ConstraintsDescription{}); storage->startup(); - query_context->addExternalTable(name, storage); - } + query_context->addExternalTable(temporary_id.table_name, storage); + } else + storage = DatabaseCatalog::instance().getTable(resolved, *query_context); /// The data will be written directly to the table. state.io.out = storage->write(ASTPtr(), *query_context); } diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index eac208c7086..54b50537ae5 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -718,7 +718,7 @@ void Context::setUser(const String & name, const String & password, const Poco:: bool Context::isTableExist(const String & database_name, const String & table_name) const { - auto table_id = resolveStorageID({database_name, table_name}, StorageNamespace::Ordinary); + auto table_id = resolveStorageID({database_name, table_name}, StorageNamespace::ResolveOrdinary); return DatabaseCatalog::instance().isTableExist(table_id, *this); } @@ -772,15 +772,6 @@ Tables Context::getExternalTables() const } -StoragePtr Context::tryGetExternalTable(const String & table_name) const -{ - auto it = external_tables_mapping.find(table_name); - if (external_tables_mapping.end() == it) - return StoragePtr(); - - return it->second->getTable(); -} - StoragePtr Context::getTable(const String & database_name, const String & table_name) const { return getTable(StorageID(database_name, table_name)); @@ -1965,24 +1956,43 @@ void Context::resetInputCallbacks() StorageID Context::resolveStorageID(StorageID storage_id, StorageNamespace where) const { auto lock = getLock(); - return resolveStorageIDUnlocked(std::move(storage_id), where); + std::optional exc; + auto resolved = resolveStorageIDImpl(std::move(storage_id), where, &exc); + if (exc) + throw *exc; + return resolved; } -StorageID Context::resolveStorageIDUnlocked(StorageID storage_id, StorageNamespace where) const +StorageID Context::tryResolveStorageID(StorageID storage_id, StorageNamespace where) const +{ + auto lock = getLock(); + return resolveStorageIDImpl(std::move(storage_id), where, nullptr); +} + +StorageID Context::resolveStorageIDImpl(StorageID storage_id, StorageNamespace where, std::optional * exception) const { if (storage_id.uuid != UUIDHelpers::Nil) return storage_id; - bool look_for_external_table = where & StorageNamespace::External; - bool in_current_database = where & StorageNamespace::CurrentDatabase; - bool in_specified_database = where & StorageNamespace::Global; + if (storage_id.empty()) + { + if (exception) + exception->emplace("Both table name and UUID are empty", ErrorCodes::UNKNOWN_TABLE); + return storage_id; + } + + bool look_for_external_table = where & StorageNamespace::ResolveExternal; + bool in_current_database = where & StorageNamespace::ResolveCurrentDatabase; + bool in_specified_database = where & StorageNamespace::ResolveGlobal; if (!storage_id.database_name.empty()) { if (in_specified_database) return storage_id; - throw Exception("External and temporary tables have no database, but " + + if (exception) + exception->emplace("External and temporary tables have no database, but " + storage_id.database_name + " is specified", ErrorCodes::UNKNOWN_TABLE); + return StorageID::createEmpty(); } if (look_for_external_table) @@ -1995,12 +2005,18 @@ StorageID Context::resolveStorageIDUnlocked(StorageID storage_id, StorageNamespa if (in_current_database) { if (current_database.empty()) - throw Exception("Default database is not selected", ErrorCodes::UNKNOWN_DATABASE); + { + if (exception) + exception->emplace("Default database is not selected", ErrorCodes::UNKNOWN_DATABASE); + return StorageID::createEmpty(); + } storage_id.database_name = current_database; return storage_id; } - throw Exception("Cannot resolve database name for table " + storage_id.getNameForLogs(), ErrorCodes::UNKNOWN_TABLE); + if (exception) + exception->emplace("Cannot resolve database name for table " + storage_id.getNameForLogs(), ErrorCodes::UNKNOWN_TABLE); + return StorageID::createEmpty(); } diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index a4cfdf7b70d..df482c43b1d 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -287,22 +287,22 @@ public: enum StorageNamespace { - Global = 1u, /// Database name must be specified - CurrentDatabase = 2u, /// Use current database - Ordinary = Global | CurrentDatabase, /// If database name is not specified, use current database - External = 4u, /// Try get external table - All = External | Ordinary /// If database name is not specified, try get external table, - /// if external table not found use current database. + ResolveGlobal = 1u, /// Database name must be specified + ResolveCurrentDatabase = 2u, /// Use current database + ResolveOrdinary = ResolveGlobal | ResolveCurrentDatabase, /// If database name is not specified, use current database + ResolveExternal = 4u, /// Try get external table + ResolveAll = ResolveExternal | ResolveOrdinary /// If database name is not specified, try get external table, + /// if external table not found use current database. }; String resolveDatabase(const String & database_name) const; - StorageID resolveStorageID(StorageID storage_id, StorageNamespace where = StorageNamespace::All) const; - StorageID resolveStorageIDUnlocked(StorageID storage_id, StorageNamespace where = StorageNamespace::All) const; + StorageID resolveStorageID(StorageID storage_id, StorageNamespace where = StorageNamespace::ResolveAll) const; + StorageID tryResolveStorageID(StorageID storage_id, StorageNamespace where = StorageNamespace::ResolveAll) const; + StorageID resolveStorageIDImpl(StorageID storage_id, StorageNamespace where, std::optional * exception) const; const Scalars & getScalars() const; const Block & getScalar(const String & name) const; Tables getExternalTables() const; - StoragePtr tryGetExternalTable(const String & table_name) const; StoragePtr getTable(const String & database_name, const String & table_name) const; StoragePtr getTable(const StorageID & table_id) const; StoragePtr tryGetTable(const String & database_name, const String & table_name) const; diff --git a/dbms/src/Interpreters/DatabaseCatalog.cpp b/dbms/src/Interpreters/DatabaseCatalog.cpp index d3a33190e1b..3f3851b68e3 100644 --- a/dbms/src/Interpreters/DatabaseCatalog.cpp +++ b/dbms/src/Interpreters/DatabaseCatalog.cpp @@ -243,20 +243,22 @@ DatabasePtr DatabaseCatalog::getDatabase(const String & database_name, const Con void DatabaseCatalog::addDependency(const StorageID & from, const StorageID & where) { std::lock_guard lock{databases_mutex}; - view_dependencies[from].insert(where); + // FIXME when loading metadata storage may not know UUIDs of it's dependencies, because they are not loaded yet, + // so UUID of `from` is not used here. (same for remove, get and update) + view_dependencies[{from.getDatabaseName(), from.getTableName()}].insert(where); } void DatabaseCatalog::removeDependency(const StorageID & from, const StorageID & where) { std::lock_guard lock{databases_mutex}; - view_dependencies[from].erase(where); + view_dependencies[{from.getDatabaseName(), from.getTableName()}].erase(where); } Dependencies DatabaseCatalog::getDependencies(const StorageID & from) const { std::lock_guard lock{databases_mutex}; - auto iter = view_dependencies.find(from); + auto iter = view_dependencies.find({from.getDatabaseName(), from.getTableName()}); if (iter == view_dependencies.end()) return {}; return Dependencies(iter->second.begin(), iter->second.end()); @@ -268,9 +270,9 @@ DatabaseCatalog::updateDependency(const StorageID & old_from, const StorageID & { std::lock_guard lock{databases_mutex}; if (!old_from.empty()) - view_dependencies[old_from].erase(old_where); + view_dependencies[{old_from.getDatabaseName(), old_from.getTableName()}].erase(old_where); if (!new_from.empty()) - view_dependencies[new_from].insert(new_where); + view_dependencies[{new_from.getDatabaseName(), new_from.getTableName()}].insert(new_where); } std::unique_ptr DatabaseCatalog::getDDLGuard(const String & database, const String & table) diff --git a/dbms/src/Interpreters/DatabaseCatalog.h b/dbms/src/Interpreters/DatabaseCatalog.h index b3c3963c047..09690c79db0 100644 --- a/dbms/src/Interpreters/DatabaseCatalog.h +++ b/dbms/src/Interpreters/DatabaseCatalog.h @@ -96,7 +96,7 @@ public: void addUUIDMapping(const UUID & uuid, DatabasePtr database, StoragePtr table); void removeUUIDMapping(const UUID & uuid); - StoragePtr getTable(const StorageID & table_id, const Context & local_context, std::optional * exception) const; + StoragePtr getTable(const StorageID & table_id, const Context & local_context, std::optional * exception = nullptr) const; void addDependency(const StorageID & from, const StorageID & where); diff --git a/dbms/src/Interpreters/ExternalTablesVisitor.h b/dbms/src/Interpreters/ExternalTablesVisitor.h index 3ff5f38f84d..cd05d61a365 100644 --- a/dbms/src/Interpreters/ExternalTablesVisitor.h +++ b/dbms/src/Interpreters/ExternalTablesVisitor.h @@ -32,8 +32,8 @@ private: static void visit(const ASTIdentifier & node, ASTPtr &, Data & data) { if (auto opt_name = IdentifierSemantic::getTableName(node)) - if (StoragePtr external_storage = data.context.tryGetExternalTable(*opt_name)) - data.external_tables[*opt_name] = external_storage; + if (auto resolved_id = data.context.tryResolveStorageID(StorageID("", *opt_name), Context::ResolveExternal)) + data.external_tables[*opt_name] = DatabaseCatalog::instance().getTable(resolved_id, data.context); } }; diff --git a/dbms/src/Interpreters/InterpreterDropQuery.cpp b/dbms/src/Interpreters/InterpreterDropQuery.cpp index af32089c7a9..4f2affb07f1 100644 --- a/dbms/src/Interpreters/InterpreterDropQuery.cpp +++ b/dbms/src/Interpreters/InterpreterDropQuery.cpp @@ -209,9 +209,10 @@ BlockIO InterpreterDropQuery::executeToTemporaryTable(const String & table_name, else { auto & context_handle = context.hasSessionContext() ? context.getSessionContext() : context; - StoragePtr table = context_handle.tryGetExternalTable(table_name); - if (table) + auto resolved_id = context_handle.tryResolveStorageID(StorageID("", table_name), Context::ResolveExternal); + if (resolved_id) { + StoragePtr table = DatabaseCatalog::instance().getTable(resolved_id, context); if (kind == ASTDropQuery::Kind::Truncate) { /// If table was already dropped by anyone, an exception will be thrown diff --git a/dbms/src/Storages/StorageID.h b/dbms/src/Storages/StorageID.h index 2cc19818de5..b93a62bd6ba 100644 --- a/dbms/src/Storages/StorageID.h +++ b/dbms/src/Storages/StorageID.h @@ -53,6 +53,11 @@ struct StorageID + (hasUUID() ? " (UUID " + toString(uuid) + ")" : ""); } + explicit operator bool () const + { + return !empty(); + } + bool empty() const { return table_name.empty() && !hasUUID(); From c479bbe9f957e45869dc681f2ccf192fe32f1e22 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 14 Feb 2020 00:00:03 +0300 Subject: [PATCH 022/712] make Context::getLock() private --- dbms/programs/client/Client.cpp | 3 ++ dbms/programs/local/LocalServer.cpp | 1 + dbms/programs/server/Server.cpp | 1 + dbms/programs/server/TCPHandler.cpp | 2 +- dbms/src/Interpreters/Context.cpp | 2 +- dbms/src/Interpreters/Context.h | 5 ++- dbms/src/Interpreters/DatabaseCatalog.cpp | 43 ++++++++++++++++--- dbms/src/Interpreters/DatabaseCatalog.h | 8 ++-- .../src/Interpreters/InterpreterDropQuery.cpp | 39 ++++------------- 9 files changed, 62 insertions(+), 42 deletions(-) diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp index c41a26a6c63..59f78dfebb0 100644 --- a/dbms/programs/client/Client.cpp +++ b/dbms/programs/client/Client.cpp @@ -224,6 +224,9 @@ private: context.setApplicationType(Context::ApplicationType::CLIENT); context.setQueryParameters(query_parameters); + /// There is no database catalog on client, but we have to initialized this singleton, because context may access it. + DatabaseCatalog::init(&context); + /// settings and limits could be specified in config file, but passed settings has higher priority for (auto && setting : context.getSettingsRef()) { diff --git a/dbms/programs/local/LocalServer.cpp b/dbms/programs/local/LocalServer.cpp index d5e08382424..eda7a45ab12 100644 --- a/dbms/programs/local/LocalServer.cpp +++ b/dbms/programs/local/LocalServer.cpp @@ -187,6 +187,7 @@ try * Otherwise, metadata of temporary File(format, EXPLICIT_PATH) tables will pollute metadata/ directory; * if such tables will not be dropped, clickhouse-server will not be able to load them due to security reasons. */ + DatabaseCatalog::init(context.get()); std::string default_database = config().getString("default_database", "_local"); DatabaseCatalog::instance().attachDatabase(default_database, std::make_shared(default_database)); context->setCurrentDatabase(default_database); diff --git a/dbms/programs/server/Server.cpp b/dbms/programs/server/Server.cpp index bde1d5c3cd3..9704f3a8085 100644 --- a/dbms/programs/server/Server.cpp +++ b/dbms/programs/server/Server.cpp @@ -545,6 +545,7 @@ int Server::main(const std::vector & /*args*/) try { + DatabaseCatalog::init(global_context.get()); loadMetadataSystem(*global_context); /// After attaching system databases we can initialize system log. global_context->initializeSystemLogs(); diff --git a/dbms/programs/server/TCPHandler.cpp b/dbms/programs/server/TCPHandler.cpp index 1174f3c465e..7b3f642daa5 100644 --- a/dbms/programs/server/TCPHandler.cpp +++ b/dbms/programs/server/TCPHandler.cpp @@ -979,7 +979,7 @@ bool TCPHandler::receiveData(bool scalar) auto resolved = query_context->tryResolveStorageID(temporary_id, Context::ResolveExternal); StoragePtr storage; /// If such a table does not exist, create it. - if (temporary_id.empty()) + if (resolved.empty()) { NamesAndTypesList columns = block.getNamesAndTypesList(); storage = StorageMemory::create(temporary_id, ColumnsDescription{columns}, ConstraintsDescription{}); diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index e7835f1fcdc..30ba76cf064 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -800,7 +800,7 @@ StoragePtr Context::getTableImpl(const StorageID & table_id, std::optional getLock() const; + /// For methods below you may need to acquire the context lock by yourself. const Context & getQueryContext() const; Context & getQueryContext(); @@ -588,6 +587,8 @@ public: MySQLWireContext mysql; private: + std::unique_lock getLock() const; + /// Compute and set actual user settings, client_info.current_user should be set void calculateUserSettings(); void calculateAccessRights(); diff --git a/dbms/src/Interpreters/DatabaseCatalog.cpp b/dbms/src/Interpreters/DatabaseCatalog.cpp index 3f3851b68e3..4b3401b48de 100644 --- a/dbms/src/Interpreters/DatabaseCatalog.cpp +++ b/dbms/src/Interpreters/DatabaseCatalog.cpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB { @@ -16,6 +17,7 @@ namespace ErrorCodes extern const int TABLE_ALREADY_EXISTS; extern const int DATABASE_ALREADY_EXISTS; extern const int DDL_GUARD_IS_ACTIVE; + extern const int DATABASE_NOT_EMPTY; } @@ -140,13 +142,32 @@ void DatabaseCatalog::attachDatabase(const String & database_name, const Databas } -DatabasePtr DatabaseCatalog::detachDatabase(const String & database_name) +DatabasePtr DatabaseCatalog::detachDatabase(const String & database_name, bool drop) { std::lock_guard lock{databases_mutex}; assertDatabaseExistsUnlocked(database_name); - auto res = databases.find(database_name)->second; + + if (!DatabaseCatalog::instance().getDatabase(database_name)->empty(*global_context)) + throw Exception("New table appeared in database being dropped or detached. Try again.", ErrorCodes::DATABASE_NOT_EMPTY); + + auto db = databases.find(database_name)->second; databases.erase(database_name); - return res; + + db->shutdown(); + + if (drop) + { + /// Delete the database. + db->drop(*global_context); + + /// Old ClickHouse versions did not store database.sql files + Poco::File database_metadata_file( + global_context->getPath() + "metadata/" + escapeForFileName(database_name) + ".sql"); + if (database_metadata_file.exists()) + database_metadata_file.remove(false); + } + + return db; } DatabasePtr DatabaseCatalog::getDatabase(const String & database_name) const @@ -227,10 +248,22 @@ void DatabaseCatalog::removeUUIDMapping(const UUID & uuid) throw Exception("Mapping for table with UUID=" + toString(uuid) + " doesn't exist", ErrorCodes::LOGICAL_ERROR); } +DatabaseCatalog::DatabaseCatalog(const Context * global_context_) + : global_context(global_context_), log(&Poco::Logger::get("DatabaseCatalog")) +{ + if (!global_context) + throw Exception("DatabaseCatalog is not initialized. It's a bug.", ErrorCodes::LOGICAL_ERROR); +} + +DatabaseCatalog & DatabaseCatalog::init(const Context * global_context_) +{ + static DatabaseCatalog database_catalog(global_context_); + return database_catalog; +} + DatabaseCatalog & DatabaseCatalog::instance() { - static DatabaseCatalog database_catalog; - return database_catalog; + return init(nullptr); } DatabasePtr DatabaseCatalog::getDatabase(const String & database_name, const Context & local_context) const diff --git a/dbms/src/Interpreters/DatabaseCatalog.h b/dbms/src/Interpreters/DatabaseCatalog.h index 09690c79db0..54358439f7e 100644 --- a/dbms/src/Interpreters/DatabaseCatalog.h +++ b/dbms/src/Interpreters/DatabaseCatalog.h @@ -61,6 +61,8 @@ public: static constexpr const char * TEMPORARY_DATABASE = "_temporary_and_external_tables"; static constexpr const char * SYSTEM_DATABASE = "system"; + static DatabaseCatalog & init(const Context * global_context_); + static DatabaseCatalog & instance(); //DatabaseCatalog(/*Context & global_context_, String default_database_*/) {} @@ -80,7 +82,7 @@ public: DatabasePtr getSystemDatabase() const; void attachDatabase(const String & database_name, const DatabasePtr & database); // ca, a - DatabasePtr detachDatabase(const String & database_name); // (sr), ca, a + DatabasePtr detachDatabase(const String & database_name, bool drop = false); // (sr), ca, a DatabasePtr getDatabase(const String & database_name, const Context & local_context) const; DatabasePtr getDatabase(const String & database_name) const; // sr, ca, a @@ -107,7 +109,7 @@ public: void updateDependency(const StorageID & old_from, const StorageID & old_where,const StorageID & new_from, const StorageID & new_where); private: - DatabaseCatalog() : log(&Poco::Logger::get("DatabaseCatalog")) {} + DatabaseCatalog(const Context * global_context_); void assertDatabaseExistsUnlocked(const String & database_name) const; void assertDatabaseDoesntExistUnlocked(const String & database_name) const; @@ -126,7 +128,7 @@ private: } private: - //[[maybe_unused]] Context & global_context; + const Context * global_context; mutable std::recursive_mutex databases_mutex; ViewDependencies view_dependencies; /// Current dependencies diff --git a/dbms/src/Interpreters/InterpreterDropQuery.cpp b/dbms/src/Interpreters/InterpreterDropQuery.cpp index 4f2affb07f1..14abeb15b37 100644 --- a/dbms/src/Interpreters/InterpreterDropQuery.cpp +++ b/dbms/src/Interpreters/InterpreterDropQuery.cpp @@ -246,17 +246,15 @@ BlockIO InterpreterDropQuery::executeToDatabase(const String & database_name, AS { throw Exception("Unable to truncate database.", ErrorCodes::SYNTAX_ERROR); } - else if (kind == ASTDropQuery::Kind::Detach) + else if (kind == ASTDropQuery::Kind::Detach || kind == ASTDropQuery::Kind::Drop) { - context.checkAccess(AccessType::DETACH_DATABASE, database_name); - DatabaseCatalog::instance().detachDatabase(database_name); - database->shutdown(); - //FIXME someone may still use tables from database - } - else if (kind == ASTDropQuery::Kind::Drop) - { - context.checkAccess(AccessType::DROP_DATABASE, database_name); + bool drop = kind == ASTDropQuery::Kind::Drop; + if (drop) + context.checkAccess(AccessType::DROP_DATABASE, database_name); + else + context.checkAccess(AccessType::DETACH_DATABASE, database_name); + /// DETACH or DROP all tables and dictionaries inside database for (auto iterator = database->getTablesIterator(context); iterator->isValid(); iterator->next()) { String current_table_name = iterator->name(); @@ -269,27 +267,8 @@ BlockIO InterpreterDropQuery::executeToDatabase(const String & database_name, AS executeToDictionary(database_name, current_dictionary, kind, false, false, false); } - auto context_lock = context.getLock(); - - /// Someone could have time to delete the database before us. - DatabaseCatalog::instance().assertDatabaseExists(database_name); - - /// Someone could have time to create a table in the database to be deleted while we deleted the tables without the context lock. - if (!DatabaseCatalog::instance().getDatabase(database_name)->empty(context)) - throw Exception("New table appeared in database being dropped. Try dropping it again.", ErrorCodes::DATABASE_NOT_EMPTY); - - /// Delete database information from the RAM - DatabaseCatalog::instance().detachDatabase(database_name); - - database->shutdown(); - - /// Delete the database. - database->drop(context); - - /// Old ClickHouse versions did not store database.sql files - Poco::File database_metadata_file(context.getPath() + "metadata/" + escapeForFileName(database_name) + ".sql"); - if (database_metadata_file.exists()) - database_metadata_file.remove(false); + /// DETACH or DROP database itself + DatabaseCatalog::instance().detachDatabase(database_name, drop); } } From d710bd18129210e1f9c949e5a7c651bf6a9ad365 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 17 Feb 2020 16:52:59 +0300 Subject: [PATCH 023/712] better get... methods --- dbms/programs/copier/ClusterCopier.cpp | 1 + dbms/programs/server/TCPHandler.cpp | 2 +- .../PushingToViewsBlockOutputStream.cpp | 2 +- dbms/src/Interpreters/Context.cpp | 67 ++++++++----------- dbms/src/Interpreters/Context.h | 5 -- dbms/src/Interpreters/DatabaseCatalog.cpp | 45 +++++++++++-- dbms/src/Interpreters/DatabaseCatalog.h | 21 +++--- dbms/src/Interpreters/ExternalTablesVisitor.h | 2 +- .../src/Interpreters/InterpreterDropQuery.cpp | 2 +- .../Interpreters/InterpreterExistsQuery.cpp | 12 ++-- dbms/src/Interpreters/SystemLog.h | 2 +- dbms/src/Storages/Kafka/StorageKafka.cpp | 4 +- .../src/Storages/LiveView/StorageLiveView.cpp | 2 +- dbms/src/Storages/LiveView/StorageLiveView.h | 2 +- dbms/src/Storages/StorageMaterializedView.cpp | 6 +- ..._transform_query_for_external_database.cpp | 1 + 16 files changed, 96 insertions(+), 80 deletions(-) diff --git a/dbms/programs/copier/ClusterCopier.cpp b/dbms/programs/copier/ClusterCopier.cpp index 4d5a7788e92..e1d161240a6 100644 --- a/dbms/programs/copier/ClusterCopier.cpp +++ b/dbms/programs/copier/ClusterCopier.cpp @@ -2453,6 +2453,7 @@ void ClusterCopierApp::mainImpl() auto context = std::make_unique(Context::createGlobal()); context->makeGlobalContext(); + DatabaseCatalog::init(context.get()); SCOPE_EXIT(context->shutdown()); context->setConfig(loaded_config.configuration); diff --git a/dbms/programs/server/TCPHandler.cpp b/dbms/programs/server/TCPHandler.cpp index 7b3f642daa5..d479a074728 100644 --- a/dbms/programs/server/TCPHandler.cpp +++ b/dbms/programs/server/TCPHandler.cpp @@ -986,7 +986,7 @@ bool TCPHandler::receiveData(bool scalar) storage->startup(); query_context->addExternalTable(temporary_id.table_name, storage); } else - storage = DatabaseCatalog::instance().getTable(resolved, *query_context); + storage = DatabaseCatalog::instance().getTable(resolved); /// The data will be written directly to the table. state.io.out = storage->write(ASTPtr(), *query_context); } diff --git a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp index 3efa098423a..0415295d0ea 100644 --- a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp +++ b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp @@ -45,7 +45,7 @@ PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream( for (const auto & database_table : dependencies) { - auto dependent_table = context.getTable(database_table); + auto dependent_table = DatabaseCatalog::instance().getTable(database_table); ASTPtr query; BlockOutputStreamPtr out; diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index 30ba76cf064..8b06eed08f0 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -710,22 +710,9 @@ void Context::setUser(const String & name, const String & password, const Poco:: calculateAccessRights(); } -bool Context::isTableExist(const String & database_name, const String & table_name) const -{ - auto table_id = resolveStorageID({database_name, table_name}, StorageNamespace::ResolveOrdinary); - return DatabaseCatalog::instance().isTableExist(table_id, *this); -} - -bool Context::isDictionaryExists(const String & database_name, const String & dictionary_name) const -{ - auto lock = getLock(); - String db = resolveDatabase(database_name); - auto db_ptr = DatabaseCatalog::instance().tryGetDatabase(db); - return db_ptr && db_ptr->isDictionaryExist(*this, dictionary_name); -} - bool Context::isExternalTableExist(const String & table_name) const { + auto lock = getLock(); return external_tables_mapping.count(table_name); } @@ -746,6 +733,7 @@ const Block & Context::getScalar(const String & name) const Tables Context::getExternalTables() const { + //FIXME getTable() may acquire some locks. Better not to call it while holding context lock auto lock = getLock(); Tables res; @@ -768,13 +756,9 @@ Tables Context::getExternalTables() const StoragePtr Context::getTable(const String & database_name, const String & table_name) const { - return getTable(StorageID(database_name, table_name)); -} - -StoragePtr Context::getTable(const StorageID & table_id) const -{ + auto resolved_id = resolveStorageID(StorageID(database_name, table_name)); std::optional exc; - auto res = getTableImpl(table_id, &exc); + auto res = DatabaseCatalog::instance().getTableImpl(resolved_id, *this, &exc); if (!res) throw *exc; return res; @@ -782,29 +766,19 @@ StoragePtr Context::getTable(const StorageID & table_id) const StoragePtr Context::tryGetTable(const String & database_name, const String & table_name) const { - return getTableImpl(StorageID(database_name, table_name), {}); -} - -StoragePtr Context::tryGetTable(const StorageID & table_id) const -{ - return getTableImpl(table_id, {}); -} - - -StoragePtr Context::getTableImpl(const StorageID & table_id, std::optional * exception) const -{ - auto resolved_id = resolveStorageID(table_id); - return DatabaseCatalog::instance().getTable(resolved_id, *this, exception); + auto resolved_id = tryResolveStorageID(StorageID(database_name, table_name)); + return DatabaseCatalog::instance().getTableImpl(resolved_id, *this, nullptr); } void Context::addExternalTable(const String & table_name, const StoragePtr & storage, const ASTPtr & ast) { + auto external_db = DatabaseCatalog::instance().getDatabaseForTemporaryTables(); + auto holder = std::make_shared(*this, *external_db, storage, ast); auto lock = getLock(); if (external_tables_mapping.end() != external_tables_mapping.find(table_name)) throw Exception("Temporary table " + backQuoteIfNeed(table_name) + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS); - auto holder = std::make_shared(*this, *DatabaseCatalog::instance().getDatabaseForTemporaryTables(), storage, ast); external_tables_mapping.emplace(table_name, std::move(holder)); } @@ -823,7 +797,16 @@ bool Context::hasScalar(const String & name) const bool Context::removeExternalTable(const String & table_name) { - return external_tables_mapping.erase(table_name); + std::shared_ptr holder; + { + auto lock = getLock(); + auto iter = external_tables_mapping.find(table_name); + if (iter == external_tables_mapping.end()) + return false; + holder = iter->second; + external_tables_mapping.erase(iter); + } + return true; } @@ -863,11 +846,15 @@ StoragePtr Context::getViewSource() ASTPtr Context::getCreateExternalTableQuery(const String & table_name) const { - auto it = external_tables_mapping.find(table_name); - if (external_tables_mapping.end() == it) - throw Exception("Temporary table " + backQuoteIfNeed(table_name) + " doesn't exist", ErrorCodes::UNKNOWN_TABLE); - - return DatabaseCatalog::instance().getDatabaseForTemporaryTables()->getCreateTableQuery(*this, it->second->getGlobalTableID().table_name); + StorageID external_id = StorageID::createEmpty(); + { + auto lock = getLock(); + auto it = external_tables_mapping.find(table_name); + if (external_tables_mapping.end() == it) + throw Exception("Temporary table " + backQuoteIfNeed(table_name) + " doesn't exist", ErrorCodes::UNKNOWN_TABLE); + external_id = it->second->getGlobalTableID(); + } + return DatabaseCatalog::instance().getDatabaseForTemporaryTables()->getCreateTableQuery(*this, external_id.table_name); } Settings Context::getSettings() const diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index d08e53a4e11..d643dee9e2c 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -280,9 +280,6 @@ public: ClientInfo & getClientInfo() { return client_info; } const ClientInfo & getClientInfo() const { return client_info; } - /// Checking the existence of the table/database. Database can be empty - in this case the current database is used. - bool isTableExist(const String & database_name, const String & table_name) const; - bool isDictionaryExists(const String & database_name, const String & dictionary_name) const; bool isExternalTableExist(const String & table_name) const; enum StorageNamespace @@ -304,9 +301,7 @@ public: const Block & getScalar(const String & name) const; Tables getExternalTables() const; StoragePtr getTable(const String & database_name, const String & table_name) const; - StoragePtr getTable(const StorageID & table_id) const; StoragePtr tryGetTable(const String & database_name, const String & table_name) const; - StoragePtr tryGetTable(const StorageID & table_id) const; void addExternalTable(const String & table_name, const StoragePtr & storage, const ASTPtr & ast = {}); void addScalar(const String & name, const Block & block); bool hasScalar(const String & name) const; diff --git a/dbms/src/Interpreters/DatabaseCatalog.cpp b/dbms/src/Interpreters/DatabaseCatalog.cpp index 4b3401b48de..fbf978c5d64 100644 --- a/dbms/src/Interpreters/DatabaseCatalog.cpp +++ b/dbms/src/Interpreters/DatabaseCatalog.cpp @@ -70,8 +70,15 @@ DatabaseAndTable DatabaseCatalog::tryGetByUUID(const UUID & uuid) const } -StoragePtr DatabaseCatalog::getTable(const StorageID & table_id, const Context & local_context, std::optional * exception) const +StoragePtr DatabaseCatalog::getTableImpl(const StorageID & table_id, const Context & local_context, std::optional * exception) const { + if (!table_id) + { + if (exception) + exception->emplace("Cannot find table: StorageID is empty", ErrorCodes::UNKNOWN_TABLE); + return {}; + } + //if (table_id.hasUUID()) //{ // auto db_and_table = tryGetByUUID(table_id.uuid); @@ -146,11 +153,12 @@ DatabasePtr DatabaseCatalog::detachDatabase(const String & database_name, bool d { std::lock_guard lock{databases_mutex}; assertDatabaseExistsUnlocked(database_name); + auto db = databases.find(database_name)->second; - if (!DatabaseCatalog::instance().getDatabase(database_name)->empty(*global_context)) + if (!db->empty(*global_context)) + if (!db->empty(*global_context)) throw Exception("New table appeared in database being dropped or detached. Try again.", ErrorCodes::DATABASE_NOT_EMPTY); - auto db = databases.find(database_name)->second; databases.erase(database_name); db->shutdown(); @@ -206,9 +214,14 @@ bool DatabaseCatalog::isTableExist(const DB::StorageID & table_id, const DB::Con // return tryGetByUUID(table_id.uuid).second != nullptr; //else //{ - std::lock_guard lock{databases_mutex}; - auto db = databases.find(table_id.database_name); - return db != databases.end() && db->second->isTableExist(context, table_id.table_name); + DatabasePtr db; + { + std::lock_guard lock{databases_mutex}; + auto iter = databases.find(table_id.database_name); + if (iter != databases.end()) + db = iter->second; + } + return db && db->isTableExist(context, table_id.table_name); //} } @@ -314,6 +327,26 @@ std::unique_ptr DatabaseCatalog::getDDLGuard(const String & database, return std::make_unique(ddl_guards[database], std::move(lock), table); } +bool DatabaseCatalog::isDictionaryExist(const StorageID & table_id, const Context & context) const +{ + auto db = tryGetDatabase(table_id.getDatabaseName()); + return db && db->isDictionaryExist(context, table_id.getTableName()); +} + +StoragePtr DatabaseCatalog::getTable(const StorageID & table_id) const +{ + std::optional exc; + auto res = getTableImpl(table_id, *global_context, &exc); + if (!res) + throw *exc; + return res; +} + +StoragePtr DatabaseCatalog::tryGetTable(const StorageID & table_id) const +{ + return getTableImpl(table_id, *global_context, nullptr); +} + DDLGuard::DDLGuard(Map & map_, std::unique_lock guards_lock_, const String & elem) : map(map_), guards_lock(std::move(guards_lock_)) diff --git a/dbms/src/Interpreters/DatabaseCatalog.h b/dbms/src/Interpreters/DatabaseCatalog.h index 54358439f7e..e02f6c80c67 100644 --- a/dbms/src/Interpreters/DatabaseCatalog.h +++ b/dbms/src/Interpreters/DatabaseCatalog.h @@ -81,24 +81,27 @@ public: DatabasePtr getDatabaseForTemporaryTables() const; DatabasePtr getSystemDatabase() const; - void attachDatabase(const String & database_name, const DatabasePtr & database); // ca, a - DatabasePtr detachDatabase(const String & database_name, bool drop = false); // (sr), ca, a + void attachDatabase(const String & database_name, const DatabasePtr & database); + DatabasePtr detachDatabase(const String & database_name, bool drop = false); DatabasePtr getDatabase(const String & database_name, const Context & local_context) const; - DatabasePtr getDatabase(const String & database_name) const; // sr, ca, a - DatabasePtr tryGetDatabase(const String & database_name) const; // sr - bool isDatabaseExist(const String & database_name) const; // sr, ca + DatabasePtr getDatabase(const String & database_name) const; + DatabasePtr tryGetDatabase(const String & database_name) const; + bool isDatabaseExist(const String & database_name) const; Databases getDatabases() const; DatabaseAndTable tryGetByUUID(const UUID & uuid) const; - void assertTableDoesntExist(const StorageID & table_id, const Context & context) const; // sr, ca - bool isTableExist(const StorageID & table_id, const Context & context) const; // sr, ca + void assertTableDoesntExist(const StorageID & table_id, const Context & context) const; + bool isTableExist(const StorageID & table_id, const Context & context) const; + bool isDictionaryExist(const StorageID & table_id, const Context & context) const; void addUUIDMapping(const UUID & uuid, DatabasePtr database, StoragePtr table); void removeUUIDMapping(const UUID & uuid); - StoragePtr getTable(const StorageID & table_id, const Context & local_context, std::optional * exception = nullptr) const; + StoragePtr getTable(const StorageID & table_id) const; + StoragePtr tryGetTable(const StorageID & table_id) const; + StoragePtr getTableImpl(const StorageID & table_id, const Context & local_context, std::optional * exception = nullptr) const; void addDependency(const StorageID & from, const StorageID & where); @@ -129,7 +132,7 @@ private: private: const Context * global_context; - mutable std::recursive_mutex databases_mutex; + mutable std::mutex databases_mutex; ViewDependencies view_dependencies; /// Current dependencies diff --git a/dbms/src/Interpreters/ExternalTablesVisitor.h b/dbms/src/Interpreters/ExternalTablesVisitor.h index cd05d61a365..73fd40e8f98 100644 --- a/dbms/src/Interpreters/ExternalTablesVisitor.h +++ b/dbms/src/Interpreters/ExternalTablesVisitor.h @@ -33,7 +33,7 @@ private: { if (auto opt_name = IdentifierSemantic::getTableName(node)) if (auto resolved_id = data.context.tryResolveStorageID(StorageID("", *opt_name), Context::ResolveExternal)) - data.external_tables[*opt_name] = DatabaseCatalog::instance().getTable(resolved_id, data.context); + data.external_tables[*opt_name] = DatabaseCatalog::instance().getTable(resolved_id); } }; diff --git a/dbms/src/Interpreters/InterpreterDropQuery.cpp b/dbms/src/Interpreters/InterpreterDropQuery.cpp index 14abeb15b37..cd66a7f9c62 100644 --- a/dbms/src/Interpreters/InterpreterDropQuery.cpp +++ b/dbms/src/Interpreters/InterpreterDropQuery.cpp @@ -212,7 +212,7 @@ BlockIO InterpreterDropQuery::executeToTemporaryTable(const String & table_name, auto resolved_id = context_handle.tryResolveStorageID(StorageID("", table_name), Context::ResolveExternal); if (resolved_id) { - StoragePtr table = DatabaseCatalog::instance().getTable(resolved_id, context); + StoragePtr table = DatabaseCatalog::instance().getTable(resolved_id); if (kind == ASTDropQuery::Kind::Truncate) { /// If table was already dropped by anyone, an exception will be thrown diff --git a/dbms/src/Interpreters/InterpreterExistsQuery.cpp b/dbms/src/Interpreters/InterpreterExistsQuery.cpp index e519cba26d4..c17d85a1ff0 100644 --- a/dbms/src/Interpreters/InterpreterExistsQuery.cpp +++ b/dbms/src/Interpreters/InterpreterExistsQuery.cpp @@ -49,22 +49,18 @@ BlockInputStreamPtr InterpreterExistsQuery::executeImpl() } else { - String database = exists_query->database; - if (database.empty()) - database = context.getCurrentDatabase(); + String database = context.resolveDatabase(exists_query->database); context.checkAccess(AccessType::EXISTS, database, exists_query->table); - result = context.isTableExist(database, exists_query->table); + result = DatabaseCatalog::instance().isTableExist({database, exists_query->table}, context); } } else if ((exists_query = query_ptr->as())) { if (exists_query->temporary) throw Exception("Temporary dictionaries are not possible.", ErrorCodes::SYNTAX_ERROR); - String database = exists_query->database; - if (database.empty()) - database = context.getCurrentDatabase(); + String database = context.resolveDatabase(exists_query->database); context.checkAccess(AccessType::EXISTS, database, exists_query->table); - result = context.isDictionaryExists(exists_query->database, exists_query->table); + result = DatabaseCatalog::instance().isDictionaryExist({database, exists_query->table}, context); } return std::make_shared(Block{{ diff --git a/dbms/src/Interpreters/SystemLog.h b/dbms/src/Interpreters/SystemLog.h index 84562704532..202f9ce2055 100644 --- a/dbms/src/Interpreters/SystemLog.h +++ b/dbms/src/Interpreters/SystemLog.h @@ -376,7 +376,7 @@ void SystemLog::prepareTable() { /// Rename the existing table. int suffix = 0; - while (context.isTableExist(database_name, table_name + "_" + toString(suffix))) + while (DatabaseCatalog::instance().isTableExist({database_name, table_name + "_" + toString(suffix)}, context)) ++suffix; auto rename = std::make_shared(); diff --git a/dbms/src/Storages/Kafka/StorageKafka.cpp b/dbms/src/Storages/Kafka/StorageKafka.cpp index 86010d43b4e..94380c6e004 100644 --- a/dbms/src/Storages/Kafka/StorageKafka.cpp +++ b/dbms/src/Storages/Kafka/StorageKafka.cpp @@ -297,7 +297,7 @@ bool StorageKafka::checkDependencies(const StorageID & table_id) // Check the dependencies are ready? for (const auto & db_tab : dependencies) { - auto table = global_context.tryGetTable(db_tab); + auto table = DatabaseCatalog::instance().tryGetTable(db_tab); if (!table) return false; @@ -351,7 +351,7 @@ void StorageKafka::threadFunc() bool StorageKafka::streamToViews() { auto table_id = getStorageID(); - auto table = global_context.getTable(table_id); + auto table = DatabaseCatalog::instance().getTable(table_id); if (!table) throw Exception("Engine table " + table_id.getNameForLogs() + " doesn't exist.", ErrorCodes::LOGICAL_ERROR); diff --git a/dbms/src/Storages/LiveView/StorageLiveView.cpp b/dbms/src/Storages/LiveView/StorageLiveView.cpp index 8c77e2c8f81..38fd3f726a2 100644 --- a/dbms/src/Storages/LiveView/StorageLiveView.cpp +++ b/dbms/src/Storages/LiveView/StorageLiveView.cpp @@ -412,7 +412,7 @@ void StorageLiveView::noUsersThread(std::shared_ptr storage, co if (drop_table) { - if (storage->global_context.tryGetTable(table_id)) + if (DatabaseCatalog::instance().tryGetTable(table_id)) { try { diff --git a/dbms/src/Storages/LiveView/StorageLiveView.h b/dbms/src/Storages/LiveView/StorageLiveView.h index b5314c7db3a..f536c3a6bdc 100644 --- a/dbms/src/Storages/LiveView/StorageLiveView.h +++ b/dbms/src/Storages/LiveView/StorageLiveView.h @@ -54,7 +54,7 @@ public: { return StorageID("", getStorageID().table_name + "_blocks"); } - StoragePtr getParentStorage() const { return global_context.getTable(select_table_id); } + StoragePtr getParentStorage() const { return DatabaseCatalog::instance().getTable(select_table_id); } NameAndTypePair getColumn(const String & column_name) const override; bool hasColumn(const String & column_name) const override; diff --git a/dbms/src/Storages/StorageMaterializedView.cpp b/dbms/src/Storages/StorageMaterializedView.cpp index f81a02e2675..11e9acb248a 100644 --- a/dbms/src/Storages/StorageMaterializedView.cpp +++ b/dbms/src/Storages/StorageMaterializedView.cpp @@ -210,7 +210,7 @@ BlockOutputStreamPtr StorageMaterializedView::write(const ASTPtr & query, const static void executeDropQuery(ASTDropQuery::Kind kind, Context & global_context, const StorageID & target_table_id) { - if (global_context.tryGetTable(target_table_id)) + if (DatabaseCatalog::instance().tryGetTable(target_table_id)) { /// We create and execute `drop` query for internal table. auto drop_query = std::make_shared(); @@ -365,12 +365,12 @@ void StorageMaterializedView::shutdown() StoragePtr StorageMaterializedView::getTargetTable() const { - return global_context.getTable(target_table_id); + return DatabaseCatalog::instance().getTable(target_table_id); } StoragePtr StorageMaterializedView::tryGetTargetTable() const { - return global_context.tryGetTable(target_table_id); + return DatabaseCatalog::instance().tryGetTable(target_table_id); } Strings StorageMaterializedView::getDataPaths() const diff --git a/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp b/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp index 856de3d89ca..1fd4dc6fb41 100644 --- a/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp +++ b/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp @@ -33,6 +33,7 @@ struct State DatabasePtr database = std::make_shared("test"); database->attachTable("table", StorageMemory::create(StorageID("test", "table"), ColumnsDescription{columns}, ConstraintsDescription{})); context.makeGlobalContext(); + DatabaseCatalog::init(&context); DatabaseCatalog::instance().attachDatabase("test", database); context.setCurrentDatabase("test"); } From e6718e199ee0750c2dfd7d9928b864c9eafb0c32 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 17 Feb 2020 22:28:25 +0300 Subject: [PATCH 024/712] less Context::getTable() usages --- dbms/src/DataStreams/tests/union_stream2.cpp | 2 +- dbms/src/Interpreters/DDLWorker.cpp | 4 +- dbms/src/Interpreters/DatabaseCatalog.cpp | 6 +- dbms/src/Interpreters/DatabaseCatalog.h | 2 +- .../Interpreters/InterpreterAlterQuery.cpp | 8 +-- .../Interpreters/InterpreterCheckQuery.cpp | 7 +- .../Interpreters/InterpreterCreateQuery.cpp | 7 +- .../InterpreterKillQueryQuery.cpp | 13 ++-- .../Interpreters/InterpreterOptimizeQuery.cpp | 2 +- .../Interpreters/InterpreterRenameQuery.cpp | 2 +- .../Interpreters/InterpreterSystemQuery.cpp | 12 ++-- .../Interpreters/InterpreterWatchQuery.cpp | 18 ++---- dbms/src/Interpreters/SystemLog.h | 36 ++++++----- dbms/src/Storages/StorageBuffer.cpp | 64 ++++++++++--------- dbms/src/Storages/StorageBuffer.h | 11 ++-- dbms/src/Storages/StorageID.cpp | 50 +++++++++++++++ dbms/src/Storages/StorageID.h | 43 +++---------- dbms/src/Storages/StorageMaterializedView.cpp | 2 +- dbms/src/Storages/StorageMergeTree.cpp | 8 +-- .../Storages/StorageReplicatedMergeTree.cpp | 20 +++--- dbms/src/Storages/StorageS3.cpp | 1 + 21 files changed, 167 insertions(+), 151 deletions(-) create mode 100644 dbms/src/Storages/StorageID.cpp diff --git a/dbms/src/DataStreams/tests/union_stream2.cpp b/dbms/src/DataStreams/tests/union_stream2.cpp index ab0b583b8e5..577d3cec41c 100644 --- a/dbms/src/DataStreams/tests/union_stream2.cpp +++ b/dbms/src/DataStreams/tests/union_stream2.cpp @@ -33,7 +33,7 @@ try Names column_names; column_names.push_back("WatchID"); - StoragePtr table = context.getTable("default", "hits6"); + StoragePtr table = DatabaseCatalog::instance().getTable({"default", "hits6"}); QueryProcessingStage::Enum stage = table->getQueryProcessingStage(context); BlockInputStreams streams = table->read(column_names, {}, context, stage, settings.max_block_size, settings.max_threads); diff --git a/dbms/src/Interpreters/DDLWorker.cpp b/dbms/src/Interpreters/DDLWorker.cpp index d99781ab816..e04ca8d55c7 100644 --- a/dbms/src/Interpreters/DDLWorker.cpp +++ b/dbms/src/Interpreters/DDLWorker.cpp @@ -627,8 +627,8 @@ void DDLWorker::processTask(DDLTask & task, const ZooKeeperPtr & zookeeper) if (!query_with_table->table.empty()) { /// It's not CREATE DATABASE - String database = query_with_table->database.empty() ? context.getCurrentDatabase() : query_with_table->database; - storage = context.tryGetTable(database, query_with_table->table); + StorageID table_id{*query_with_table, context}; + storage = DatabaseCatalog::instance().tryGetTable(table_id); } /// For some reason we check consistency of cluster definition only diff --git a/dbms/src/Interpreters/DatabaseCatalog.cpp b/dbms/src/Interpreters/DatabaseCatalog.cpp index fbf978c5d64..05551a88d1c 100644 --- a/dbms/src/Interpreters/DatabaseCatalog.cpp +++ b/dbms/src/Interpreters/DatabaseCatalog.cpp @@ -6,6 +6,7 @@ #include #include #include +#include namespace DB { @@ -149,14 +150,13 @@ void DatabaseCatalog::attachDatabase(const String & database_name, const Databas } -DatabasePtr DatabaseCatalog::detachDatabase(const String & database_name, bool drop) +DatabasePtr DatabaseCatalog::detachDatabase(const String & database_name, bool drop, bool check_empty) { std::lock_guard lock{databases_mutex}; assertDatabaseExistsUnlocked(database_name); auto db = databases.find(database_name)->second; - if (!db->empty(*global_context)) - if (!db->empty(*global_context)) + if (check_empty && !db->empty(*global_context)) throw Exception("New table appeared in database being dropped or detached. Try again.", ErrorCodes::DATABASE_NOT_EMPTY); databases.erase(database_name); diff --git a/dbms/src/Interpreters/DatabaseCatalog.h b/dbms/src/Interpreters/DatabaseCatalog.h index e02f6c80c67..0439e98198e 100644 --- a/dbms/src/Interpreters/DatabaseCatalog.h +++ b/dbms/src/Interpreters/DatabaseCatalog.h @@ -82,7 +82,7 @@ public: DatabasePtr getSystemDatabase() const; void attachDatabase(const String & database_name, const DatabasePtr & database); - DatabasePtr detachDatabase(const String & database_name, bool drop = false); + DatabasePtr detachDatabase(const String & database_name, bool drop = false, bool check_empty = true); DatabasePtr getDatabase(const String & database_name, const Context & local_context) const; DatabasePtr getDatabase(const String & database_name) const; diff --git a/dbms/src/Interpreters/InterpreterAlterQuery.cpp b/dbms/src/Interpreters/InterpreterAlterQuery.cpp index b155019187b..d8f045b90d2 100644 --- a/dbms/src/Interpreters/InterpreterAlterQuery.cpp +++ b/dbms/src/Interpreters/InterpreterAlterQuery.cpp @@ -41,14 +41,12 @@ BlockIO InterpreterAlterQuery::execute() return executeDDLQueryOnCluster(query_ptr, context, getRequiredAccess()); context.checkAccess(getRequiredAccess()); - - const String & table_name = alter.table; - String database_name = alter.database.empty() ? context.getCurrentDatabase() : alter.database; - StoragePtr table = context.getTable(database_name, table_name); + StorageID table_id{alter, context}; + StoragePtr table = DatabaseCatalog::instance().getTable(table_id); /// Add default database to table identifiers that we can encounter in e.g. default expressions, /// mutation expression, etc. - AddDefaultDatabaseVisitor visitor(database_name); + AddDefaultDatabaseVisitor visitor(table_id.getDatabaseName()); ASTPtr command_list_ptr = alter.command_list->ptr(); visitor.visit(command_list_ptr); diff --git a/dbms/src/Interpreters/InterpreterCheckQuery.cpp b/dbms/src/Interpreters/InterpreterCheckQuery.cpp index d75777cc4ce..f1ba695789a 100644 --- a/dbms/src/Interpreters/InterpreterCheckQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCheckQuery.cpp @@ -38,11 +38,10 @@ InterpreterCheckQuery::InterpreterCheckQuery(const ASTPtr & query_ptr_, const Co BlockIO InterpreterCheckQuery::execute() { const auto & check = query_ptr->as(); - const String & table_name = check.table; - String database_name = check.database.empty() ? context.getCurrentDatabase() : check.database; + StorageID table_id{check, context}; - context.checkAccess(AccessType::SHOW, database_name, table_name); - StoragePtr table = context.getTable(database_name, table_name); + context.checkAccess(AccessType::SHOW, table_id.database_name, table_id.table_name); + StoragePtr table = DatabaseCatalog::instance().getTable(table_id); auto check_results = table->checkData(query_ptr, context); Block block; diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp index addb6edd836..69ccb387bd9 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp @@ -147,6 +147,7 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) bool renamed = false; try { + //FIXME is it possible to attach db only after it was loaded? (no, loadStoredObjects adds view dependencies) DatabaseCatalog::instance().attachDatabase(database_name, database); added = true; @@ -163,7 +164,7 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) if (renamed) Poco::File(metadata_file_tmp_path).remove(); if (added) - DatabaseCatalog::instance().detachDatabase(database_name); + DatabaseCatalog::instance().detachDatabase(database_name, false, false); throw; } @@ -411,8 +412,8 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::setProperties(AS } else if (!create.as_table.empty()) { - String as_database_name = create.as_database.empty() ? context.getCurrentDatabase() : create.as_database; - StoragePtr as_storage = context.getTable(as_database_name, create.as_table); + String as_database_name = context.resolveDatabase(create.as_database); + StoragePtr as_storage = DatabaseCatalog::instance().getTable({as_database_name, create.as_table}); /// as_storage->getColumns() and setEngine(...) must be called under structure lock of other_table for CREATE ... AS other_table. as_storage_lock = as_storage->lockStructureForShare(false, context.getCurrentQueryId()); diff --git a/dbms/src/Interpreters/InterpreterKillQueryQuery.cpp b/dbms/src/Interpreters/InterpreterKillQueryQuery.cpp index 63fa814aadc..c27f07f86b6 100644 --- a/dbms/src/Interpreters/InterpreterKillQueryQuery.cpp +++ b/dbms/src/Interpreters/InterpreterKillQueryQuery.cpp @@ -236,23 +236,22 @@ BlockIO InterpreterKillQueryQuery::execute() header.insert(0, {ColumnString::create(), std::make_shared(), "kill_status"}); MutableColumns res_columns = header.cloneEmptyColumns(); - String database_name, table_name; + auto table_id = StorageID::createEmpty(); for (size_t i = 0; i < mutations_block.rows(); ++i) { - database_name = database_col.getDataAt(i).toString(); - table_name = table_col.getDataAt(i).toString(); + table_id = StorageID{database_col.getDataAt(i).toString(), table_col.getDataAt(i).toString()}; auto mutation_id = mutation_id_col.getDataAt(i).toString(); CancellationCode code = CancellationCode::Unknown; if (!query.test) { - auto storage = context.tryGetTable(database_name, table_name); + auto storage = DatabaseCatalog::instance().tryGetTable(table_id); if (!storage) code = CancellationCode::NotFound; else { - if (!context.getAccessRights()->isGranted(&Poco::Logger::get("InterpreterKillQueryQuery"), AccessType::KILL_MUTATION, database_name, table_name)) + if (!context.getAccessRights()->isGranted(&Poco::Logger::get("InterpreterKillQueryQuery"), AccessType::KILL_MUTATION, table_id.database_name, table_id.table_name)) continue; code = storage->killMutation(mutation_id); } @@ -261,9 +260,9 @@ BlockIO InterpreterKillQueryQuery::execute() insertResultRow(i, code, mutations_block, header, res_columns); } - if (res_columns[0]->empty() && !table_name.empty()) + if (res_columns[0]->empty() && table_id) throw Exception( - "Not allowed to kill mutation on " + backQuoteIfNeed(database_name) + "." + backQuoteIfNeed(table_name), + "Not allowed to kill mutation on " + table_id.getNameForLogs(), ErrorCodes::ACCESS_DENIED); res_io.in = std::make_shared(header.cloneWithColumns(std::move(res_columns))); diff --git a/dbms/src/Interpreters/InterpreterOptimizeQuery.cpp b/dbms/src/Interpreters/InterpreterOptimizeQuery.cpp index 74551329343..470dede3540 100644 --- a/dbms/src/Interpreters/InterpreterOptimizeQuery.cpp +++ b/dbms/src/Interpreters/InterpreterOptimizeQuery.cpp @@ -25,7 +25,7 @@ BlockIO InterpreterOptimizeQuery::execute() context.checkAccess(getRequiredAccess()); - StoragePtr table = context.getTable(ast.database, ast.table); + StoragePtr table = DatabaseCatalog::instance().getTable(StorageID{ast, context}); table->optimize(query_ptr, ast.partition, ast.final, ast.deduplicate, context); return {}; } diff --git a/dbms/src/Interpreters/InterpreterRenameQuery.cpp b/dbms/src/Interpreters/InterpreterRenameQuery.cpp index 820676fa2d8..c6e2fbf48f7 100644 --- a/dbms/src/Interpreters/InterpreterRenameQuery.cpp +++ b/dbms/src/Interpreters/InterpreterRenameQuery.cpp @@ -93,7 +93,7 @@ BlockIO InterpreterRenameQuery::execute() for (auto & elem : descriptions) { database_catalog.assertTableDoesntExist(StorageID(elem.to_database_name, elem.to_table_name), context); - auto from_table = context.getTable(elem.from_database_name, elem.from_table_name); + auto from_table = database_catalog.getTable({elem.from_database_name, elem.from_table_name}); auto from_table_lock = from_table->lockExclusively(context.getCurrentQueryId()); database_catalog.getDatabase(elem.from_database_name)->renameTable( diff --git a/dbms/src/Interpreters/InterpreterSystemQuery.cpp b/dbms/src/Interpreters/InterpreterSystemQuery.cpp index ca6de13d0fd..81b22919e55 100644 --- a/dbms/src/Interpreters/InterpreterSystemQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSystemQuery.cpp @@ -313,14 +313,14 @@ BlockIO InterpreterSystemQuery::execute() StoragePtr InterpreterSystemQuery::tryRestartReplica(const String & database_name, const String & table_name, Context & system_context) { context.checkAccess(AccessType::RESTART_REPLICA, database_name, table_name); - auto database = DatabaseCatalog::instance().getDatabase(database_name, system_context); + auto database = DatabaseCatalog::instance().getDatabase(database_name); auto table_ddl_guard = DatabaseCatalog::instance().getDDLGuard(database_name, table_name); ASTPtr create_ast; /// Detach actions { - auto table = system_context.tryGetTable(database_name, table_name); + auto table = DatabaseCatalog::instance().tryGetTable({database_name, table_name}); if (!table || !dynamic_cast(table.get())) return nullptr; @@ -385,11 +385,11 @@ void InterpreterSystemQuery::restartReplicas(Context & system_context) void InterpreterSystemQuery::syncReplica(ASTSystemQuery & query) { - String database_name = !query.database.empty() ? query.database : context.getCurrentDatabase(); + String database_name = context.resolveDatabase(query.database); const String & table_name = query.table; context.checkAccess(AccessType::SYNC_REPLICA, database_name, table_name); - StoragePtr table = context.getTable(database_name, table_name); + StoragePtr table = DatabaseCatalog::instance().getTable({database_name, table_name}); if (auto storage_replicated = dynamic_cast(table.get())) { @@ -409,11 +409,11 @@ void InterpreterSystemQuery::syncReplica(ASTSystemQuery & query) void InterpreterSystemQuery::flushDistributed(ASTSystemQuery & query) { - String database_name = !query.database.empty() ? query.database : context.getCurrentDatabase(); + String database_name = context.resolveDatabase(query.database); String & table_name = query.table; context.checkAccess(AccessType::FLUSH_DISTRIBUTED, database_name, table_name); - if (auto storage_distributed = dynamic_cast(context.getTable(database_name, table_name).get())) + if (auto storage_distributed = dynamic_cast(DatabaseCatalog::instance().getTable({database_name, table_name}).get())) storage_distributed->flushClusterNodesAllData(); else throw Exception("Table " + database_name + "." + table_name + " is not distributed", ErrorCodes::BAD_ARGUMENTS); diff --git a/dbms/src/Interpreters/InterpreterWatchQuery.cpp b/dbms/src/Interpreters/InterpreterWatchQuery.cpp index 6a1fac7b97b..f03b91a2e42 100644 --- a/dbms/src/Interpreters/InterpreterWatchQuery.cpp +++ b/dbms/src/Interpreters/InterpreterWatchQuery.cpp @@ -41,28 +41,18 @@ BlockIO InterpreterWatchQuery::execute() BlockIO res; const ASTWatchQuery & query = typeid_cast(*query_ptr); - String database; - String table; - /// Get database - if (!query.database.empty()) - database = query.database; - else - database = context.getCurrentDatabase(); - - /// Get table - table = query.table; + StorageID table_id{query, context}; /// Get storage - storage = context.tryGetTable(database, table); + storage = DatabaseCatalog::instance().tryGetTable(table_id); if (!storage) - throw Exception("Table " + backQuoteIfNeed(database) + "." + - backQuoteIfNeed(table) + " doesn't exist.", + throw Exception("Table " + table_id.getNameForLogs() + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE); /// List of columns to read to execute the query. Names required_columns = storage->getColumns().getNamesOfPhysical(); - context.checkAccess(AccessType::SELECT, database, table, required_columns); + context.checkAccess(AccessType::SELECT, table_id.database_name, table_id.table_name, required_columns); /// Get context settings for this query const Settings & settings = context.getSettingsRef(); diff --git a/dbms/src/Interpreters/SystemLog.h b/dbms/src/Interpreters/SystemLog.h index 202f9ce2055..e0153506a40 100644 --- a/dbms/src/Interpreters/SystemLog.h +++ b/dbms/src/Interpreters/SystemLog.h @@ -127,8 +127,9 @@ protected: private: /* Saving thread data */ Context & context; - const String database_name; - const String table_name; + const StorageID table_id; + //const String database_name; + //const String table_name; const String storage_def; StoragePtr table; bool is_prepared = false; @@ -170,11 +171,12 @@ SystemLog::SystemLog(Context & context_, const String & table_name_, const String & storage_def_, size_t flush_interval_milliseconds_) - : context(context_), - database_name(database_name_), table_name(table_name_), storage_def(storage_def_), + : context(context_) + , table_id(database_name_, table_name_) + , storage_def(storage_def_), flush_interval_milliseconds(flush_interval_milliseconds_) { - log = &Logger::get("SystemLog (" + database_name + "." + table_name + ")"); + log = &Logger::get("SystemLog (" + database_name_ + "." + table_name_ + ")"); saving_thread = ThreadFromGlobalPool([this] { savingThreadFunction(); }); } @@ -338,8 +340,8 @@ void SystemLog::flushImpl(const std::vector & to_flush, /// This is needed to support DEFAULT-columns in table. std::unique_ptr insert = std::make_unique(); - insert->database = database_name; - insert->table = table_name; + insert->database = table_id.database_name; + insert->table = table_id.table_name; ASTPtr query_ptr(insert.release()); InterpreterInsertQuery interpreter(query_ptr, context); @@ -363,9 +365,9 @@ void SystemLog::flushImpl(const std::vector & to_flush, template void SystemLog::prepareTable() { - String description = backQuoteIfNeed(database_name) + "." + backQuoteIfNeed(table_name); + String description = table_id.getNameForLogs(); - table = context.tryGetTable(database_name, table_name); + table = DatabaseCatalog::instance().tryGetTable(table_id); if (table) { @@ -376,18 +378,18 @@ void SystemLog::prepareTable() { /// Rename the existing table. int suffix = 0; - while (DatabaseCatalog::instance().isTableExist({database_name, table_name + "_" + toString(suffix)}, context)) + while (DatabaseCatalog::instance().isTableExist({table_id.database_name, table_id.table_name + "_" + toString(suffix)}, context)) ++suffix; auto rename = std::make_shared(); ASTRenameQuery::Table from; - from.database = database_name; - from.table = table_name; + from.database = table_id.database_name; + from.table = table_id.table_name; ASTRenameQuery::Table to; - to.database = database_name; - to.table = table_name + "_" + toString(suffix); + to.database = table_id.database_name; + to.table = table_id.table_name + "_" + toString(suffix); ASTRenameQuery::Element elem; elem.from = from; @@ -414,8 +416,8 @@ void SystemLog::prepareTable() auto create = std::make_shared(); - create->database = database_name; - create->table = table_name; + create->database = table_id.database_name; + create->table = table_id.table_name; Block sample = LogElement::createBlock(); @@ -433,7 +435,7 @@ void SystemLog::prepareTable() interpreter.setInternal(true); interpreter.execute(); - table = context.getTable(database_name, table_name); + table = DatabaseCatalog::instance().getTable(table_id); } is_prepared = true; diff --git a/dbms/src/Storages/StorageBuffer.cpp b/dbms/src/Storages/StorageBuffer.cpp index 58cd66e4ee3..fbf80b3af00 100644 --- a/dbms/src/Storages/StorageBuffer.cpp +++ b/dbms/src/Storages/StorageBuffer.cpp @@ -5,9 +5,7 @@ #include #include #include -#include #include -#include #include #include #include @@ -25,8 +23,6 @@ #include #include #include -#include -#include #include #include #include @@ -68,17 +64,14 @@ StorageBuffer::StorageBuffer( size_t num_shards_, const Thresholds & min_thresholds_, const Thresholds & max_thresholds_, - const String & destination_database_, - const String & destination_table_, + const StorageID & destination_id_, bool allow_materialized_) : IStorage(table_id_) , global_context(context_) , num_shards(num_shards_), buffers(num_shards_) , min_thresholds(min_thresholds_) , max_thresholds(max_thresholds_) - , destination_database(destination_database_) - , destination_table(destination_table_) - , no_destination(destination_database.empty() && destination_table.empty()) + , destination_id(destination_id_) , allow_materialized(allow_materialized_) , log(&Logger::get("StorageBuffer (" + table_id_.getFullTableName() + ")")) { @@ -142,9 +135,9 @@ private: QueryProcessingStage::Enum StorageBuffer::getQueryProcessingStage(const Context & context) const { - if (!no_destination) + if (destination_id) { - auto destination = context.getTable(destination_database, destination_table); + auto destination = DatabaseCatalog::instance().getTable(destination_id); if (destination.get() == this) throw Exception("Destination table is myself. Read will cause infinite loop.", ErrorCodes::INFINITE_LOOP); @@ -186,9 +179,9 @@ Pipes StorageBuffer::readWithProcessors( { Pipes pipes_from_dst; - if (!no_destination) + if (destination_id) { - auto destination = context.getTable(destination_database, destination_table); + auto destination = DatabaseCatalog::instance().getTable(destination_id); if (destination.get() == this) throw Exception("Destination table is myself. Read will cause infinite loop.", ErrorCodes::INFINITE_LOOP); @@ -219,7 +212,7 @@ Pipes StorageBuffer::readWithProcessors( { if (!destination->hasColumn(column_name)) { - LOG_WARNING(log, "Destination table " << backQuoteIfNeed(destination_database) << "." << backQuoteIfNeed(destination_table) + LOG_WARNING(log, "Destination table " << destination_id.getNameForLogs() << " doesn't have column " << backQuoteIfNeed(column_name) << ". The default values are used."); boost::range::remove_erase(columns_intersection, column_name); continue; @@ -228,7 +221,7 @@ Pipes StorageBuffer::readWithProcessors( const auto & col = getColumn(column_name); if (!dst_col.type->equals(*col.type)) { - LOG_WARNING(log, "Destination table " << backQuoteIfNeed(destination_database) << "." << backQuoteIfNeed(destination_table) + LOG_WARNING(log, "Destination table " << destination_id.getNameForLogs() << " has different type of column " << backQuoteIfNeed(column_name) << " (" << dst_col.type->getName() << " != " << col.type->getName() << "). Data from destination table are converted."); header_after_adding_defaults.getByName(column_name) = ColumnWithTypeAndName(dst_col.type, column_name); @@ -237,7 +230,7 @@ Pipes StorageBuffer::readWithProcessors( if (columns_intersection.empty()) { - LOG_WARNING(log, "Destination table " << backQuoteIfNeed(destination_database) << "." << backQuoteIfNeed(destination_table) + LOG_WARNING(log, "Destination table " << destination_id.getNameForLogs() << " has no common columns with block in buffer. Block of data is skipped."); } else @@ -366,9 +359,9 @@ public: return; StoragePtr destination; - if (!storage.no_destination) + if (storage.destination_id) { - destination = storage.global_context.tryGetTable(storage.destination_database, storage.destination_table); + destination = DatabaseCatalog::instance().tryGetTable(storage.destination_id); if (destination.get() == &storage) throw Exception("Destination table is myself. Write will cause infinite loop.", ErrorCodes::INFINITE_LOOP); } @@ -378,7 +371,7 @@ public: /// If the block already exceeds the maximum limit, then we skip the buffer. if (rows > storage.max_thresholds.rows || bytes > storage.max_thresholds.bytes) { - if (!storage.no_destination) + if (storage.destination_id) { LOG_TRACE(storage.log, "Writing block with " << rows << " rows, " << bytes << " bytes directly."); storage.writeBlockToDestination(block, destination); @@ -462,10 +455,10 @@ BlockOutputStreamPtr StorageBuffer::write(const ASTPtr & /*query*/, const Contex bool StorageBuffer::mayBenefitFromIndexForIn(const ASTPtr & left_in_operand, const Context & query_context) const { - if (no_destination) + if (!destination_id) return false; - auto destination = global_context.getTable(destination_database, destination_table); + auto destination = DatabaseCatalog::instance().getTable(destination_id); if (destination.get() == this) throw Exception("Destination table is myself. Read will cause infinite loop.", ErrorCodes::INFINITE_LOOP); @@ -621,7 +614,7 @@ void StorageBuffer::flushBuffer(Buffer & buffer, bool check_thresholds, bool loc LOG_TRACE(log, "Flushing buffer with " << rows << " rows, " << bytes << " bytes, age " << time_passed << " seconds."); - if (no_destination) + if (!destination_id) return; /** For simplicity, buffer is locked during write. @@ -632,7 +625,7 @@ void StorageBuffer::flushBuffer(Buffer & buffer, bool check_thresholds, bool loc */ try { - writeBlockToDestination(block_to_write, global_context.tryGetTable(destination_database, destination_table)); + writeBlockToDestination(block_to_write, DatabaseCatalog::instance().tryGetTable(destination_id)); } catch (...) { @@ -656,12 +649,12 @@ void StorageBuffer::flushBuffer(Buffer & buffer, bool check_thresholds, bool loc void StorageBuffer::writeBlockToDestination(const Block & block, StoragePtr table) { - if (no_destination || !block) + if (!destination_id || !block) return; if (!table) { - LOG_ERROR(log, "Destination table " << backQuoteIfNeed(destination_database) << "." << backQuoteIfNeed(destination_table) << " doesn't exist. Block of data is discarded."); + LOG_ERROR(log, "Destination table " << destination_id.getNameForLogs() << " doesn't exist. Block of data is discarded."); return; } @@ -669,8 +662,8 @@ void StorageBuffer::writeBlockToDestination(const Block & block, StoragePtr tabl auto insert = std::make_shared(); - insert->database = destination_database; - insert->table = destination_table; + insert->database = destination_id.database_name; + insert->table = destination_id.table_name; /** We will insert columns that are the intersection set of columns of the buffer table and the subordinate table. * This will support some of the cases (but not all) when the table structure does not match. @@ -685,7 +678,7 @@ void StorageBuffer::writeBlockToDestination(const Block & block, StoragePtr tabl auto column = block.getByName(dst_col.name); if (!column.type->equals(*dst_col.type)) { - LOG_WARNING(log, "Destination table " << backQuoteIfNeed(destination_database) << "." << backQuoteIfNeed(destination_table) + LOG_WARNING(log, "Destination table " << destination_id.getNameForLogs() << " have different type of column " << backQuoteIfNeed(column.name) << " (" << dst_col.type->getName() << " != " << column.type->getName() << "). Block of data is converted."); @@ -699,14 +692,14 @@ void StorageBuffer::writeBlockToDestination(const Block & block, StoragePtr tabl if (block_to_write.columns() == 0) { - LOG_ERROR(log, "Destination table " << backQuoteIfNeed(destination_database) << "." << backQuoteIfNeed(destination_table) + LOG_ERROR(log, "Destination table " << destination_id.getNameForLogs() << " have no common columns with block in buffer. Block of data is discarded."); return; } if (block_to_write.columns() != block.columns()) LOG_WARNING(log, "Not all columns from block in buffer exist in destination table " - << backQuoteIfNeed(destination_database) << "." << backQuoteIfNeed(destination_table) << ". Some columns are discarded."); + << destination_id.getNameForLogs() << ". Some columns are discarded."); auto list_of_columns = std::make_shared(); insert->columns = list_of_columns; @@ -788,6 +781,7 @@ void registerStorageBuffer(StorageFactory & factory) " destination_database, destination_table, num_buckets, min_time, max_time, min_rows, max_rows, min_bytes, max_bytes.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + //FIXME currentDatabase() at the moment of table creation can be different from currentDatabase() at the moment when table is loaded|used engine_args[0] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[0], args.local_context); engine_args[1] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[1], args.local_context); @@ -803,6 +797,14 @@ void registerStorageBuffer(StorageFactory & factory) UInt64 min_bytes = applyVisitor(FieldVisitorConvertToNumber(), engine_args[7]->as().value); UInt64 max_bytes = applyVisitor(FieldVisitorConvertToNumber(), engine_args[8]->as().value); + /// If destination_id is not set, do not write data from the buffer, but simply empty the buffer. + StorageID destination_id = StorageID::createEmpty(); + if (!destination_table.empty()) + { + destination_id.database_name = args.context.resolveDatabase(destination_database); + destination_id.table_name = destination_table; + } + return StorageBuffer::create( args.table_id, args.columns, @@ -811,7 +813,7 @@ void registerStorageBuffer(StorageFactory & factory) num_buckets, StorageBuffer::Thresholds{min_time, min_rows, min_bytes}, StorageBuffer::Thresholds{max_time, max_rows, max_bytes}, - destination_database, destination_table, + destination_id, static_cast(args.local_context.getSettingsRef().insert_allow_materialized_columns)); }); } diff --git a/dbms/src/Storages/StorageBuffer.h b/dbms/src/Storages/StorageBuffer.h index d10288dacc7..772a8af2497 100644 --- a/dbms/src/Storages/StorageBuffer.h +++ b/dbms/src/Storages/StorageBuffer.h @@ -76,9 +76,9 @@ public: bool supportsSampling() const override { return true; } bool supportsPrewhere() const override { - if (no_destination) + if (!destination_id) return false; - auto dest = global_context.tryGetTable(destination_database, destination_table); + auto dest = DatabaseCatalog::instance().tryGetTable(destination_id); if (dest && dest.get() != this) return dest->supportsPrewhere(); return false; @@ -112,9 +112,7 @@ private: const Thresholds min_thresholds; const Thresholds max_thresholds; - const String destination_database; - const String destination_table; - bool no_destination; /// If set, do not write data from the buffer, but simply empty the buffer. + StorageID destination_id; bool allow_materialized; Poco::Logger * log; @@ -146,8 +144,7 @@ protected: size_t num_shards_, const Thresholds & min_thresholds_, const Thresholds & max_thresholds_, - const String & destination_database_, - const String & destination_table_, + const StorageID & destination_id, bool allow_materialized_); }; diff --git a/dbms/src/Storages/StorageID.cpp b/dbms/src/Storages/StorageID.cpp new file mode 100644 index 00000000000..2605b4e2c31 --- /dev/null +++ b/dbms/src/Storages/StorageID.cpp @@ -0,0 +1,50 @@ +#include +#include +#include +#include +#include + +namespace DB +{ + +StorageID::StorageID(const ASTQueryWithTableAndOutput & query, const Context & local_context) +{ + database_name = local_context.resolveDatabase(query.database); + table_name = query.table; + uuid = query.uuid; + assertNotEmpty(); +} + +String StorageID::getNameForLogs() const +{ + assertNotEmpty(); + return (database_name.empty() ? "" : backQuoteIfNeed(database_name) + ".") + backQuoteIfNeed(table_name) + + (hasUUID() ? " (UUID " + toString(uuid) + ")" : ""); +} + +bool StorageID::operator<(const StorageID & rhs) const +{ + assertNotEmpty(); + /// It's needed for ViewDependencies + if (!hasUUID() && !rhs.hasUUID()) + /// If both IDs don't have UUID, compare them like pair of strings + return std::tie(database_name, table_name) < std::tie(rhs.database_name, rhs.table_name); + else if (hasUUID() && rhs.hasUUID()) + /// If both IDs have UUID, compare UUIDs and ignore database and table name + return uuid < rhs.uuid; + else + /// All IDs without UUID are less, then all IDs with UUID + return !hasUUID(); +} + +void StorageID::assertNotEmpty() const +{ + if (empty()) + throw Exception("Both table name and UUID are empty", ErrorCodes::LOGICAL_ERROR); + if (table_name == TABLE_WITH_UUID_NAME_PLACEHOLDER && !hasUUID()) + throw Exception("Table name was replaced with placeholder, but UUID is Nil", ErrorCodes::LOGICAL_ERROR); + if (table_name.empty() && !database_name.empty()) + throw Exception("Table name is empty, but database name is not", ErrorCodes::LOGICAL_ERROR); +} + +} diff --git a/dbms/src/Storages/StorageID.h b/dbms/src/Storages/StorageID.h index b93a62bd6ba..b940d3b4180 100644 --- a/dbms/src/Storages/StorageID.h +++ b/dbms/src/Storages/StorageID.h @@ -1,8 +1,6 @@ #pragma once #include #include -#include -#include #include namespace DB @@ -15,19 +13,24 @@ namespace ErrorCodes static constexpr char const * TABLE_WITH_UUID_NAME_PLACEHOLDER = "_"; +class ASTQueryWithTableAndOutput; +class Context; + struct StorageID { String database_name; String table_name; - UUID uuid = UUID{UInt128(0, 0)}; + UUID uuid = UUIDHelpers::Nil; - StorageID(const String & database, const String & table, UUID uuid_ = UUID{UInt128(0, 0)}) + StorageID(const String & database, const String & table, UUID uuid_ = UUIDHelpers::Nil) : database_name(database), table_name(table), uuid(uuid_) { assertNotEmpty(); } + StorageID(const ASTQueryWithTableAndOutput & query, const Context & local_context); + String getDatabaseName() const { assertNotEmpty(); @@ -46,12 +49,7 @@ struct StorageID return (database_name.empty() ? "" : database_name + ".") + table_name; } - String getNameForLogs() const - { - assertNotEmpty(); - return (database_name.empty() ? "" : backQuoteIfNeed(database_name) + ".") + backQuoteIfNeed(table_name) - + (hasUUID() ? " (UUID " + toString(uuid) + ")" : ""); - } + String getNameForLogs() const; explicit operator bool () const { @@ -68,30 +66,9 @@ struct StorageID return uuid != UUID{UInt128(0, 0)}; } - bool operator<(const StorageID & rhs) const - { - assertNotEmpty(); - /// It's needed for ViewDependencies - if (!hasUUID() && !rhs.hasUUID()) - /// If both IDs don't have UUID, compare them like pair of strings - return std::tie(database_name, table_name) < std::tie(rhs.database_name, rhs.table_name); - else if (hasUUID() && rhs.hasUUID()) - /// If both IDs have UUID, compare UUIDs and ignore database and table name - return uuid < rhs.uuid; - else - /// All IDs without UUID are less, then all IDs with UUID - return !hasUUID(); - } + bool operator<(const StorageID & rhs) const; - void assertNotEmpty() const - { - if (empty()) - throw Exception("Both table name and UUID are empty", ErrorCodes::LOGICAL_ERROR); - if (table_name == TABLE_WITH_UUID_NAME_PLACEHOLDER && !hasUUID()) - throw Exception("Table name was replaced with placeholder, but UUID is Nil", ErrorCodes::LOGICAL_ERROR); - if (table_name.empty() && !database_name.empty()) - throw Exception("Table name is empty, but database name is not", ErrorCodes::LOGICAL_ERROR); - } + void assertNotEmpty() const; /// Avoid implicit construction of empty StorageID. However, it's needed for deferred initialization. static StorageID createEmpty() { return {}; } diff --git a/dbms/src/Storages/StorageMaterializedView.cpp b/dbms/src/Storages/StorageMaterializedView.cpp index 11e9acb248a..349ff9c34da 100644 --- a/dbms/src/Storages/StorageMaterializedView.cpp +++ b/dbms/src/Storages/StorageMaterializedView.cpp @@ -146,7 +146,7 @@ StorageMaterializedView::StorageMaterializedView( create_interpreter.setInternal(true); create_interpreter.execute(); - target_table_id = global_context.getTable(manual_create_query->database, manual_create_query->table)->getStorageID(); + target_table_id = DatabaseCatalog::instance().getTable({manual_create_query->database, manual_create_query->table})->getStorageID(); } if (!select_table_id.empty()) diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index bce6214e1ae..56f0da2eb81 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -1044,8 +1044,8 @@ void StorageMergeTree::alterPartition(const ASTPtr & query, const PartitionComma break; case PartitionCommand::MoveDestinationType::TABLE: checkPartitionCanBeDropped(command.partition); - String dest_database = command.to_database.empty() ? context.getCurrentDatabase() : command.to_database; - auto dest_storage = context.getTable(dest_database, command.to_table); + String dest_database = context.resolveDatabase(command.to_database); + auto dest_storage = DatabaseCatalog::instance().getTable({dest_database, command.to_table}); movePartitionToTable(dest_storage, command.partition, context); break; } @@ -1056,8 +1056,8 @@ void StorageMergeTree::alterPartition(const ASTPtr & query, const PartitionComma case PartitionCommand::REPLACE_PARTITION: { checkPartitionCanBeDropped(command.partition); - String from_database = command.from_database.empty() ? context.getCurrentDatabase() : command.from_database; - auto from_storage = context.getTable(from_database, command.from_table); + String from_database = context.resolveDatabase(command.from_database); + auto from_storage = DatabaseCatalog::instance().getTable({from_database, command.from_table}); replacePartitionFrom(from_storage, command.partition, command.replace, context); } break; diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index a2931a35f31..6c11b1a5662 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -1670,14 +1670,14 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) StoragePtr source_table; TableStructureReadLockHolder table_lock_holder_src_table; - String source_table_name = entry_replace.from_database + "." + entry_replace.from_table; + StorageID source_table_id{entry_replace.from_database, entry_replace.from_table}; auto clone_data_parts_from_source_table = [&] () -> size_t { - source_table = global_context.tryGetTable(entry_replace.from_database, entry_replace.from_table); + source_table = DatabaseCatalog::instance().tryGetTable(source_table_id); if (!source_table) { - LOG_DEBUG(log, "Can't use " << source_table_name << " as source table for REPLACE PARTITION command. It does not exist."); + LOG_DEBUG(log, "Can't use " << source_table_id.getNameForLogs() << " as source table for REPLACE PARTITION command. It does not exist."); return 0; } @@ -1688,7 +1688,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) } catch (Exception &) { - LOG_INFO(log, "Can't use " << source_table_name << " as source table for REPLACE PARTITION command. Will fetch all parts." + LOG_INFO(log, "Can't use " << source_table_id.getNameForLogs() << " as source table for REPLACE PARTITION command. Will fetch all parts." << " Reason: " << getCurrentExceptionMessage(false)); return 0; } @@ -1704,7 +1704,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) auto src_part = src_data->getPartIfExists(part_desc->src_part_info, valid_states); if (!src_part) { - LOG_DEBUG(log, "There is no part " << part_desc->src_part_name << " in " << source_table_name); + LOG_DEBUG(log, "There is no part " << part_desc->src_part_name << " in " << source_table_id.getNameForLogs()); continue; } @@ -1716,7 +1716,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) if (checksum_hex != part_desc->checksum_hex) { - LOG_DEBUG(log, "Part " << part_desc->src_part_name << " of " << source_table_name << " has inappropriate checksum"); + LOG_DEBUG(log, "Part " << part_desc->src_part_name << " of " << source_table_id.getNameForLogs() << " has inappropriate checksum"); /// TODO: check version continue; } @@ -3556,8 +3556,8 @@ void StorageReplicatedMergeTree::alterPartition(const ASTPtr & query, const Part break; case PartitionCommand::MoveDestinationType::TABLE: checkPartitionCanBeDropped(command.partition); - String dest_database = command.to_database.empty() ? query_context.getCurrentDatabase() : command.to_database; - auto dest_storage = query_context.getTable(dest_database, command.to_table); + String dest_database = query_context.resolveDatabase(command.to_database); + auto dest_storage = DatabaseCatalog::instance().getTable({dest_database, command.to_table}); movePartitionToTable(dest_storage, command.partition, query_context); break; } @@ -3567,8 +3567,8 @@ void StorageReplicatedMergeTree::alterPartition(const ASTPtr & query, const Part case PartitionCommand::REPLACE_PARTITION: { checkPartitionCanBeDropped(command.partition); - String from_database = command.from_database.empty() ? query_context.getCurrentDatabase() : command.from_database; - auto from_storage = query_context.getTable(from_database, command.from_table); + String from_database = query_context.resolveDatabase(command.from_database); + auto from_storage = DatabaseCatalog::instance().getTable({from_database, command.from_table}); replacePartitionFrom(from_storage, command.partition, command.replace, query_context); } break; diff --git a/dbms/src/Storages/StorageS3.cpp b/dbms/src/Storages/StorageS3.cpp index 4cdea957b3d..42984522d23 100644 --- a/dbms/src/Storages/StorageS3.cpp +++ b/dbms/src/Storages/StorageS3.cpp @@ -28,6 +28,7 @@ #include #include +#include #include From a739e4274048d241a57047f687ace2b83a96eade Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 19 Feb 2020 21:41:22 +0300 Subject: [PATCH 025/712] Better build time --- dbms/programs/server/Server.cpp | 2 +- dbms/src/Interpreters/Context.cpp | 25 +++++++++++++++++++++++++ dbms/src/Interpreters/Context.h | 25 ++++--------------------- 3 files changed, 30 insertions(+), 22 deletions(-) diff --git a/dbms/programs/server/Server.cpp b/dbms/programs/server/Server.cpp index b97ed1cea2a..2ea4fa24644 100644 --- a/dbms/programs/server/Server.cpp +++ b/dbms/programs/server/Server.cpp @@ -1009,7 +1009,7 @@ int Server::main(const std::vector & /*args*/) global_context->getConfigRef(), graphite_key, async_metrics)); } - SessionCleaner session_cleaner(*global_context); + global_context->createSessionCleaner(); waitForTerminationRequest(); } diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index f6e280c9e2c..2c19110a8d7 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -2153,6 +2153,26 @@ void Context::resetInputCallbacks() } +class SessionCleaner +{ +public: + SessionCleaner(Context & context_) + : context{context_} + { + } + ~SessionCleaner(); + +private: + void run(); + + Context & context; + + std::mutex mutex; + std::condition_variable cond; + std::atomic quit{false}; + ThreadFromGlobalPool thread{&SessionCleaner::run, this}; +}; + SessionCleaner::~SessionCleaner() { try @@ -2187,5 +2207,10 @@ void SessionCleaner::run() } } +void Context::createSessionCleaner() +{ + session_cleaner = std::make_unique(*this); +} + } diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index 07fa6b06c1f..74bffcd3d07 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -93,6 +93,7 @@ class DiskSelector; class StoragePolicy; using StoragePolicyPtr = std::shared_ptr; class StoragePolicySelector; +class SessionCleaner; class IOutputFormat; using OutputFormatPtr = std::shared_ptr; @@ -186,6 +187,7 @@ private: Context * session_context = nullptr; /// Session context or nullptr. Could be equal to this. Context * global_context = nullptr; /// Global context. Could be equal to this. + std::unique_ptr session_cleaner; /// It will launch a thread to clean old named HTTP sessions. See 'createSessionCleaner'. UInt64 session_close_cycle = 0; bool session_is_used = false; @@ -593,6 +595,8 @@ public: void dropCompiledExpressionCache() const; #endif + void createSessionCleaner(); + /// Add started bridge command. It will be killed after context destruction void addXDBCBridgeCommand(std::unique_ptr cmd) const; @@ -663,25 +667,4 @@ private: std::unique_lock table_lock; }; - -class SessionCleaner -{ -public: - SessionCleaner(Context & context_) - : context{context_} - { - } - ~SessionCleaner(); - -private: - void run(); - - Context & context; - - std::mutex mutex; - std::condition_variable cond; - std::atomic quit{false}; - ThreadFromGlobalPool thread{&SessionCleaner::run, this}; -}; - } From f888c6ac882818c7a5068bc2d47e047c0bfe06dc Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 20 Feb 2020 16:21:58 +0300 Subject: [PATCH 026/712] Fixed build --- dbms/src/Interpreters/Context.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index 74bffcd3d07..1119323dd0f 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -187,7 +187,7 @@ private: Context * session_context = nullptr; /// Session context or nullptr. Could be equal to this. Context * global_context = nullptr; /// Global context. Could be equal to this. - std::unique_ptr session_cleaner; /// It will launch a thread to clean old named HTTP sessions. See 'createSessionCleaner'. + std::shared_ptr session_cleaner; /// It will launch a thread to clean old named HTTP sessions. See 'createSessionCleaner'. UInt64 session_close_cycle = 0; bool session_is_used = false; From 29a993a66f8b04d90ce8e5a427fb3aa5c5a71ee0 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 21 Feb 2020 18:22:28 +0300 Subject: [PATCH 027/712] remove another method from Context --- dbms/src/Interpreters/Context.cpp | 13 ------------- dbms/src/Interpreters/Context.h | 8 +++----- .../Interpreters/InterpreterShowCreateQuery.cpp | 16 ++++++++-------- dbms/src/Storages/StorageMerge.cpp | 4 ++-- 4 files changed, 13 insertions(+), 28 deletions(-) diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index 8b06eed08f0..9343e41e490 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -844,19 +844,6 @@ StoragePtr Context::getViewSource() return view_source; } -ASTPtr Context::getCreateExternalTableQuery(const String & table_name) const -{ - StorageID external_id = StorageID::createEmpty(); - { - auto lock = getLock(); - auto it = external_tables_mapping.find(table_name); - if (external_tables_mapping.end() == it) - throw Exception("Temporary table " + backQuoteIfNeed(table_name) + " doesn't exist", ErrorCodes::UNKNOWN_TABLE); - external_id = it->second->getGlobalTableID(); - } - return DatabaseCatalog::instance().getDatabaseForTemporaryTables()->getCreateTableQuery(*this, external_id.table_name); -} - Settings Context::getSettings() const { return settings; diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index d643dee9e2c..abc0f97ad50 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -175,8 +175,11 @@ private: using TemporaryTablesMapping = std::map>; TemporaryTablesMapping external_tables_mapping; Scalars scalars; + + //TODO maybe replace with temporary tables? StoragePtr view_source; /// Temporary StorageValues used to generate alias columns for materialized views Tables table_function_results; /// Temporary tables obtained by execution of table functions. Keyed by AST tree id. + Context * query_context = nullptr; Context * session_context = nullptr; /// Session context or nullptr. Could be equal to this. Context * global_context = nullptr; /// Global context. Could be equal to this. @@ -385,9 +388,6 @@ public: std::optional getTCPPortSecure() const; - /// Get query for the CREATE table. - ASTPtr getCreateExternalTableQuery(const String & table_name) const; - std::shared_ptr acquireSession(const String & session_id, std::chrono::steady_clock::duration timeout, bool session_check) const; void releaseSession(const String & session_id, std::chrono::steady_clock::duration timeout); @@ -595,8 +595,6 @@ private: EmbeddedDictionaries & getEmbeddedDictionariesImpl(bool throw_on_error) const; - StoragePtr getTableImpl(const StorageID & table_id, std::optional * exception) const; - SessionKey getSessionKey(const String & session_id) const; /// Session will be closed after specified timeout. diff --git a/dbms/src/Interpreters/InterpreterShowCreateQuery.cpp b/dbms/src/Interpreters/InterpreterShowCreateQuery.cpp index 4f59b6665d5..758fef1dba5 100644 --- a/dbms/src/Interpreters/InterpreterShowCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterShowCreateQuery.cpp @@ -47,18 +47,17 @@ BlockInputStreamPtr InterpreterShowCreateQuery::executeImpl() ASTQueryWithTableAndOutput * show_query; if ((show_query = query_ptr->as())) { - if (show_query->temporary) - create_query = context.getCreateExternalTableQuery(show_query->table); - else - { - context.checkAccess(AccessType::SHOW, show_query->database, show_query->table); - create_query = DatabaseCatalog::instance().getDatabase(show_query->database, context)->getCreateTableQuery(context, show_query->table); - } + StorageID table_id{show_query->database, show_query->table}; + auto resolve_table_type = show_query->temporary ? Context::ResolveExternal : Context::ResolveOrdinary; + table_id = context.resolveStorageID(table_id, resolve_table_type); + context.checkAccess(AccessType::SHOW, table_id.database_name, table_id.table_name); + create_query = DatabaseCatalog::instance().getDatabase(table_id.database_name)->getCreateTableQuery(context, table_id.table_name); } else if ((show_query = query_ptr->as())) { if (show_query->temporary) throw Exception("Temporary databases are not possible.", ErrorCodes::SYNTAX_ERROR); + show_query->database = context.resolveDatabase(show_query->database); context.checkAccess(AccessType::SHOW, show_query->database); create_query = DatabaseCatalog::instance().getDatabase(show_query->database)->getCreateDatabaseQuery(context); } @@ -66,8 +65,9 @@ BlockInputStreamPtr InterpreterShowCreateQuery::executeImpl() { if (show_query->temporary) throw Exception("Temporary dictionaries are not possible.", ErrorCodes::SYNTAX_ERROR); + show_query->database = context.resolveDatabase(show_query->database); context.checkAccess(AccessType::SHOW, show_query->database, show_query->table); - create_query = DatabaseCatalog::instance().getDatabase(show_query->database, context)->getCreateDictionaryQuery(context, show_query->table); + create_query = DatabaseCatalog::instance().getDatabase(show_query->database)->getCreateDictionaryQuery(context, show_query->table); } if (!create_query && show_query && show_query->temporary) diff --git a/dbms/src/Storages/StorageMerge.cpp b/dbms/src/Storages/StorageMerge.cpp index 15c90106824..dd2c06c6b8f 100644 --- a/dbms/src/Storages/StorageMerge.cpp +++ b/dbms/src/Storages/StorageMerge.cpp @@ -396,10 +396,10 @@ StorageMerge::StorageListWithLocks StorageMerge::getSelectedTables(const ASTPtr } -DatabaseTablesIteratorPtr StorageMerge::getDatabaseIterator(const Context & context) const +DatabaseTablesIteratorPtr StorageMerge::getDatabaseIterator() const { checkStackSize(); - auto database = DatabaseCatalog::instance().getDatabase(source_database, context); + auto database = DatabaseCatalog::instance().getDatabase(source_database); auto table_name_match = [this](const String & table_name_) { return table_name_regexp.match(table_name_); }; return database->getTablesIterator(global_context, table_name_match); } From 1e013c742ac039a39c96626f81907d0d8d1e5eb0 Mon Sep 17 00:00:00 2001 From: Denis Zhuravlev Date: Tue, 25 Feb 2020 16:00:28 -0400 Subject: [PATCH 028/712] Update settings.md max_insert_threads RU description --- docs/ru/operations/settings/settings.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 298dd7364c3..04039393e51 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -550,6 +550,10 @@ log_query_threads=1 Чем меньше `max_threads`, тем меньше будет использоваться оперативки. +## max_insert_threads {#settings-max_insert_threads} + +Максимальное количество потоков для выполнения запроса `INSERT SELECT`. По умолчанию (0 - auto) определяется автоматически исходя из количества потоков данных которое выдает SELECT. Если параметр max_insert_threads задан (не 0), то будет использовано минимальное значение из заданного и автоматического. + ## max_compress_block_size Максимальный размер блоков не сжатых данных перед сжатием при записи в таблицу. По умолчанию - 1 048 576 (1 MiB). При уменьшении размера, незначительно уменьшается коэффициент сжатия, незначительно возрастает скорость сжатия и разжатия за счёт кэш-локальности, и уменьшается потребление оперативки. Как правило, не имеет смысла менять эту настройку. From c509e9038f14ed3304b2f2dd21bee6ec861f5909 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 26 Feb 2020 13:16:32 +0300 Subject: [PATCH 029/712] trigger tests again --- docs/ru/operations/settings/settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 04039393e51..f71b24ff269 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -552,7 +552,7 @@ log_query_threads=1 ## max_insert_threads {#settings-max_insert_threads} -Максимальное количество потоков для выполнения запроса `INSERT SELECT`. По умолчанию (0 - auto) определяется автоматически исходя из количества потоков данных которое выдает SELECT. Если параметр max_insert_threads задан (не 0), то будет использовано минимальное значение из заданного и автоматического. +Максимальное количество потоков для выполнения запроса `INSERT SELECT`. По умолчанию (0 - auto) определяется автоматически исходя из количества потоков данных которое выдает SELECT. Если параметр `max_insert_threads` задан (не 0), то будет использовано минимальное значение из заданного и автоматического. ## max_compress_block_size From 44226a61cf906e82878e56bc142ab5869b91eebc Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Thu, 27 Feb 2020 19:47:40 +0300 Subject: [PATCH 030/712] Storage MergeTree initial support for S3. --- .../CachedCompressedReadBuffer.cpp | 9 +- .../Compression/CachedCompressedReadBuffer.h | 7 +- dbms/src/Disks/DiskLocal.cpp | 2 + dbms/src/Disks/DiskMemory.cpp | 15 +- dbms/src/Disks/DiskS3.cpp | 48 ++-- dbms/src/Disks/IDisk.h | 3 + .../Storages/MergeTree/IMergeTreeDataPart.cpp | 241 +++++++++--------- .../Storages/MergeTree/IMergeTreeDataPart.h | 7 +- .../MergeTree/IMergeTreeDataPartWriter.cpp | 25 +- .../MergeTree/IMergeTreeDataPartWriter.h | 8 +- .../Storages/MergeTree/IMergeTreeReader.cpp | 6 +- .../src/Storages/MergeTree/IMergeTreeReader.h | 2 - .../MergeTree/IMergedBlockOutputStream.cpp | 3 +- .../MergeTree/IMergedBlockOutputStream.h | 1 + dbms/src/Storages/MergeTree/MergeTreeData.cpp | 97 +++---- dbms/src/Storages/MergeTree/MergeTreeData.h | 1 + .../MergeTree/MergeTreeDataPartChecksum.cpp | 15 +- .../MergeTree/MergeTreeDataPartChecksum.h | 13 +- .../MergeTree/MergeTreeDataPartCompact.cpp | 2 +- .../MergeTree/MergeTreeDataPartWide.cpp | 28 +- .../MergeTreeDataPartWriterCompact.cpp | 4 +- .../MergeTreeDataPartWriterCompact.h | 1 + .../MergeTree/MergeTreeDataPartWriterWide.cpp | 4 +- .../MergeTree/MergeTreeDataPartWriterWide.h | 1 + .../MergeTree/MergeTreeDataSelectExecutor.cpp | 2 +- .../MergeTreeIndexGranularityInfo.cpp | 17 +- .../MergeTree/MergeTreeIndexGranularityInfo.h | 5 +- .../MergeTree/MergeTreeIndexReader.cpp | 3 +- .../MergeTree/MergeTreeMarksLoader.cpp | 23 +- .../Storages/MergeTree/MergeTreeMarksLoader.h | 3 + .../MergeTree/MergeTreeMutationEntry.cpp | 44 ++-- .../MergeTree/MergeTreeMutationEntry.h | 8 +- .../Storages/MergeTree/MergeTreePartition.cpp | 23 +- .../Storages/MergeTree/MergeTreePartition.h | 9 +- .../MergeTree/MergeTreeReaderCompact.cpp | 19 +- .../MergeTree/MergeTreeReaderStream.cpp | 18 +- .../MergeTree/MergeTreeReaderStream.h | 2 + .../MergeTree/MergeTreeReaderWide.cpp | 6 +- .../MergeTreeReverseSelectProcessor.cpp | 2 +- .../MergeTree/MergeTreeSelectProcessor.cpp | 2 +- .../Storages/MergeTree/MergeTreeSettings.h | 1 + ...rgeTreeThreadSelectBlockInputProcessor.cpp | 2 +- .../MergeTree/MergedBlockOutputStream.cpp | 27 +- .../MergeTree/MergedBlockOutputStream.h | 2 - dbms/src/Storages/StorageMergeTree.cpp | 21 +- .../test_merge_tree_s3/__init__.py | 0 .../configs/config.d/log_conf.xml | 12 + .../test_merge_tree_s3/configs/config.xml | 40 +++ .../test_merge_tree_s3/configs/users.xml | 23 ++ .../integration/test_merge_tree_s3/test.py | 91 +++++++ 50 files changed, 588 insertions(+), 360 deletions(-) create mode 100644 dbms/tests/integration/test_merge_tree_s3/__init__.py create mode 100644 dbms/tests/integration/test_merge_tree_s3/configs/config.d/log_conf.xml create mode 100644 dbms/tests/integration/test_merge_tree_s3/configs/config.xml create mode 100644 dbms/tests/integration/test_merge_tree_s3/configs/users.xml create mode 100644 dbms/tests/integration/test_merge_tree_s3/test.py diff --git a/dbms/src/Compression/CachedCompressedReadBuffer.cpp b/dbms/src/Compression/CachedCompressedReadBuffer.cpp index 878c980d53f..00cb5fe3fe9 100644 --- a/dbms/src/Compression/CachedCompressedReadBuffer.cpp +++ b/dbms/src/Compression/CachedCompressedReadBuffer.cpp @@ -72,11 +72,18 @@ bool CachedCompressedReadBuffer::nextImpl() } +CachedCompressedReadBuffer::CachedCompressedReadBuffer(std::unique_ptr file_in_, UncompressedCache * cache_) + : ReadBuffer(nullptr, 0), cache(cache_), file_in(std::move(file_in_)), path(file_in->getFileName()), file_pos(0) +{ + compressed_in = file_in.get(); +} + + CachedCompressedReadBuffer::CachedCompressedReadBuffer( const std::string & path_, UncompressedCache * cache_, size_t estimated_size_, size_t aio_threshold_, size_t mmap_threshold_, size_t buf_size_) - : ReadBuffer(nullptr, 0), path(path_), cache(cache_), buf_size(buf_size_), estimated_size(estimated_size_), + : ReadBuffer(nullptr, 0), cache(cache_), path(path_), buf_size(buf_size_), estimated_size(estimated_size_), aio_threshold(aio_threshold_), mmap_threshold(mmap_threshold_), file_pos(0) { } diff --git a/dbms/src/Compression/CachedCompressedReadBuffer.h b/dbms/src/Compression/CachedCompressedReadBuffer.h index 8821a824a0f..2fcfbdba51b 100644 --- a/dbms/src/Compression/CachedCompressedReadBuffer.h +++ b/dbms/src/Compression/CachedCompressedReadBuffer.h @@ -20,14 +20,15 @@ namespace DB class CachedCompressedReadBuffer : public CompressedReadBufferBase, public ReadBuffer { private: - const std::string path; UncompressedCache * cache; + std::unique_ptr file_in; + + const std::string path; size_t buf_size; size_t estimated_size; size_t aio_threshold; size_t mmap_threshold; - std::unique_ptr file_in; size_t file_pos; /// A piece of data from the cache, or a piece of read data that we put into the cache. @@ -41,6 +42,8 @@ private: clockid_t clock_type {}; public: + CachedCompressedReadBuffer(std::unique_ptr file_in, UncompressedCache * cache_); + CachedCompressedReadBuffer( const std::string & path_, UncompressedCache * cache_, size_t estimated_size_, size_t aio_threshold_, size_t mmap_threshold_, diff --git a/dbms/src/Disks/DiskLocal.cpp b/dbms/src/Disks/DiskLocal.cpp index f4197497924..6ac305cd943 100644 --- a/dbms/src/Disks/DiskLocal.cpp +++ b/dbms/src/Disks/DiskLocal.cpp @@ -67,6 +67,8 @@ public: return dir_path + iter.name(); } + String name() const override { return iter.name(); } + private: String dir_path; Poco::DirectoryIterator iter; diff --git a/dbms/src/Disks/DiskMemory.cpp b/dbms/src/Disks/DiskMemory.cpp index e8634147959..b939a78f230 100644 --- a/dbms/src/Disks/DiskMemory.cpp +++ b/dbms/src/Disks/DiskMemory.cpp @@ -6,6 +6,7 @@ #include #include #include +#include namespace DB @@ -23,7 +24,7 @@ namespace ErrorCodes class DiskMemoryDirectoryIterator : public IDiskDirectoryIterator { public: - explicit DiskMemoryDirectoryIterator(std::vector && dir_file_paths_) + explicit DiskMemoryDirectoryIterator(std::vector && dir_file_paths_) : dir_file_paths(std::move(dir_file_paths_)), iter(dir_file_paths.begin()) { } @@ -32,11 +33,13 @@ public: bool isValid() const override { return iter != dir_file_paths.end(); } - String path() const override { return *iter; } + String path() const override { return (*iter).toString(); } + + String name() const override { return (*iter).getFileName(); } private: - std::vector dir_file_paths; - std::vector::iterator iter; + std::vector dir_file_paths; + std::vector::iterator iter; }; /// Adapter with actual behaviour as ReadBufferFromString. @@ -264,10 +267,10 @@ DiskDirectoryIteratorPtr DiskMemory::iterateDirectory(const String & path) if (!path.empty() && files.find(path) == files.end()) throw Exception("Directory '" + path + "' does not exist", ErrorCodes::DIRECTORY_DOESNT_EXIST); - std::vector dir_file_paths; + std::vector dir_file_paths; for (const auto & file : files) if (parentPath(file.first) == path) - dir_file_paths.push_back(file.first); + dir_file_paths.emplace_back(file.first); return std::make_unique(std::move(dir_file_paths)); } diff --git a/dbms/src/Disks/DiskS3.cpp b/dbms/src/Disks/DiskS3.cpp index e91dbf55f8d..ef0f4e07356 100644 --- a/dbms/src/Disks/DiskS3.cpp +++ b/dbms/src/Disks/DiskS3.cpp @@ -157,19 +157,36 @@ namespace off_t seek(off_t offset_, int whence) override { - if (whence != SEEK_SET) - throw Exception("Only SEEK_SET mode is allowed.", ErrorCodes::CANNOT_SEEK_THROUGH_FILE); + if (whence == SEEK_CUR) + { + /// If position within current working buffer - shift pos. + if (working_buffer.size() && size_t(getPosition() + offset_) < absolute_position) + { + pos += offset_; + return getPosition(); + } + else + { + absolute_position += offset_; + } + } + else if (whence == SEEK_SET) + { + /// If position within current working buffer - shift pos. + if (working_buffer.size() && size_t(offset_) >= absolute_position - working_buffer.size() + && size_t(offset_) < absolute_position) + { + pos = working_buffer.end() - (absolute_position - offset_); + return getPosition(); + } + else + { + absolute_position = offset_; + } + } + else + throw Exception("Only SEEK_SET or SEEK_CUR modes are allowed.", ErrorCodes::CANNOT_SEEK_THROUGH_FILE); - if (offset_ < 0 || metadata.total_size <= static_cast(offset_)) - throw Exception( - "Seek position is out of bounds. " - "Offset: " - + std::to_string(offset_) + ", Max: " + std::to_string(metadata.total_size), - ErrorCodes::SEEK_POSITION_OUT_OF_BOUND); - - absolute_position = offset_; - - /// TODO: Do not re-initialize buffer if current position within working buffer. current_buf = initialize(); pos = working_buffer.end(); @@ -187,8 +204,7 @@ namespace for (UInt32 i = 0; i < metadata.s3_objects_count; ++i) { current_buf_idx = i; - auto path = metadata.s3_objects[i].first; - auto size = metadata.s3_objects[i].second; + auto [path, size] = metadata.s3_objects[i]; if (size > offset) { auto buf = std::make_unique(client_ptr, bucket, path, buf_size); @@ -325,6 +341,8 @@ public: return folder_path + iter.name(); } + String name() const override { return iter.name(); } + private: Poco::DirectoryIterator iter; String folder_path; @@ -547,7 +565,7 @@ void DiskS3::removeRecursive(const String & path) Poco::File file(metadata_path + path); if (file.isFile()) { - remove(metadata_path + path); + remove(path); } else { diff --git a/dbms/src/Disks/IDisk.h b/dbms/src/Disks/IDisk.h index b737c7a53df..1f8abe2af79 100644 --- a/dbms/src/Disks/IDisk.h +++ b/dbms/src/Disks/IDisk.h @@ -162,6 +162,9 @@ public: /// Path to the file that the iterator currently points to. virtual String path() const = 0; + /// Name of the file that the iterator currently points to. + virtual String name() const = 0; + virtual ~IDiskDirectoryIterator() = default; }; diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 10e8a1d8e97..e8c9f2a113e 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -1,30 +1,23 @@ #include "IMergeTreeDataPart.h" #include +#include +#include +#include +#include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include #include #include -#include -#include +#include +#include +#include #include +#include namespace DB { - namespace ErrorCodes { extern const int DIRECTORY_ALREADY_EXISTS; @@ -42,37 +35,43 @@ namespace ErrorCodes } -static ReadBufferFromFile openForReading(const String & path) +static std::unique_ptr openForReading(const DiskPtr & disk, const String & path) { - return ReadBufferFromFile(path, std::min(static_cast(DBMS_DEFAULT_BUFFER_SIZE), Poco::File(path).getSize())); + return disk->readFile(path, std::min(size_t(DBMS_DEFAULT_BUFFER_SIZE), disk->getFileSize(path))); } -void IMergeTreeDataPart::MinMaxIndex::load(const MergeTreeData & data, const String & part_path) +void IMergeTreeDataPart::MinMaxIndex::load(const MergeTreeData & data, const DiskPtr & disk_, const String & part_path) { size_t minmax_idx_size = data.minmax_idx_column_types.size(); parallelogram.reserve(minmax_idx_size); for (size_t i = 0; i < minmax_idx_size; ++i) { String file_name = part_path + "minmax_" + escapeForFileName(data.minmax_idx_columns[i]) + ".idx"; - ReadBufferFromFile file = openForReading(file_name); + auto file = openForReading(disk_, file_name); const DataTypePtr & data_type = data.minmax_idx_column_types[i]; Field min_val; - data_type->deserializeBinary(min_val, file); + data_type->deserializeBinary(min_val, *file); Field max_val; - data_type->deserializeBinary(max_val, file); + data_type->deserializeBinary(max_val, *file); parallelogram.emplace_back(min_val, true, max_val, true); } initialized = true; } -void IMergeTreeDataPart::MinMaxIndex::store(const MergeTreeData & data, const String & part_path, Checksums & out_checksums) const +void IMergeTreeDataPart::MinMaxIndex::store( + const MergeTreeData & data, const DiskPtr & disk_, const String & part_path, Checksums & out_checksums) const { - store(data.minmax_idx_columns, data.minmax_idx_column_types, part_path, out_checksums); + store(data.minmax_idx_columns, data.minmax_idx_column_types, disk_, part_path, out_checksums); } -void IMergeTreeDataPart::MinMaxIndex::store(const Names & column_names, const DataTypes & data_types, const String & part_path, Checksums & out_checksums) const +void IMergeTreeDataPart::MinMaxIndex::store( + const Names & column_names, + const DataTypes & data_types, + const DiskPtr & disk_, + const String & part_path, + Checksums & out_checksums) const { if (!initialized) throw Exception("Attempt to store uninitialized MinMax index for part " + part_path + ". This is a bug.", @@ -83,8 +82,8 @@ void IMergeTreeDataPart::MinMaxIndex::store(const Names & column_names, const Da String file_name = "minmax_" + escapeForFileName(column_names[i]) + ".idx"; const DataTypePtr & data_type = data_types.at(i); - WriteBufferFromFile out(part_path + file_name); - HashingWriteBuffer out_hashing(out); + auto out = disk_->writeFile(part_path + file_name); + HashingWriteBuffer out_hashing(*out); data_type->serializeBinary(parallelogram[i].left, out_hashing); data_type->serializeBinary(parallelogram[i].right, out_hashing); out_hashing.next(); @@ -139,11 +138,7 @@ void IMergeTreeDataPart::MinMaxIndex::merge(const MinMaxIndex & other) IMergeTreeDataPart::IMergeTreeDataPart( - MergeTreeData & storage_, - const String & name_, - const DiskPtr & disk_, - const std::optional & relative_path_, - Type part_type_) + MergeTreeData & storage_, const String & name_, const DiskPtr & disk_, const std::optional & relative_path_, Type part_type_) : storage(storage_) , name(name_) , info(MergeTreePartInfo::fromPartName(name_, storage.format_version)) @@ -155,12 +150,12 @@ IMergeTreeDataPart::IMergeTreeDataPart( } IMergeTreeDataPart::IMergeTreeDataPart( - const MergeTreeData & storage_, - const String & name_, - const MergeTreePartInfo & info_, - const DiskPtr & disk_, - const std::optional & relative_path_, - Type part_type_) + const MergeTreeData & storage_, + const String & name_, + const MergeTreePartInfo & info_, + const DiskPtr & disk_, + const std::optional & relative_path_, + Type part_type_) : storage(storage_) , name(name_) , info(info_) @@ -385,7 +380,6 @@ String IMergeTreeDataPart::getColumnNameWithMinumumCompressedSize() const return *minimum_size_column; } - String IMergeTreeDataPart::getFullPath() const { assertOnDisk(); @@ -396,6 +390,16 @@ String IMergeTreeDataPart::getFullPath() const return storage.getFullPathOnDisk(disk) + relative_path + "/"; } +String IMergeTreeDataPart::getFullRelativePath() const +{ + assertOnDisk(); + + if (relative_path.empty()) + throw Exception("Part relative_path cannot be empty. It's bug.", ErrorCodes::LOGICAL_ERROR); + + return storage.relative_data_path + relative_path + "/"; +} + void IMergeTreeDataPart::loadColumnsChecksumsIndexes(bool require_columns_checksums, bool check_consistency) { assertOnDisk(); @@ -408,7 +412,7 @@ void IMergeTreeDataPart::loadColumnsChecksumsIndexes(bool require_columns_checks loadColumns(require_columns_checksums); loadChecksums(require_columns_checksums); loadIndexGranularity(); - loadIndex(); /// Must be called after loadIndexGranularity as it uses the value of `index_granularity` + loadIndex(); /// Must be called after loadIndexGranularity as it uses the value of `index_granularity` loadRowsCount(); /// Must be called after loadIndex() as it uses the value of `index_granularity`. loadPartitionAndMinMaxIndex(); loadTTLInfos(); @@ -441,12 +445,12 @@ void IMergeTreeDataPart::loadIndex() loaded_index[i]->reserve(index_granularity.getMarksCount()); } - String index_path = getFullPath() + "primary.idx"; - ReadBufferFromFile index_file = openForReading(index_path); + String index_path = getFullRelativePath() + "primary.idx"; + auto index_file = openForReading(disk, index_path); - for (size_t i = 0; i < index_granularity.getMarksCount(); ++i) //-V756 + for (size_t i = 0; i < index_granularity.getMarksCount(); ++i) //-V756 for (size_t j = 0; j < key_size; ++j) - storage.primary_key_data_types[j]->deserializeBinary(*loaded_index[j], index_file); + storage.primary_key_data_types[j]->deserializeBinary(*loaded_index[j], *index_file); for (size_t i = 0; i < key_size; ++i) { @@ -457,8 +461,8 @@ void IMergeTreeDataPart::loadIndex() ErrorCodes::CANNOT_READ_ALL_DATA); } - if (!index_file.eof()) - throw Exception("Index file " + index_path + " is unexpectedly long", ErrorCodes::EXPECTED_END_OF_FILE); + if (!index_file->eof()) + throw Exception("Index file " + fullPath(disk, index_path) + " is unexpectedly long", ErrorCodes::EXPECTED_END_OF_FILE); index.assign(std::make_move_iterator(loaded_index.begin()), std::make_move_iterator(loaded_index.end())); } @@ -478,10 +482,10 @@ void IMergeTreeDataPart::loadPartitionAndMinMaxIndex() } else { - String path = getFullPath(); - partition.load(storage, path); + String path = getFullRelativePath(); + partition.load(storage, disk, path); if (!isEmpty()) - minmax_idx.load(storage, path); + minmax_idx.load(storage, disk, path); } String calculated_partition_id = partition.getID(storage.partition_key_sample); @@ -494,14 +498,13 @@ void IMergeTreeDataPart::loadPartitionAndMinMaxIndex() void IMergeTreeDataPart::loadChecksums(bool require) { - String path = getFullPath() + "checksums.txt"; - Poco::File checksums_file(path); - if (checksums_file.exists()) + String path = getFullRelativePath() + "checksums.txt"; + if (disk->exists(path)) { - ReadBufferFromFile file = openForReading(path); - if (checksums.read(file)) + auto buf = openForReading(disk, path); + if (checksums.read(*buf)) { - assertEOF(file); + assertEOF(*buf); bytes_on_disk = checksums.getTotalSizeOnDisk(); } else @@ -518,19 +521,19 @@ void IMergeTreeDataPart::loadChecksums(bool require) void IMergeTreeDataPart::loadRowsCount() { - String path = getFullPath() + "count.txt"; + String path = getFullRelativePath() + "count.txt"; if (index_granularity.empty()) { rows_count = 0; } else if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING || part_type == Type::COMPACT) { - if (!Poco::File(path).exists()) + if (!disk->exists(path)) throw Exception("No count.txt in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); - ReadBufferFromFile file = openForReading(path); - readIntText(rows_count, file); - assertEOF(file); + auto buf = openForReading(disk, path); + readIntText(rows_count, *buf); + assertEOF(*buf); } else { @@ -572,20 +575,20 @@ void IMergeTreeDataPart::loadRowsCount() void IMergeTreeDataPart::loadTTLInfos() { - String path = getFullPath() + "ttl.txt"; - if (Poco::File(path).exists()) + String path = getFullRelativePath() + "ttl.txt"; + if (disk->exists(path)) { - ReadBufferFromFile in = openForReading(path); - assertString("ttl format version: ", in); + auto in = openForReading(disk, path); + assertString("ttl format version: ", *in); size_t format_version; - readText(format_version, in); - assertChar('\n', in); + readText(format_version, *in); + assertChar('\n', *in); if (format_version == 1) { try { - ttl_infos.read(in); + ttl_infos.read(*in); } catch (const JSONException &) { @@ -599,9 +602,8 @@ void IMergeTreeDataPart::loadTTLInfos() void IMergeTreeDataPart::loadColumns(bool require) { - String path = getFullPath() + "columns.txt"; - Poco::File poco_file_path{path}; - if (!poco_file_path.exists()) + String path = getFullRelativePath() + "columns.txt"; + if (!disk->exists(path)) { /// We can get list of columns only from columns.txt in compact parts. if (require || part_type == Type::COMPACT) @@ -609,23 +611,21 @@ void IMergeTreeDataPart::loadColumns(bool require) /// If there is no file with a list of columns, write it down. for (const NameAndTypePair & column : storage.getColumns().getAllPhysical()) - if (Poco::File(getFullPath() + getFileNameForColumn(column) + ".bin").exists()) + if (disk->exists(getFullRelativePath() + getFileNameForColumn(column) + ".bin")) columns.push_back(column); if (columns.empty()) throw Exception("No columns in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); { - WriteBufferFromFile out(path + ".tmp", 4096); - columns.writeText(out); + auto buf = disk->writeFile(path + ".tmp", 4096); + columns.writeText(*buf); } - Poco::File(path + ".tmp").renameTo(path); + disk->moveFile(path + ".tmp", path); } else { - is_frozen = !poco_file_path.canWrite(); - ReadBufferFromFile file = openForReading(path); - columns.readText(file); + columns.readText(*disk->readFile(path)); } size_t pos = 0; @@ -651,34 +651,32 @@ void IMergeTreeDataPart::renameTo(const String & new_relative_path, bool remove_ { assertOnDisk(); - String from = getFullPath(); - String to = storage.getFullPathOnDisk(disk) + new_relative_path + "/"; + String from = getFullRelativePath(); + String to = storage.relative_data_path + new_relative_path + "/"; - Poco::File from_file(from); - if (!from_file.exists()) - throw Exception("Part directory " + from + " doesn't exist. Most likely it is logical error.", ErrorCodes::FILE_DOESNT_EXIST); + if (!disk->exists(from)) + throw Exception("Part directory " + fullPath(disk, from) + " doesn't exist. Most likely it is logical error.", ErrorCodes::FILE_DOESNT_EXIST); - Poco::File to_file(to); - if (to_file.exists()) + if (disk->exists(to)) { if (remove_new_dir_if_exists) { + /// TODO: List files. Names files; - Poco::File(from).list(files); - LOG_WARNING(storage.log, "Part directory " << to << " already exists" + LOG_WARNING(storage.log, "Part directory " << fullPath(disk, to) << " already exists" << " and contains " << files.size() << " files. Removing it."); - to_file.remove(true); + disk->removeRecursive(to); } else { - throw Exception("Part directory " + to + " already exists", ErrorCodes::DIRECTORY_ALREADY_EXISTS); + throw Exception("Part directory " + fullPath(disk, to) + " already exists", ErrorCodes::DIRECTORY_ALREADY_EXISTS); } } - from_file.setLastModified(Poco::Timestamp::fromEpochTime(time(nullptr))); - from_file.renameTo(to); + //from_file.setLastModified(Poco::Timestamp::fromEpochTime(time(nullptr))); + disk->moveFile(from, to); relative_path = new_relative_path; } @@ -703,37 +701,33 @@ void IMergeTreeDataPart::remove() const * And a race condition can happen that will lead to "File not found" error here. */ - String full_path = storage.getFullPathOnDisk(disk); - String from = full_path + relative_path; - String to = full_path + "delete_tmp_" + name; + String from_ = storage.relative_data_path + relative_path; + String to_ = storage.relative_data_path + "delete_tmp_" + name; // TODO directory delete_tmp_ is never removed if server crashes before returning from this function - Poco::File from_dir{from}; - Poco::File to_dir{to}; - - if (to_dir.exists()) + if (disk->exists(to_)) { - LOG_WARNING(storage.log, "Directory " << to << " (to which part must be renamed before removing) already exists." + LOG_WARNING(storage.log, "Directory " << fullPath(disk, to_) << " (to which part must be renamed before removing) already exists." " Most likely this is due to unclean restart. Removing it."); try { - to_dir.remove(true); + disk->removeRecursive(to_); } catch (...) { - LOG_ERROR(storage.log, "Cannot remove directory " << to << ". Check owner and access rights."); + LOG_ERROR(storage.log, "Cannot remove directory " << fullPath(disk, to_) << ". Check owner and access rights."); throw; } } try { - from_dir.renameTo(to); + disk->moveFile(from_, to_); } catch (const Poco::FileNotFoundException &) { - LOG_ERROR(storage.log, "Directory " << from << " (part to remove) doesn't exist or one of nested files has gone." + LOG_ERROR(storage.log, "Directory " << fullPath(disk, to_) << " (part to remove) doesn't exist or one of nested files has gone." " Most likely this is due to manual removing. This should be discouraged. Ignoring."); return; @@ -744,28 +738,29 @@ void IMergeTreeDataPart::remove() const /// Remove each expected file in directory, then remove directory itself. #if !__clang__ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-variable" +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wunused-variable" #endif std::shared_lock lock(columns_lock); + /// TODO: IDisk doesn't support `unlink()` and `rmdir()` functionality. + auto to = fullPath(disk, to_); + for (const auto & [file, _] : checksums.files) { String path_to_remove = to + "/" + file; if (0 != unlink(path_to_remove.c_str())) - throwFromErrnoWithPath("Cannot unlink file " + path_to_remove, path_to_remove, - ErrorCodes::CANNOT_UNLINK); + throwFromErrnoWithPath("Cannot unlink file " + path_to_remove, path_to_remove, ErrorCodes::CANNOT_UNLINK); } #if !__clang__ -#pragma GCC diagnostic pop +# pragma GCC diagnostic pop #endif for (const auto & file : {"checksums.txt", "columns.txt"}) { String path_to_remove = to + "/" + file; if (0 != unlink(path_to_remove.c_str())) - throwFromErrnoWithPath("Cannot unlink file " + path_to_remove, path_to_remove, - ErrorCodes::CANNOT_UNLINK); + throwFromErrnoWithPath("Cannot unlink file " + path_to_remove, path_to_remove, ErrorCodes::CANNOT_UNLINK); } if (0 != rmdir(to.c_str())) @@ -775,10 +770,10 @@ void IMergeTreeDataPart::remove() const { /// Recursive directory removal does many excessive "stat" syscalls under the hood. - LOG_ERROR(storage.log, "Cannot quickly remove directory " << to << " by removing files; fallback to recursive removal. Reason: " + LOG_ERROR(storage.log, "Cannot quickly remove directory " << fullPath(disk, to_) << " by removing files; fallback to recursive removal. Reason: " << getCurrentExceptionMessage(false)); - to_dir.remove(true); + disk->removeRecursive(to_ + "/"); } } @@ -796,8 +791,7 @@ String IMergeTreeDataPart::getRelativePathForDetachedPart(const String & prefix) */ for (int try_no = 0; try_no < 10; try_no++) { - res = "detached/" + (prefix.empty() ? "" : prefix + "_") - + name + (try_no ? "_try" + DB::toString(try_no) : ""); + res = "detached/" + (prefix.empty() ? "" : prefix + "_") + name + (try_no ? "_try" + DB::toString(try_no) : ""); if (!Poco::File(storage.getFullPathOnDisk(disk) + res).exists()) return res; @@ -845,7 +839,7 @@ void IMergeTreeDataPart::makeCloneOnDiskDetached(const ReservationPtr & reservat void IMergeTreeDataPart::checkConsistencyBase() const { - String path = getFullPath(); + String path = getFullRelativePath(); if (!checksums.empty()) { @@ -870,31 +864,30 @@ void IMergeTreeDataPart::checkConsistencyBase() const } } - checksums.checkSizes(path); + checksums.checkSizes(disk, path); } else { - auto check_file_not_empty = [&path](const String & file_path) - { - Poco::File file(file_path); - if (!file.exists() || file.getSize() == 0) - throw Exception("Part " + path + " is broken: " + file_path + " is empty", ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); - return file.getSize(); + auto check_file_not_empty = [&path](const DiskPtr & disk_, const String & file_path) { + UInt64 file_size; + if (!disk_->exists(file_path) || (file_size = disk_->getFileSize(file_path)) == 0) + throw Exception("Part " + fullPath(disk_, path) + " is broken: " + fullPath(disk_, file_path) + " is empty", ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); + return file_size; }; /// Check that the primary key index is not empty. if (!storage.primary_key_columns.empty()) - check_file_not_empty(path + "primary.idx"); + check_file_not_empty(disk, path + "primary.idx"); if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) { - check_file_not_empty(path + "count.txt"); + check_file_not_empty(disk, path + "count.txt"); if (storage.partition_key_expr) - check_file_not_empty(path + "partition.dat"); + check_file_not_empty(disk, path + "partition.dat"); for (const String & col_name : storage.minmax_idx_columns) - check_file_not_empty(path + "minmax_" + escapeForFileName(col_name) + ".idx"); + check_file_not_empty(disk, path + "minmax_" + escapeForFileName(col_name) + ".idx"); } } } diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h index 61b5ba43165..48ae37edb87 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -268,9 +268,9 @@ public: { } - void load(const MergeTreeData & storage, const String & part_path); - void store(const MergeTreeData & storage, const String & part_path, Checksums & checksums) const; - void store(const Names & column_names, const DataTypes & data_types, const String & part_path, Checksums & checksums) const; + void load(const MergeTreeData & storage, const DiskPtr & disk_, const String & part_path); + void store(const MergeTreeData & storage, const DiskPtr & disk_, const String & part_path, Checksums & checksums) const; + void store(const Names & column_names, const DataTypes & data_types, const DiskPtr & disk_, const String & part_path, Checksums & checksums) const; void update(const Block & block, const Names & column_names); void merge(const MinMaxIndex & other); @@ -294,6 +294,7 @@ public: UInt64 getMarksCount() const; size_t getFileSizeOrZero(const String & file_name) const; + String getFullRelativePath() const; String getFullPath() const; void renameTo(const String & new_relative_path, bool remove_new_dir_if_exists = false) const; void renameToDetached(const String & prefix) const; diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp index fad9d9ff488..ffbded9770f 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp @@ -1,6 +1,6 @@ #include -#include -#include + +#include namespace DB { @@ -24,11 +24,12 @@ void IMergeTreeDataPartWriter::Stream::finalize() void IMergeTreeDataPartWriter::Stream::sync() { plain_file->sync(); - marks_file.sync(); + marks_file->sync(); } IMergeTreeDataPartWriter::Stream::Stream( const String & escaped_column_name_, + DiskPtr disk_, const String & data_path_, const std::string & data_file_extension_, const std::string & marks_path_, @@ -40,9 +41,9 @@ IMergeTreeDataPartWriter::Stream::Stream( escaped_column_name(escaped_column_name_), data_file_extension{data_file_extension_}, marks_file_extension{marks_file_extension_}, - plain_file(createWriteBufferFromFileBase(data_path_ + data_file_extension, estimated_size_, aio_threshold_, max_compress_block_size_)), + plain_file(disk_->writeFile(data_path_ + data_file_extension, max_compress_block_size_, WriteMode::Rewrite, estimated_size_, aio_threshold_)), plain_hashing(*plain_file), compressed_buf(plain_hashing, compression_codec_), compressed(compressed_buf), - marks_file(marks_path_ + marks_file_extension, 4096, O_TRUNC | O_CREAT | O_WRONLY), marks(marks_file) + marks_file(disk_->writeFile(marks_path_ + marks_file_extension, 4096, WriteMode::Rewrite)), marks(*marks_file) { } @@ -62,6 +63,7 @@ void IMergeTreeDataPartWriter::Stream::addToChecksums(MergeTreeData::DataPart::C IMergeTreeDataPartWriter::IMergeTreeDataPartWriter( + DiskPtr disk_, const String & part_path_, const MergeTreeData & storage_, const NamesAndTypesList & columns_list_, @@ -71,7 +73,8 @@ IMergeTreeDataPartWriter::IMergeTreeDataPartWriter( const MergeTreeWriterSettings & settings_, const MergeTreeIndexGranularity & index_granularity_, bool need_finish_last_granule_) - : part_path(part_path_) + : disk(std::move(disk_)) + , part_path(part_path_) , storage(storage_) , columns_list(columns_list_) , marks_file_extension(marks_file_extension_) @@ -86,10 +89,8 @@ IMergeTreeDataPartWriter::IMergeTreeDataPartWriter( if (settings.blocks_are_granules_size && !index_granularity.empty()) throw Exception("Can't take information about index granularity from blocks, when non empty index_granularity array specified", ErrorCodes::LOGICAL_ERROR); - Poco::File part_dir(part_path); - if (!part_dir.exists()) - part_dir.createDirectories(); - + if (!disk->exists(part_path)) + disk->createDirectories(part_path); } IMergeTreeDataPartWriter::~IMergeTreeDataPartWriter() = default; @@ -172,8 +173,7 @@ void IMergeTreeDataPartWriter::initPrimaryIndex() { if (storage.hasPrimaryKey()) { - index_file_stream = std::make_unique( - part_path + "primary.idx", DBMS_DEFAULT_BUFFER_SIZE, O_TRUNC | O_CREAT | O_WRONLY); + index_file_stream = disk->writeFile(part_path + "primary.idx", DBMS_DEFAULT_BUFFER_SIZE, WriteMode::Rewrite); index_stream = std::make_unique(*index_file_stream); } @@ -188,6 +188,7 @@ void IMergeTreeDataPartWriter::initSkipIndices() skip_indices_streams.emplace_back( std::make_unique( stream_name, + disk, part_path + stream_name, INDEX_FILE_EXTENSION, part_path + stream_name, marks_file_extension, default_codec, settings.max_compress_block_size, diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h index d2a6c5d4994..4eb842f9279 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h @@ -7,6 +7,7 @@ #include #include #include +#include namespace DB @@ -26,6 +27,7 @@ public: { Stream( const String & escaped_column_name_, + DiskPtr disk_, const String & data_path_, const std::string & data_file_extension_, const std::string & marks_path_, @@ -46,7 +48,7 @@ public: HashingWriteBuffer compressed; /// marks -> marks_file - WriteBufferFromFile marks_file; + std::unique_ptr marks_file; HashingWriteBuffer marks; void finalize(); @@ -59,6 +61,7 @@ public: using StreamPtr = std::unique_ptr; IMergeTreeDataPartWriter( + DiskPtr disk, const String & part_path, const MergeTreeData & storage, const NamesAndTypesList & columns_list, @@ -113,6 +116,7 @@ protected: using SerializationState = IDataType::SerializeBinaryBulkStatePtr; using SerializationStates = std::unordered_map; + DiskPtr disk; String part_path; const MergeTreeData & storage; NamesAndTypesList columns_list; @@ -146,7 +150,7 @@ protected: MergeTreeIndexAggregators skip_indices_aggregators; std::vector skip_index_filling; - std::unique_ptr index_file_stream; + std::unique_ptr index_file_stream; std::unique_ptr index_stream; MutableColumns index_columns; DataTypes index_types; diff --git a/dbms/src/Storages/MergeTree/IMergeTreeReader.cpp b/dbms/src/Storages/MergeTree/IMergeTreeReader.cpp index 161b468913f..0a0bf25b399 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeReader.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeReader.cpp @@ -26,7 +26,7 @@ IMergeTreeReader::IMergeTreeReader(const MergeTreeData::DataPartPtr & data_part_ const NamesAndTypesList & columns_, UncompressedCache * uncompressed_cache_, MarkCache * mark_cache_, const MarkRanges & all_mark_ranges_, const MergeTreeReaderSettings & settings_, const ValueSizeMap & avg_value_size_hints_) - : data_part(data_part_), avg_value_size_hints(avg_value_size_hints_), path(data_part_->getFullPath()) + : data_part(data_part_), avg_value_size_hints(avg_value_size_hints_) , columns(columns_), uncompressed_cache(uncompressed_cache_), mark_cache(mark_cache_) , settings(settings_), storage(data_part_->storage) , all_mark_ranges(all_mark_ranges_) @@ -140,7 +140,7 @@ void IMergeTreeReader::fillMissingColumns(Columns & res_columns, bool & should_e catch (Exception & e) { /// Better diagnostics. - e.addMessage("(while reading from part " + path + ")"); + e.addMessage("(while reading from part " + data_part->getFullPath() + ")"); throw; } } @@ -177,7 +177,7 @@ void IMergeTreeReader::evaluateMissingDefaults(Block additional_columns, Columns catch (Exception & e) { /// Better diagnostics. - e.addMessage("(while reading from part " + path + ")"); + e.addMessage("(while reading from part " + data_part->getFullPath() + ")"); throw; } } diff --git a/dbms/src/Storages/MergeTree/IMergeTreeReader.h b/dbms/src/Storages/MergeTree/IMergeTreeReader.h index ad3c47c1c0b..ac4dde5a246 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeReader.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeReader.h @@ -59,8 +59,6 @@ protected: ValueSizeMap avg_value_size_hints; /// Stores states for IDataType::deserializeBinaryBulk DeserializeBinaryBulkStateMap deserialize_binary_bulk_state_map; - /// Path to the directory containing the part - String path; /// Columns that are read. NamesAndTypesList columns; diff --git a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp index 72b82e29e9b..d7da78a327b 100644 --- a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp @@ -9,7 +9,8 @@ namespace DB IMergedBlockOutputStream::IMergedBlockOutputStream( const MergeTreeDataPartPtr & data_part) : storage(data_part->storage) - , part_path(data_part->getFullPath()) + , disk(data_part->disk) + , part_path(data_part->getFullRelativePath()) { } diff --git a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h index 983750862e8..ae7aeb6e4e1 100644 --- a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h +++ b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h @@ -30,6 +30,7 @@ protected: protected: const MergeTreeData & storage; + DiskPtr disk; String part_path; static Block getBlockAndPermute(const Block & block, const Names & names, const IColumn::Permutation * permutation); diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 576ef53316d..474f8512980 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -200,48 +200,47 @@ MergeTreeData::MergeTreeData( setTTLExpressions(metadata.columns.getColumnTTLs(), metadata.ttl_for_table_ast); - // format_file always contained on any data path - String version_file_path; - + /// format_file always contained on any data path + PathWithDisk version_file; /// Creating directories, if not exist. - auto paths = getDataPaths(); - for (const String & path : paths) + for (const auto & [path, disk] : getRelativeDataPathsWithDisks()) { - Poco::File(path).createDirectories(); - Poco::File(path + "detached").createDirectory(); + disk->createDirectories(path); + disk->createDirectories(path + "detached"); auto current_version_file_path = path + "format_version.txt"; - if (Poco::File{current_version_file_path}.exists()) + if (disk->exists(current_version_file_path)) { - if (!version_file_path.empty()) + if (!version_file.first.empty()) { - LOG_ERROR(log, "Duplication of version file " << version_file_path << " and " << current_version_file_path); + LOG_ERROR(log, "Duplication of version file " << + fullPath(version_file.second, version_file.first) << " and " << current_version_file_path); throw Exception("Multiple format_version.txt file", ErrorCodes::CORRUPTED_DATA); } - version_file_path = current_version_file_path; + version_file = {current_version_file_path, disk}; } } /// If not choose any - if (version_file_path.empty()) - version_file_path = getFullPathOnDisk(storage_policy->getAnyDisk()) + "format_version.txt"; + if (version_file.first.empty()) + version_file = {relative_data_path + "format_version.txt", storage_policy->getAnyDisk()}; - bool version_file_exists = Poco::File(version_file_path).exists(); + bool version_file_exists = version_file.second->exists(version_file.first); // When data path or file not exists, ignore the format_version check if (!attach || !version_file_exists) { format_version = min_format_version; - WriteBufferFromFile buf(version_file_path); - writeIntText(format_version.toUnderType(), buf); + auto buf = version_file.second->writeFile(version_file.first); + writeIntText(format_version.toUnderType(), *buf); } else { - ReadBufferFromFile buf(version_file_path); + auto buf = version_file.second->readFile(version_file.first); UInt32 read_format_version; - readIntText(read_format_version, buf); + readIntText(read_format_version, *buf); format_version = read_format_version; - if (!buf.eof()) - throw Exception("Bad version file: " + version_file_path, ErrorCodes::CORRUPTED_DATA); + if (!buf->eof()) + throw Exception("Bad version file: " + fullPath(version_file.second, version_file.first), ErrorCodes::CORRUPTED_DATA); } if (format_version < min_format_version) @@ -916,15 +915,15 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) for (const auto & disk_ptr : disks) defined_disk_names.insert(disk_ptr->getName()); - for (auto & [disk_name, disk_ptr] : global_context.getDiskSelector().getDisksMap()) + for (auto & [disk_name, disk] : global_context.getDiskSelector().getDisksMap()) { - if (defined_disk_names.count(disk_name) == 0 && Poco::File(getFullPathOnDisk(disk_ptr)).exists()) + if (defined_disk_names.count(disk_name) == 0 && disk->exists(relative_data_path)) { - for (Poco::DirectoryIterator it(getFullPathOnDisk(disk_ptr)); it != end; ++it) + for (const auto it = disk->iterateDirectory(relative_data_path); it->isValid(); it->next()) { MergeTreePartInfo part_info; - if (MergeTreePartInfo::tryParsePartName(it.name(), &part_info, format_version)) - throw Exception("Part " + backQuote(it.name()) + " was found on disk " + backQuote(disk_name) + " which is not defined in the storage policy", ErrorCodes::UNKNOWN_DISK); + if (MergeTreePartInfo::tryParsePartName(it->name(), &part_info, format_version)) + throw Exception("Part " + backQuote(it->name()) + " was found on disk " + backQuote(disk_name) + " which is not defined in the storage policy", ErrorCodes::UNKNOWN_DISK); } } } @@ -935,13 +934,13 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) for (auto disk_it = disks.rbegin(); disk_it != disks.rend(); ++disk_it) { auto disk_ptr = *disk_it; - for (Poco::DirectoryIterator it(getFullPathOnDisk(disk_ptr)); it != end; ++it) + for (auto it = disk_ptr->iterateDirectory(relative_data_path); it->isValid(); it->next()) { /// Skip temporary directories. - if (startsWith(it.name(), "tmp")) + if (startsWith(it->name(), "tmp")) continue; - part_names_with_disks.emplace_back(it.name(), disk_ptr); + part_names_with_disks.emplace_back(it->name(), disk_ptr); } } @@ -982,9 +981,9 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) auto part = createPart(part_name, part_info, part_disk_ptr, part_name); bool broken = false; - Poco::Path part_path(getFullPathOnDisk(part_disk_ptr), part_name); - Poco::Path marker_path(part_path, DELETE_ON_DESTROY_MARKER_PATH); - if (Poco::File(marker_path).exists()) + String part_path = relative_data_path + "/" + part_name; + String marker_path = part_path + "/" + DELETE_ON_DESTROY_MARKER_PATH; + if (part_disk_ptr->exists(marker_path)) { LOG_WARNING(log, "Detaching stale part " << getFullPathOnDisk(part_disk_ptr) << part_name << ", which should have been deleted after a move. That can only happen after unclean restart of ClickHouse after move of a part having an operation blocking that stale copy of part."); std::lock_guard loading_lock(mutex); @@ -1075,6 +1074,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) else has_adaptive_parts.store(true, std::memory_order_relaxed); + /// TODO: Add modification time functionality to IDisk. part->modification_time = Poco::File(getFullPathOnDisk(part_disk_ptr) + part_name).getLastModified().epochTime(); /// Assume that all parts are Committed, covered parts will be detected and marked as Outdated later part->state = DataPartState::Committed; @@ -1185,24 +1185,22 @@ void MergeTreeData::clearOldTemporaryDirectories(ssize_t custom_directories_life ? current_time - custom_directories_lifetime_seconds : current_time - settings->temporary_directories_lifetime.totalSeconds(); - const auto full_paths = getDataPaths(); - /// Delete temporary directories older than a day. - Poco::DirectoryIterator end; - for (auto && full_data_path : full_paths) + for (const auto & [path, disk] : getRelativeDataPathsWithDisks()) { - for (Poco::DirectoryIterator it{full_data_path}; it != end; ++it) + for (auto it = disk->iterateDirectory(path); it->isValid(); it->next()) { - if (startsWith(it.name(), "tmp_")) + if (startsWith(it->name(), "tmp_")) { - Poco::File tmp_dir(full_data_path + it.name()); + /// TODO: Add modification time functionality to IDisk. + Poco::File tmp_dir(fullPath(disk, it->path())); try { if (tmp_dir.isDirectory() && isOldPartDirectory(tmp_dir, deadline)) { - LOG_WARNING(log, "Removing temporary directory " << full_data_path << it.name()); - Poco::File(full_data_path + it.name()).remove(true); + LOG_WARNING(log, "Removing temporary directory " << fullPath(disk, it->path())); + tmp_dir.remove(true); } } catch (const Poco::FileNotFoundException &) @@ -1400,10 +1398,8 @@ void MergeTreeData::dropAllData() /// Removing of each data part before recursive removal of directory is to speed-up removal, because there will be less number of syscalls. clearPartsFromFilesystem(all_parts); - auto full_paths = getDataPaths(); - - for (auto && full_data_path : full_paths) - Poco::File(full_data_path).remove(true); + for (const auto & [path, disk] : getRelativeDataPathsWithDisks()) + disk->removeRecursive(path); LOG_TRACE(log, "dropAllData: done."); } @@ -1713,8 +1709,8 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::createPart( const DiskPtr & disk, const String & relative_path) const { MergeTreeDataPartType type; - auto full_path = getFullPathOnDisk(disk) + relative_path + "/"; - auto mrk_ext = MergeTreeIndexGranularityInfo::getMrkExtensionFromFS(full_path); + auto full_path = relative_data_path + relative_path + "/"; + auto mrk_ext = MergeTreeIndexGranularityInfo::getMrkExtensionFromFS(disk, full_path); if (mrk_ext) type = getPartTypeFromMarkExtension(*mrk_ext); @@ -3674,6 +3670,15 @@ MergeTreeData::PathsWithDisks MergeTreeData::getDataPathsWithDisks() const return res; } +MergeTreeData::PathsWithDisks MergeTreeData::getRelativeDataPathsWithDisks() const +{ + PathsWithDisks res; + auto disks = storage_policy->getDisks(); + for (const auto & disk : disks) + res.emplace_back(relative_data_path, disk); + return res; +} + void MergeTreeData::freezePartitionsByMatcher(MatcherFn matcher, const String & with_name, const Context & context) { String clickhouse_path = Poco::Path(context.getPath()).makeAbsolute().toString(); diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index aad681aae48..f9581926f3a 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -683,6 +683,7 @@ public: using PathWithDisk = std::pair; using PathsWithDisks = std::vector; PathsWithDisks getDataPathsWithDisks() const; + PathsWithDisks getRelativeDataPathsWithDisks() const; /// Reserves space at least 1MB. ReservationPtr reserveSpace(UInt64 expected_size) const; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp index a39be4f4d47..a136649e698 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp @@ -44,14 +44,13 @@ void MergeTreeDataPartChecksum::checkEqual(const MergeTreeDataPartChecksum & rhs throw Exception("Checksum mismatch for file " + name + " in data part", ErrorCodes::CHECKSUM_DOESNT_MATCH); } -void MergeTreeDataPartChecksum::checkSize(const String & path) const +void MergeTreeDataPartChecksum::checkSize(const DiskPtr & disk, const String & path) const { - Poco::File file(path); - if (!file.exists()) - throw Exception(path + " doesn't exist", ErrorCodes::FILE_DOESNT_EXIST); - UInt64 size = file.getSize(); + if (!disk->exists(path)) + throw Exception(fullPath(disk, path) + " doesn't exist", ErrorCodes::FILE_DOESNT_EXIST); + UInt64 size = disk->getFileSize(path); if (size != file_size) - throw Exception(path + " has unexpected size: " + toString(size) + " instead of " + toString(file_size), + throw Exception(fullPath(disk, path) + " has unexpected size: " + toString(size) + " instead of " + toString(file_size), ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); } @@ -78,12 +77,12 @@ void MergeTreeDataPartChecksums::checkEqual(const MergeTreeDataPartChecksums & r } } -void MergeTreeDataPartChecksums::checkSizes(const String & path) const +void MergeTreeDataPartChecksums::checkSizes(const DiskPtr & disk, const String & path) const { for (const auto & it : files) { const String & name = it.first; - it.second.checkSize(path + name); + it.second.checkSize(disk, path + name); } } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.h index 051e54d5139..f7b77f40dc0 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.h @@ -1,10 +1,11 @@ #pragma once -#include -#include -#include -#include #include #include +#include +#include +#include +#include +#include class SipHash; @@ -32,7 +33,7 @@ struct MergeTreeDataPartChecksum uncompressed_size(uncompressed_size_), uncompressed_hash(uncompressed_hash_) {} void checkEqual(const MergeTreeDataPartChecksum & rhs, bool have_uncompressed, const String & name) const; - void checkSize(const String & path) const; + void checkSize(const DiskPtr & disk, const String & path) const; }; @@ -64,7 +65,7 @@ struct MergeTreeDataPartChecksums static bool isBadChecksumsErrorCode(int code); /// Checks that the directory contains all the needed files of the correct size. Does not check the checksum. - void checkSizes(const String & path) const; + void checkSizes(const DiskPtr & disk, const String & path) const; /// Returns false if the checksum is too old. bool read(ReadBuffer & in); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index 9b297e57110..09947fce7d3 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -68,7 +68,7 @@ IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartCompact::getWriter( { return *getColumnPosition(lhs.name) < *getColumnPosition(rhs.name); }); return std::make_unique( - getFullPath(), storage, ordered_columns_list, indices_to_recalc, + disk, getFullRelativePath(), storage, ordered_columns_list, indices_to_recalc, index_granularity_info.marks_file_extension, default_codec, writer_settings, computed_index_granularity); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index 6d9e5579de1..44d8715f3d1 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -59,7 +59,7 @@ IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartWide::getWriter( const MergeTreeIndexGranularity & computed_index_granularity) const { return std::make_unique( - getFullPath(), storage, columns_list, indices_to_recalc, + disk, getFullRelativePath(), storage, columns_list, indices_to_recalc, index_granularity_info.marks_file_extension, default_codec, writer_settings, computed_index_granularity); } @@ -115,8 +115,8 @@ ColumnSize MergeTreeDataPartWide::getColumnSize(const String & column_name, cons void MergeTreeDataPartWide::loadIndexGranularity() { - String full_path = getFullPath(); - index_granularity_info.changeGranularityIfRequired(full_path); + String full_path = getFullRelativePath(); + index_granularity_info.changeGranularityIfRequired(disk, full_path); if (columns.empty()) @@ -124,10 +124,10 @@ void MergeTreeDataPartWide::loadIndexGranularity() /// We can use any column, it doesn't matter std::string marks_file_path = index_granularity_info.getMarksFilePath(full_path + getFileNameForColumn(columns.front())); - if (!Poco::File(marks_file_path).exists()) - throw Exception("Marks file '" + marks_file_path + "' doesn't exist", ErrorCodes::NO_FILE_IN_DATA_PART); + if (!disk->exists(marks_file_path)) + throw Exception("Marks file '" + fullPath(disk, marks_file_path) + "' doesn't exist", ErrorCodes::NO_FILE_IN_DATA_PART); - size_t marks_file_size = Poco::File(marks_file_path).getSize(); + size_t marks_file_size = disk->getFileSize(marks_file_path); if (!index_granularity_info.is_adaptive) { @@ -136,17 +136,17 @@ void MergeTreeDataPartWide::loadIndexGranularity() } else { - ReadBufferFromFile buffer(marks_file_path, marks_file_size, -1); - while (!buffer.eof()) + auto buffer = disk->readFile(marks_file_path, marks_file_size); + while (!buffer->eof()) { - buffer.seek(sizeof(size_t) * 2, SEEK_CUR); /// skip offset_in_compressed file and offset_in_decompressed_block + buffer->seek(sizeof(size_t) * 2, SEEK_CUR); /// skip offset_in_compressed file and offset_in_decompressed_block size_t granularity; - readIntBinary(granularity, buffer); + readIntBinary(granularity, *buffer); index_granularity.appendMark(granularity); } if (index_granularity.getMarksCount() * index_granularity_info.getMarkSizeInBytes() != marks_file_size) - throw Exception("Cannot read all marks from file " + marks_file_path, ErrorCodes::CANNOT_READ_ALL_DATA); + throw Exception("Cannot read all marks from file " + fullPath(disk, marks_file_path), ErrorCodes::CANNOT_READ_ALL_DATA); } index_granularity.setInitialized(); @@ -166,9 +166,9 @@ void MergeTreeDataPartWide::accumulateColumnSizes(ColumnToSize & column_to_size) IDataType::SubstreamPath path; name_type.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path) { - Poco::File bin_file(getFullPath() + IDataType::getFileNameForStream(name_type.name, substream_path) + ".bin"); - if (bin_file.exists()) - column_to_size[name_type.name] += bin_file.getSize(); + auto bin_file_path = getFullRelativePath() + IDataType::getFileNameForStream(name_type.name, substream_path) + ".bin"; + if (disk->exists(bin_file_path)) + column_to_size[name_type.name] += disk->getFileSize(bin_file_path); }, path); } } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index 7d0c16f2729..bd507866e9b 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -6,6 +6,7 @@ namespace DB MergeTreeDataPartWriterCompact::MergeTreeDataPartWriterCompact( + DiskPtr disk_, const String & part_path_, const MergeTreeData & storage_, const NamesAndTypesList & columns_list_, @@ -14,7 +15,7 @@ MergeTreeDataPartWriterCompact::MergeTreeDataPartWriterCompact( const CompressionCodecPtr & default_codec_, const MergeTreeWriterSettings & settings_, const MergeTreeIndexGranularity & index_granularity_) -: IMergeTreeDataPartWriter(part_path_, +: IMergeTreeDataPartWriter(disk_, part_path_, storage_, columns_list_, indices_to_recalc_, marks_file_extension_, default_codec_, settings_, index_granularity_, true) @@ -26,6 +27,7 @@ MergeTreeDataPartWriterCompact::MergeTreeDataPartWriterCompact( stream = std::make_unique( data_file_name, + disk_, part_path + data_file_name, DataPart::DATA_FILE_EXTENSION, part_path + data_file_name, marks_file_extension, default_codec, diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h index 2d9c67bd5df..3b405bbf1c5 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h @@ -8,6 +8,7 @@ class MergeTreeDataPartWriterCompact : public IMergeTreeDataPartWriter { public: MergeTreeDataPartWriterCompact( + DiskPtr disk, const String & part_path, const MergeTreeData & storage, const NamesAndTypesList & columns_list, diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index 0b170312d37..f4e90f658df 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -13,6 +13,7 @@ namespace } MergeTreeDataPartWriterWide::MergeTreeDataPartWriterWide( + DiskPtr disk_, const String & part_path_, const MergeTreeData & storage_, const NamesAndTypesList & columns_list_, @@ -21,7 +22,7 @@ MergeTreeDataPartWriterWide::MergeTreeDataPartWriterWide( const CompressionCodecPtr & default_codec_, const MergeTreeWriterSettings & settings_, const MergeTreeIndexGranularity & index_granularity_) - : IMergeTreeDataPartWriter(part_path_, + : IMergeTreeDataPartWriter(disk_, part_path_, storage_, columns_list_, indices_to_recalc_, marks_file_extension_, default_codec_, settings_, index_granularity_, false) { @@ -48,6 +49,7 @@ void MergeTreeDataPartWriterWide::addStreams( column_streams[stream_name] = std::make_unique( stream_name, + disk, part_path + stream_name, DATA_FILE_EXTENSION, part_path + stream_name, marks_file_extension, effective_codec, diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h index 1240626745a..d6b01f9c45c 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h @@ -11,6 +11,7 @@ public: using ColumnToSize = std::map; MergeTreeDataPartWriterWide( + DiskPtr disk, const String & part_path, const MergeTreeData & storage, const NamesAndTypesList & columns_list, diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index f3fd3be8e7f..891256aa200 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -1263,7 +1263,7 @@ MarkRanges MergeTreeDataSelectExecutor::filterMarksUsingIndex( const MarkRanges & ranges, const Settings & settings) const { - if (!Poco::File(part->getFullPath() + index->getFileName() + ".idx").exists()) + if (!part->disk->exists(part->getFullRelativePath() + index->getFileName() + ".idx")) { LOG_DEBUG(log, "File for index " << backQuote(index->name) << " does not exist. Skipping it."); return ranges; diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp index 165b80d23ab..6cd58f0eb93 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp @@ -1,9 +1,6 @@ #include #include #include -#include -#include -#include namespace DB { @@ -14,14 +11,14 @@ namespace ErrorCodes extern const int UNKNOWN_PART_TYPE; } -std::optional MergeTreeIndexGranularityInfo::getMrkExtensionFromFS(const std::string & path_to_part) +std::optional MergeTreeIndexGranularityInfo::getMrkExtensionFromFS(const DiskPtr & disk, const String & path_to_part) { - if (Poco::File(path_to_part).exists()) + if (disk->exists(path_to_part)) { - Poco::DirectoryIterator end; - for (Poco::DirectoryIterator part_it(path_to_part); part_it != end; ++part_it) + for (DiskDirectoryIteratorPtr it = disk->iterateDirectory(path_to_part); it->isValid(); it->next()) { - const auto & ext = "." + part_it.path().getExtension(); + Poco::Path path(it->path()); + const auto & ext = "." + path.getExtension(); if (ext == getNonAdaptiveMrkExtension() || ext == getAdaptiveMrkExtension(MergeTreeDataPartType::WIDE) || ext == getAdaptiveMrkExtension(MergeTreeDataPartType::COMPACT)) @@ -48,9 +45,9 @@ MergeTreeIndexGranularityInfo::MergeTreeIndexGranularityInfo(const MergeTreeData setAdaptive(storage_settings->index_granularity_bytes); } -void MergeTreeIndexGranularityInfo::changeGranularityIfRequired(const std::string & path_to_part) +void MergeTreeIndexGranularityInfo::changeGranularityIfRequired(const DiskPtr & disk, const String & path_to_part) { - auto mrk_ext = getMrkExtensionFromFS(path_to_part); + auto mrk_ext = getMrkExtensionFromFS(disk, path_to_part); if (mrk_ext && *mrk_ext == getNonAdaptiveMrkExtension()) setNonAdaptive(); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h index 7bcb208980e..465c80030de 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB { @@ -27,7 +28,7 @@ public: MergeTreeIndexGranularityInfo(const MergeTreeData & storage, MergeTreeDataPartType type_); - void changeGranularityIfRequired(const std::string & path_to_part); + void changeGranularityIfRequired(const DiskPtr & disk, const String & path_to_part); String getMarksFilePath(const String & path_prefix) const { @@ -36,7 +37,7 @@ public: size_t getMarkSizeInBytes(size_t columns_num = 1) const; - static std::optional getMrkExtensionFromFS(const std::string & path_to_table); + static std::optional getMrkExtensionFromFS(const DiskPtr & disk, const String & path_to_table); private: MergeTreeDataPartType type; diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeIndexReader.cpp index bf53b669571..55eeb41ceb5 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexReader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexReader.cpp @@ -7,7 +7,8 @@ namespace DB MergeTreeIndexReader::MergeTreeIndexReader( MergeTreeIndexPtr index_, MergeTreeData::DataPartPtr part_, size_t marks_count_, const MarkRanges & all_mark_ranges_) : index(index_), stream( - part_->getFullPath() + index->getFileName(), ".idx", marks_count_, + part_->disk, + part_->getFullRelativePath() + index->getFileName(), ".idx", marks_count_, all_mark_ranges_, MergeTreeReaderSettings{}, nullptr, nullptr, part_->getFileSizeOrZero(index->getFileName() + ".idx"), diff --git a/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.cpp b/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.cpp index 944a86857ac..4f52a3734da 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.cpp @@ -3,6 +3,8 @@ #include #include +#include + namespace DB { @@ -14,13 +16,15 @@ namespace ErrorCodes } MergeTreeMarksLoader::MergeTreeMarksLoader( + DiskPtr disk_, MarkCache * mark_cache_, const String & mrk_path_, size_t marks_count_, const MergeTreeIndexGranularityInfo & index_granularity_info_, bool save_marks_in_cache_, size_t columns_in_mark_) - : mark_cache(mark_cache_) + : disk(std::move(disk_)) + , mark_cache(mark_cache_) , mrk_path(mrk_path_) , marks_count(marks_count_) , index_granularity_info(index_granularity_info_) @@ -46,13 +50,13 @@ MarkCache::MappedPtr MergeTreeMarksLoader::loadMarksImpl() /// Memory for marks must not be accounted as memory usage for query, because they are stored in shared cache. auto temporarily_disable_memory_tracker = getCurrentMemoryTrackerActionLock(); - size_t file_size = Poco::File(mrk_path).getSize(); + size_t file_size = disk->getFileSize(mrk_path); size_t mark_size = index_granularity_info.getMarkSizeInBytes(columns_in_mark); size_t expected_file_size = mark_size * marks_count; if (expected_file_size != file_size) throw Exception( - "Bad size of marks file '" + mrk_path + "': " + std::to_string(file_size) + ", must be: " + std::to_string(expected_file_size), + "Bad size of marks file '" + fullPath(disk, mrk_path) + "': " + std::to_string(file_size) + ", must be: " + std::to_string(expected_file_size), ErrorCodes::CORRUPTED_DATA); auto res = std::make_shared(marks_count * columns_in_mark); @@ -60,19 +64,20 @@ MarkCache::MappedPtr MergeTreeMarksLoader::loadMarksImpl() if (!index_granularity_info.is_adaptive) { /// Read directly to marks. - ReadBufferFromFile buffer(mrk_path, file_size, -1, reinterpret_cast(res->data())); + auto buffer = disk->readFile(mrk_path, file_size); + buffer->readStrict(reinterpret_cast(res->data()), sizeof(*res->data())); - if (buffer.eof() || buffer.buffer().size() != file_size) + if (buffer->eof() || buffer->buffer().size() != file_size) throw Exception("Cannot read all marks from file " + mrk_path, ErrorCodes::CANNOT_READ_ALL_DATA); } else { - ReadBufferFromFile buffer(mrk_path, file_size, -1); + auto buffer = disk->readFile(mrk_path, file_size); size_t i = 0; - while (!buffer.eof()) + while (!buffer->eof()) { - res->read(buffer, i * columns_in_mark, columns_in_mark); - buffer.seek(sizeof(size_t), SEEK_CUR); + res->read(*buffer, i * columns_in_mark, columns_in_mark); + buffer->seek(sizeof(size_t), SEEK_CUR); ++i; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.h b/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.h index 927e78ed8b9..3299356511c 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.h +++ b/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.h @@ -1,3 +1,4 @@ +#include #include namespace DB @@ -11,6 +12,7 @@ public: using MarksPtr = MarkCache::MappedPtr; MergeTreeMarksLoader( + DiskPtr disk_, MarkCache * mark_cache_, const String & mrk_path, size_t marks_count_, @@ -23,6 +25,7 @@ public: bool initialized() const { return marks != nullptr; } private: + DiskPtr disk; MarkCache * mark_cache = nullptr; String mrk_path; size_t marks_count; diff --git a/dbms/src/Storages/MergeTree/MergeTreeMutationEntry.cpp b/dbms/src/Storages/MergeTree/MergeTreeMutationEntry.cpp index d2ddf03f7bb..feb7dcb7685 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeMutationEntry.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeMutationEntry.cpp @@ -7,26 +7,29 @@ #include #include +#include + namespace DB { -MergeTreeMutationEntry::MergeTreeMutationEntry(MutationCommands commands_, const String & path_prefix_, Int64 tmp_number) +MergeTreeMutationEntry::MergeTreeMutationEntry(MutationCommands commands_, DiskPtr disk_, const String & path_prefix_, Int64 tmp_number) : create_time(time(nullptr)) , commands(std::move(commands_)) + , disk(std::move(disk_)) , path_prefix(path_prefix_) , file_name("tmp_mutation_" + toString(tmp_number) + ".txt") , is_temp(true) { try { - WriteBufferFromFile out(path_prefix + file_name); - out << "format version: 1\n" + auto out = disk->writeFile(path_prefix + file_name); + *out << "format version: 1\n" << "create time: " << LocalDateTime(create_time) << "\n"; - out << "commands: "; - commands.writeText(out); - out << "\n"; - out.sync(); + *out << "commands: "; + commands.writeText(*out); + *out << "\n"; + out->sync(); } catch (...) { @@ -39,7 +42,7 @@ void MergeTreeMutationEntry::commit(Int64 block_number_) { block_number = block_number_; String new_file_name = "mutation_" + toString(block_number) + ".txt"; - Poco::File(path_prefix + file_name).renameTo(path_prefix + new_file_name); + disk->moveFile(path_prefix + file_name, path_prefix + new_file_name); is_temp = false; file_name = new_file_name; } @@ -48,17 +51,17 @@ void MergeTreeMutationEntry::removeFile() { if (!file_name.empty()) { - Poco::File file(path_prefix + file_name); - if (!file.exists()) + if (!disk->exists(path_prefix + file_name)) return; - file.remove(false); + disk->remove(path_prefix + file_name); file_name.clear(); } } -MergeTreeMutationEntry::MergeTreeMutationEntry(const String & path_prefix_, const String & file_name_) - : path_prefix(path_prefix_) +MergeTreeMutationEntry::MergeTreeMutationEntry(DiskPtr disk_, const String & path_prefix_, const String & file_name_) + : disk(std::move(disk_)) + , path_prefix(path_prefix_) , file_name(file_name_) , is_temp(false) { @@ -66,20 +69,19 @@ MergeTreeMutationEntry::MergeTreeMutationEntry(const String & path_prefix_, cons file_name_buf >> "mutation_" >> block_number >> ".txt"; assertEOF(file_name_buf); - ReadBufferFromFile buf(path_prefix + file_name); + auto buf = disk->readFile(path_prefix + file_name); - buf >> "format version: 1\n"; + *buf >> "format version: 1\n"; LocalDateTime create_time_dt; - buf >> "create time: " >> create_time_dt >> "\n"; + *buf >> "create time: " >> create_time_dt >> "\n"; create_time = create_time_dt; - buf >> "commands: "; - commands.readText(buf); - buf >> "\n"; - - assertEOF(buf); + *buf >> "commands: "; + commands.readText(*buf); + *buf >> "\n"; + assertEOF(*buf); } MergeTreeMutationEntry::~MergeTreeMutationEntry() diff --git a/dbms/src/Storages/MergeTree/MergeTreeMutationEntry.h b/dbms/src/Storages/MergeTree/MergeTreeMutationEntry.h index 2b3bde72552..ccdea771e93 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeMutationEntry.h +++ b/dbms/src/Storages/MergeTree/MergeTreeMutationEntry.h @@ -1,8 +1,9 @@ #pragma once #include -#include +#include #include +#include namespace DB @@ -15,6 +16,7 @@ struct MergeTreeMutationEntry time_t create_time = 0; MutationCommands commands; + DiskPtr disk; String path_prefix; String file_name; bool is_temp = false; @@ -27,7 +29,7 @@ struct MergeTreeMutationEntry String latest_fail_reason; /// Create a new entry and write it to a temporary file. - MergeTreeMutationEntry(MutationCommands commands_, const String & path_prefix_, Int64 tmp_number); + MergeTreeMutationEntry(MutationCommands commands_, DiskPtr disk, const String & path_prefix_, Int64 tmp_number); MergeTreeMutationEntry(const MergeTreeMutationEntry &) = delete; MergeTreeMutationEntry(MergeTreeMutationEntry &&) = default; @@ -37,7 +39,7 @@ struct MergeTreeMutationEntry void removeFile(); /// Load an existing entry. - MergeTreeMutationEntry(const String & path_prefix_, const String & file_name_); + MergeTreeMutationEntry(DiskPtr disk_, const String & path_prefix_, const String & file_name_); ~MergeTreeMutationEntry(); }; diff --git a/dbms/src/Storages/MergeTree/MergeTreePartition.cpp b/dbms/src/Storages/MergeTree/MergeTreePartition.cpp index 2b278963deb..000d0abad43 100644 --- a/dbms/src/Storages/MergeTree/MergeTreePartition.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreePartition.cpp @@ -1,7 +1,6 @@ #include #include #include -#include #include #include #include @@ -12,7 +11,6 @@ #include #include -#include namespace DB { @@ -21,9 +19,9 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -static ReadBufferFromFile openForReading(const String & path) +static std::unique_ptr openForReading(const DiskPtr & disk, const String & path) { - return ReadBufferFromFile(path, std::min(static_cast(DBMS_DEFAULT_BUFFER_SIZE), Poco::File(path).getSize())); + return disk->readFile(path, std::min(size_t(DBMS_DEFAULT_BUFFER_SIZE), disk->getFileSize(path))); } String MergeTreePartition::getID(const MergeTreeData & storage) const @@ -123,29 +121,30 @@ void MergeTreePartition::serializeText(const MergeTreeData & storage, WriteBuffe } } -void MergeTreePartition::load(const MergeTreeData & storage, const String & part_path) +void MergeTreePartition::load(const MergeTreeData & storage, const DiskPtr & disk, const String & part_path) { if (!storage.partition_key_expr) return; - ReadBufferFromFile file = openForReading(part_path + "partition.dat"); + auto partition_file_path = part_path + "partition.dat"; + auto file = openForReading(disk, partition_file_path); value.resize(storage.partition_key_sample.columns()); for (size_t i = 0; i < storage.partition_key_sample.columns(); ++i) - storage.partition_key_sample.getByPosition(i).type->deserializeBinary(value[i], file); + storage.partition_key_sample.getByPosition(i).type->deserializeBinary(value[i], *file); } -void MergeTreePartition::store(const MergeTreeData & storage, const String & part_path, MergeTreeDataPartChecksums & checksums) const +void MergeTreePartition::store(const MergeTreeData & storage, const DiskPtr & disk, const String & part_path, MergeTreeDataPartChecksums & checksums) const { - store(storage.partition_key_sample, part_path, checksums); + store(storage.partition_key_sample, disk, part_path, checksums); } -void MergeTreePartition::store(const Block & partition_key_sample, const String & part_path, MergeTreeDataPartChecksums & checksums) const +void MergeTreePartition::store(const Block & partition_key_sample, const DiskPtr & disk, const String & part_path, MergeTreeDataPartChecksums & checksums) const { if (!partition_key_sample) return; - WriteBufferFromFile out(part_path + "partition.dat"); - HashingWriteBuffer out_hashing(out); + auto out = disk->writeFile(part_path + "partition.dat"); + HashingWriteBuffer out_hashing(*out); for (size_t i = 0; i < value.size(); ++i) partition_key_sample.getByPosition(i).type->serializeBinary(value[i], out_hashing); out_hashing.next(); diff --git a/dbms/src/Storages/MergeTree/MergeTreePartition.h b/dbms/src/Storages/MergeTree/MergeTreePartition.h index 678bf97a23c..2a589339ba8 100644 --- a/dbms/src/Storages/MergeTree/MergeTreePartition.h +++ b/dbms/src/Storages/MergeTree/MergeTreePartition.h @@ -1,7 +1,8 @@ #pragma once -#include #include +#include +#include #include namespace DB @@ -30,9 +31,9 @@ public: void serializeText(const MergeTreeData & storage, WriteBuffer & out, const FormatSettings & format_settings) const; - void load(const MergeTreeData & storage, const String & part_path); - void store(const MergeTreeData & storage, const String & part_path, MergeTreeDataPartChecksums & checksums) const; - void store(const Block & partition_key_sample, const String & part_path, MergeTreeDataPartChecksums & checksums) const; + void load(const MergeTreeData & storage, const DiskPtr & disk, const String & part_path); + void store(const MergeTreeData & storage, const DiskPtr & disk, const String & part_path, MergeTreeDataPartChecksums & checksums) const; + void store(const Block & partition_key_sample, const DiskPtr & disk, const String & part_path, MergeTreeDataPartChecksums & checksums) const; void assign(const MergeTreePartition & other) { value.assign(other.value); } }; diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index 4086c346a6a..8698ba78676 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -27,18 +27,22 @@ MergeTreeReaderCompact::MergeTreeReaderCompact( : IMergeTreeReader(data_part_, columns_, uncompressed_cache_, mark_cache_, mark_ranges_, settings_, avg_value_size_hints_) - , marks_loader(mark_cache, - data_part->index_granularity_info.getMarksFilePath(path + MergeTreeDataPartCompact::DATA_FILE_NAME), + , marks_loader( + data_part->disk, + mark_cache, + data_part->index_granularity_info.getMarksFilePath(data_part->getFullRelativePath() + MergeTreeDataPartCompact::DATA_FILE_NAME), data_part->getMarksCount(), data_part->index_granularity_info, settings.save_marks_in_cache, data_part->getColumns().size()) { size_t buffer_size = settings.max_read_buffer_size; - const String full_data_path = path + MergeTreeDataPartCompact::DATA_FILE_NAME_WITH_EXTENSION; + const String full_data_path = data_part->getFullRelativePath() + MergeTreeDataPartCompact::DATA_FILE_NAME_WITH_EXTENSION; if (uncompressed_cache) { - auto buffer = std::make_unique( - full_data_path, uncompressed_cache, 0, settings.min_bytes_to_use_direct_io, buffer_size); + auto buffer = + std::make_unique( + data_part->disk->readFile(full_data_path, buffer_size, 0, settings.min_bytes_to_use_direct_io, 0), + uncompressed_cache); if (profile_callback_) buffer->setProfileCallback(profile_callback_, clock_type_); @@ -48,8 +52,9 @@ MergeTreeReaderCompact::MergeTreeReaderCompact( } else { - auto buffer = std::make_unique( - full_data_path, 0, settings.min_bytes_to_use_direct_io, buffer_size); + auto buffer = + std::make_unique( + data_part->disk->readFile(full_data_path, buffer_size, 0, settings.min_bytes_to_use_direct_io, 0)); if (profile_callback_) buffer->setProfileCallback(profile_callback_, clock_type_); diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp index 140971fb6d4..d0e73cd16e0 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp @@ -1,6 +1,7 @@ #include #include -#include + +#include namespace DB @@ -13,6 +14,7 @@ namespace ErrorCodes MergeTreeReaderStream::MergeTreeReaderStream( + DiskPtr disk_, const String & path_prefix_, const String & data_file_extension_, size_t marks_count_, const MarkRanges & all_mark_ranges, const MergeTreeReaderSettings & settings, @@ -20,10 +22,10 @@ MergeTreeReaderStream::MergeTreeReaderStream( UncompressedCache * uncompressed_cache, size_t file_size, const MergeTreeIndexGranularityInfo * index_granularity_info_, const ReadBufferFromFileBase::ProfileCallback & profile_callback, clockid_t clock_type) - : path_prefix(path_prefix_), data_file_extension(data_file_extension_), marks_count(marks_count_) + : disk(std::move(disk_)), path_prefix(path_prefix_), data_file_extension(data_file_extension_), marks_count(marks_count_) , mark_cache(mark_cache_), save_marks_in_cache(settings.save_marks_in_cache) , index_granularity_info(index_granularity_info_) - , marks_loader(mark_cache, index_granularity_info->getMarksFilePath(path_prefix), + , marks_loader(disk, mark_cache, index_granularity_info->getMarksFilePath(path_prefix), marks_count, *index_granularity_info, save_marks_in_cache) { /// Compute the size of the buffer. @@ -77,8 +79,9 @@ MergeTreeReaderStream::MergeTreeReaderStream( if (uncompressed_cache) { auto buffer = std::make_unique( - path_prefix + data_file_extension, uncompressed_cache, sum_mark_range_bytes, - settings.min_bytes_to_use_direct_io, settings.min_bytes_to_use_mmap_io, buffer_size); + disk->readFile(path_prefix + data_file_extension, buffer_size, + sum_mark_range_bytes, settings.min_bytes_to_use_direct_io, settings.min_bytes_to_use_mmap_io), + uncompressed_cache); if (profile_callback) buffer->setProfileCallback(profile_callback, clock_type); @@ -89,8 +92,9 @@ MergeTreeReaderStream::MergeTreeReaderStream( else { auto buffer = std::make_unique( - path_prefix + data_file_extension, sum_mark_range_bytes, - settings.min_bytes_to_use_direct_io, settings.min_bytes_to_use_mmap_io, buffer_size); + disk->readFile(path_prefix + data_file_extension, buffer_size, + sum_mark_range_bytes, settings.min_bytes_to_use_direct_io, settings.min_bytes_to_use_mmap_io) + ); if (profile_callback) buffer->setProfileCallback(profile_callback, clock_type); diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.h b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.h index ccadbcd8e4a..529098987a6 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.h @@ -17,6 +17,7 @@ class MergeTreeReaderStream { public: MergeTreeReaderStream( + DiskPtr disk_, const String & path_prefix_, const String & data_file_extension_, size_t marks_count_, const MarkRanges & all_mark_ranges, const MergeTreeReaderSettings & settings_, @@ -31,6 +32,7 @@ public: ReadBuffer * data_buffer; private: + DiskPtr disk; std::string path_prefix; std::string data_file_extension; diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp index 21f8b31aaea..e7cf3947138 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp @@ -137,7 +137,7 @@ size_t MergeTreeReaderWide::readRows(size_t from_mark, bool continue_reading, si storage.reportBrokenPart(data_part->name); /// Better diagnostics. - e.addMessage("(while reading from part " + path + " " + e.addMessage("(while reading from part " + data_part->getFullPath() + " " "from mark " + toString(from_mark) + " " "with max_rows_to_read = " + toString(max_rows_to_read) + ")"); throw; @@ -171,8 +171,8 @@ void MergeTreeReaderWide::addStreams(const String & name, const IDataType & type return; streams.emplace(stream_name, std::make_unique( - path + stream_name, DATA_FILE_EXTENSION, data_part->getMarksCount(), - all_mark_ranges, settings, mark_cache, + data_part->disk, data_part->getFullRelativePath() + stream_name, DATA_FILE_EXTENSION, + data_part->getMarksCount(), all_mark_ranges, settings, mark_cache, uncompressed_cache, data_part->getFileSizeOrZero(stream_name + DATA_FILE_EXTENSION), &data_part->index_granularity_info, profile_callback, clock_type)); diff --git a/dbms/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp b/dbms/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp index 750001e1384..47b68aa1b7f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp @@ -57,7 +57,7 @@ MergeTreeReverseSelectProcessor::MergeTreeReverseSelectProcessor( part_columns_lock(data_part->columns_lock), all_mark_ranges(std::move(mark_ranges_)), part_index_in_query(part_index_in_query_), - path(data_part->getFullPath()) + path(data_part->getFullRelativePath()) { /// Let's estimate total number of rows for progress bar. for (const auto & range : all_mark_ranges) diff --git a/dbms/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp b/dbms/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp index 4eb8a42b741..55d541b60c5 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp @@ -58,7 +58,7 @@ MergeTreeSelectProcessor::MergeTreeSelectProcessor( all_mark_ranges(std::move(mark_ranges_)), part_index_in_query(part_index_in_query_), check_columns(check_columns_), - path(data_part->getFullPath()) + path(data_part->getFullRelativePath()) { /// Let's estimate total number of rows for progress bar. for (const auto & range : all_mark_ranges) diff --git a/dbms/src/Storages/MergeTree/MergeTreeSettings.h b/dbms/src/Storages/MergeTree/MergeTreeSettings.h index bbd1fd6cbeb..7d53f161620 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSettings.h +++ b/dbms/src/Storages/MergeTree/MergeTreeSettings.h @@ -42,6 +42,7 @@ struct MergeTreeSettings : public SettingsCollection M(SettingUInt64, number_of_free_entries_in_pool_to_execute_mutation, 10, "When there is less than specified number of free entries in pool, do not execute part mutations. This is to leave free threads for regular merges and avoid \"Too many parts\"", 0) \ M(SettingSeconds, old_parts_lifetime, 8 * 60, "How many seconds to keep obsolete parts.", 0) \ M(SettingSeconds, temporary_directories_lifetime, 86400, "How many seconds to keep tmp_-directories.", 0) \ + M(SettingBool, disable_background_merges, false, "Disable background merges.", 0) \ \ /** Inserts settings. */ \ M(SettingUInt64, parts_to_delay_insert, 150, "If table contains at least that many active parts in single partition, artificially slow down insert into table.", 0) \ diff --git a/dbms/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputProcessor.cpp b/dbms/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputProcessor.cpp index 41a7bb877b5..aa8c550839d 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputProcessor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputProcessor.cpp @@ -57,7 +57,7 @@ bool MergeTreeThreadSelectBlockInputProcessor::getNewTask() return false; } - const std::string path = task->data_part->getFullPath(); + const std::string path = task->data_part->getFullRelativePath(); /// Allows pool to reduce number of threads in case of too slow reads. auto profile_callback = [this](ReadBufferFromFileBase::ProfileInfo info_) { pool->profileFeedback(info_); }; diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp index c46618a55ae..e04d1b41ffe 100644 --- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -47,18 +47,13 @@ MergedBlockOutputStream::MergedBlockOutputStream( } } - Poco::File(part_path).createDirectories(); + disk->createDirectories(part_path); writer = data_part->getWriter(columns_list, data_part->storage.getSkipIndices(), default_codec, writer_settings); writer->initPrimaryIndex(); writer->initSkipIndices(); } -std::string MergedBlockOutputStream::getPartPath() const -{ - return part_path; -} - /// If data is pre-sorted. void MergedBlockOutputStream::write(const Block & block) { @@ -99,15 +94,15 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING || isCompactPart(new_part)) { - new_part->partition.store(storage, part_path, checksums); + new_part->partition.store(storage, disk, part_path, checksums); if (new_part->minmax_idx.initialized) - new_part->minmax_idx.store(storage, part_path, checksums); + new_part->minmax_idx.store(storage, disk, part_path, checksums); else if (rows_count) throw Exception("MinMax index was not initialized for new non-empty part " + new_part->name + ". It is a bug.", ErrorCodes::LOGICAL_ERROR); - WriteBufferFromFile count_out(part_path + "count.txt", 4096); - HashingWriteBuffer count_out_hashing(count_out); + auto count_out = disk->writeFile(part_path + "count.txt", 4096); + HashingWriteBuffer count_out_hashing(*count_out); writeIntText(rows_count, count_out_hashing); count_out_hashing.next(); checksums.files["count.txt"].file_size = count_out_hashing.count(); @@ -117,8 +112,8 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( if (!new_part->ttl_infos.empty()) { /// Write a file with ttl infos in json format. - WriteBufferFromFile out(part_path + "ttl.txt", 4096); - HashingWriteBuffer out_hashing(out); + auto out = disk->writeFile(part_path + "ttl.txt", 4096); + HashingWriteBuffer out_hashing(*out); new_part->ttl_infos.write(out_hashing); checksums.files["ttl.txt"].file_size = out_hashing.count(); checksums.files["ttl.txt"].file_hash = out_hashing.getHash(); @@ -126,14 +121,14 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( { /// Write a file with a description of columns. - WriteBufferFromFile out(part_path + "columns.txt", 4096); - total_column_list->writeText(out); + auto out = disk->writeFile(part_path + "columns.txt", 4096); + total_column_list->writeText(*out); } { /// Write file with checksums. - WriteBufferFromFile out(part_path + "checksums.txt", 4096); - checksums.write(out); + auto out = disk->writeFile(part_path + "checksums.txt", 4096); + checksums.write(*out); } new_part->rows_count = rows_count; diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.h b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.h index d7514957bdd..ee453f41a31 100644 --- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.h +++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.h @@ -27,8 +27,6 @@ public: size_t aio_threshold, bool blocks_are_granules_size = false); - std::string getPartPath() const; - Block getHeader() const override { return storage.getSampleBlock(); } /// If the data is pre-sorted. diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 06cb4b4be0d..192b4194b7a 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -95,7 +95,8 @@ void StorageMergeTree::startup() /// NOTE background task will also do the above cleanups periodically. time_after_previous_cleanup.restart(); - merging_mutating_task_handle = global_context.getBackgroundPool().addTask([this] { return mergeMutateTask(); }); + if (!getSettings()->disable_background_merges) + merging_mutating_task_handle = global_context.getBackgroundPool().addTask([this] { return mergeMutateTask(); }); if (areBackgroundMovesNeeded()) moving_task_handle = global_context.getBackgroundMovePool().addTask([this] { return movePartsTask(); }); } @@ -417,7 +418,7 @@ void StorageMergeTree::mutate(const MutationCommands & commands, const Context & /// Choose any disk, because when we load mutations we search them at each disk /// where storage can be placed. See loadMutations(). auto disk = storage_policy->getAnyDisk(); - MergeTreeMutationEntry entry(commands, getFullPathOnDisk(disk), insert_increment.get()); + MergeTreeMutationEntry entry(commands, disk, relative_data_path, insert_increment.get()); String file_name; Int64 version; { @@ -558,22 +559,20 @@ CancellationCode StorageMergeTree::killMutation(const String & mutation_id) void StorageMergeTree::loadMutations() { - Poco::DirectoryIterator end; - const auto full_paths = getDataPaths(); - for (const String & full_path : full_paths) + for (const auto & [path, disk] : getRelativeDataPathsWithDisks()) { - for (auto it = Poco::DirectoryIterator(full_path); it != end; ++it) + for (auto it = disk->iterateDirectory(path); it->isValid(); it->next()) { - if (startsWith(it.name(), "mutation_")) + if (startsWith(it->name(), "mutation_")) { - MergeTreeMutationEntry entry(full_path, it.name()); + MergeTreeMutationEntry entry(disk, it->path(), it->name()); Int64 block_number = entry.block_number; - auto insertion = current_mutations_by_id.emplace(it.name(), std::move(entry)); + auto insertion = current_mutations_by_id.emplace(it->name(), std::move(entry)); current_mutations_by_version.emplace(block_number, insertion.first->second); } - else if (startsWith(it.name(), "tmp_mutation_")) + else if (startsWith(it->name(), "tmp_mutation_")) { - it->remove(); + disk->remove(it->path()); } } } diff --git a/dbms/tests/integration/test_merge_tree_s3/__init__.py b/dbms/tests/integration/test_merge_tree_s3/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/integration/test_merge_tree_s3/configs/config.d/log_conf.xml b/dbms/tests/integration/test_merge_tree_s3/configs/config.d/log_conf.xml new file mode 100644 index 00000000000..318a6bca95d --- /dev/null +++ b/dbms/tests/integration/test_merge_tree_s3/configs/config.d/log_conf.xml @@ -0,0 +1,12 @@ + + 3 + + trace + /var/log/clickhouse-server/log.log + /var/log/clickhouse-server/log.err.log + 1000M + 10 + /var/log/clickhouse-server/stderr.log + /var/log/clickhouse-server/stdout.log + + diff --git a/dbms/tests/integration/test_merge_tree_s3/configs/config.xml b/dbms/tests/integration/test_merge_tree_s3/configs/config.xml new file mode 100644 index 00000000000..63b4d951eb7 --- /dev/null +++ b/dbms/tests/integration/test_merge_tree_s3/configs/config.xml @@ -0,0 +1,40 @@ + + + + trace + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + 1000M + 10 + + + + + + s3 + http://minio1:9001/root/data/ + minio + minio123 + + + + + + 9000 + 127.0.0.1 + + + + true + none + + AcceptCertificateHandler + + + + + 500 + 5368709120 + ./clickhouse/ + users.xml + diff --git a/dbms/tests/integration/test_merge_tree_s3/configs/users.xml b/dbms/tests/integration/test_merge_tree_s3/configs/users.xml new file mode 100644 index 00000000000..6061af8e33d --- /dev/null +++ b/dbms/tests/integration/test_merge_tree_s3/configs/users.xml @@ -0,0 +1,23 @@ + + + + + + + + + + + + ::/0 + + default + default + + + + + + + + diff --git a/dbms/tests/integration/test_merge_tree_s3/test.py b/dbms/tests/integration/test_merge_tree_s3/test.py new file mode 100644 index 00000000000..c79745642a0 --- /dev/null +++ b/dbms/tests/integration/test_merge_tree_s3/test.py @@ -0,0 +1,91 @@ +import logging +import random +import string + +import pytest +from helpers.cluster import ClickHouseCluster + +logging.getLogger().setLevel(logging.INFO) +logging.getLogger().addHandler(logging.StreamHandler()) + + +# Creates S3 bucket for tests and allows anonymous read-write access to it. +def prepare_s3_bucket(cluster): + minio_client = cluster.minio_client + + if minio_client.bucket_exists(cluster.minio_bucket): + minio_client.remove_bucket(cluster.minio_bucket) + + minio_client.make_bucket(cluster.minio_bucket) + + +@pytest.fixture(scope="module") +def cluster(): + try: + cluster = ClickHouseCluster(__file__) + cluster.add_instance("node", config_dir="configs", with_minio=True) + logging.info("Starting cluster...") + cluster.start() + logging.info("Cluster started") + + prepare_s3_bucket(cluster) + logging.info("S3 bucket created") + + yield cluster + finally: + cluster.shutdown() + + +def random_string(length): + letters = string.ascii_letters + return ''.join(random.choice(letters) for i in range(length)) + + +def generate_values(date_str, count): + data = [[date_str, i, random_string(10)] for i in range(count)] + data.sort(key=lambda tup: tup[1]) + return ",".join(["('{}',{},'{}')".format(x, y, z) for x, y, z in data]) + + +@pytest.mark.parametrize( + "min_rows_for_wide_part,files_overhead,files_per_part", + [ + (0, 1, 14), + (8192, 1, 10) + ] +) +def test_log_family_s3(cluster, min_rows_for_wide_part, files_overhead, files_per_part): + node = cluster.instances["node"] + minio = cluster.minio_client + + node.query( + """ + CREATE TABLE s3_test( + dt Date, + id UInt64, + data String, + INDEX min_max (id) TYPE minmax GRANULARITY 3 + ) ENGINE=MergeTree() + PARTITION BY dt + ORDER BY (dt, id) + SETTINGS disable_background_merges='true', index_granularity=512, min_rows_for_wide_part={} + """ + .format(min_rows_for_wide_part) + ) + assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == 1 + + values1 = generate_values('2020-01-03', 4096) + node.query("INSERT INTO s3_test VALUES {}".format(values1)) + assert node.query("SELECT * FROM s3_test order by dt, id FORMAT Values") == values1 + assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == files_overhead + files_per_part + + values2 = generate_values('2020-01-04', 4096) + node.query("INSERT INTO s3_test VALUES {}".format(values2)) + assert node.query("SELECT * FROM s3_test ORDER BY dt, id FORMAT Values") == values1 + "," + values2 + assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == files_overhead + 2 * files_per_part + + assert node.query("SELECT count(*) FROM s3_test where id = 0 FORMAT Values") == "(2)" + + node.query("DROP TABLE s3_test") + assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == 0 + From 8c09902310f872c69f120310608f2dea5ac3f65e Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Thu, 27 Feb 2020 20:57:49 +0300 Subject: [PATCH 031/712] Code style issues. --- dbms/src/Disks/DiskS3.cpp | 1 - dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dbms/src/Disks/DiskS3.cpp b/dbms/src/Disks/DiskS3.cpp index ef0f4e07356..eefa15c4e71 100644 --- a/dbms/src/Disks/DiskS3.cpp +++ b/dbms/src/Disks/DiskS3.cpp @@ -28,7 +28,6 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; extern const int FILE_ALREADY_EXISTS; extern const int PATH_ACCESS_DENIED; - extern const int SEEK_POSITION_OUT_OF_BOUND; extern const int CANNOT_SEEK_THROUGH_FILE; extern const int UNKNOWN_FORMAT; } diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp index e8c9f2a113e..4145dd1ff9d 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -868,7 +868,8 @@ void IMergeTreeDataPart::checkConsistencyBase() const } else { - auto check_file_not_empty = [&path](const DiskPtr & disk_, const String & file_path) { + auto check_file_not_empty = [&path](const DiskPtr & disk_, const String & file_path) + { UInt64 file_size; if (!disk_->exists(file_path) || (file_size = disk_->getFileSize(file_path)) == 0) throw Exception("Part " + fullPath(disk_, path) + " is broken: " + fullPath(disk_, file_path) + " is empty", ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); From d91ef06370511cd7919f225dae09676417c8675d Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Fri, 28 Feb 2020 14:54:18 +0300 Subject: [PATCH 032/712] Code style and logical issues fix. --- dbms/src/Compression/CachedCompressedReadBuffer.h | 8 ++++---- dbms/src/Storages/MergeTree/MergeTreeMarksLoader.cpp | 7 ++++--- utils/convert-month-partitioned-parts/main.cpp | 6 ++++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/dbms/src/Compression/CachedCompressedReadBuffer.h b/dbms/src/Compression/CachedCompressedReadBuffer.h index 2fcfbdba51b..4f9f1fce480 100644 --- a/dbms/src/Compression/CachedCompressedReadBuffer.h +++ b/dbms/src/Compression/CachedCompressedReadBuffer.h @@ -24,10 +24,10 @@ private: std::unique_ptr file_in; const std::string path; - size_t buf_size; - size_t estimated_size; - size_t aio_threshold; - size_t mmap_threshold; + size_t buf_size {}; + size_t estimated_size {}; + size_t aio_threshold {}; + size_t mmap_threshold {}; size_t file_pos; diff --git a/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.cpp b/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.cpp index 4f52a3734da..0df125d69ff 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.cpp @@ -65,10 +65,11 @@ MarkCache::MappedPtr MergeTreeMarksLoader::loadMarksImpl() { /// Read directly to marks. auto buffer = disk->readFile(mrk_path, file_size); - buffer->readStrict(reinterpret_cast(res->data()), sizeof(*res->data())); + buffer->readStrict(reinterpret_cast(res->data()), file_size); - if (buffer->eof() || buffer->buffer().size() != file_size) - throw Exception("Cannot read all marks from file " + mrk_path, ErrorCodes::CANNOT_READ_ALL_DATA); + if (!buffer->eof()) + throw Exception("Cannot read all marks from file " + mrk_path + ", eof: " + std::to_string(buffer->eof()) + + ", buffer size: " + std::to_string(buffer->buffer().size()) + ", file size: " + std::to_string(file_size), ErrorCodes::CANNOT_READ_ALL_DATA); } else { diff --git a/utils/convert-month-partitioned-parts/main.cpp b/utils/convert-month-partitioned-parts/main.cpp index c1b89f2251e..8f1ca05dd32 100644 --- a/utils/convert-month-partitioned-parts/main.cpp +++ b/utils/convert-month-partitioned-parts/main.cpp @@ -14,6 +14,7 @@ #include #include +#include namespace DB { @@ -27,6 +28,7 @@ namespace ErrorCodes void run(String part_path, String date_column, String dest_path) { + std::shared_ptr disk = std::make_shared("local", "/", 0); auto old_part_path = Poco::Path::forDirectory(part_path); String old_part_name = old_part_path.directory(old_part_path.depth() - 1); String old_part_path_str = old_part_path.toString(); @@ -83,12 +85,12 @@ void run(String part_path, String date_column, String dest_path) IMergeTreeDataPart::MinMaxIndex minmax_idx(min_date, max_date); Names minmax_idx_columns = {date_column}; DataTypes minmax_idx_column_types = {std::make_shared()}; - minmax_idx.store(minmax_idx_columns, minmax_idx_column_types, new_tmp_part_path_str, checksums); + minmax_idx.store(minmax_idx_columns, minmax_idx_column_types, disk, new_tmp_part_path_str, checksums); Block partition_key_sample{{nullptr, std::make_shared(), makeASTFunction("toYYYYMM", std::make_shared(date_column))->getColumnName()}}; MergeTreePartition partition(yyyymm); - partition.store(partition_key_sample, new_tmp_part_path_str, checksums); + partition.store(partition_key_sample, disk, new_tmp_part_path_str, checksums); String partition_id = partition.getID(partition_key_sample); Poco::File(new_tmp_part_path_str + "checksums.txt").setWriteable(); From b0b391d472c2a28331a9a67e520b71b7fe214b31 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 28 Feb 2020 19:29:16 +0300 Subject: [PATCH 033/712] Change endless performance tests (1) --- dbms/tests/performance/array_element.xml | 8 ++++---- dbms/tests/performance/array_join.xml | 14 +++++++------- dbms/tests/performance/base64.xml | 12 ++++++------ dbms/tests/performance/base64_hits.xml | 10 +++++----- dbms/tests/performance/bitCount.xml | 4 ++-- 5 files changed, 24 insertions(+), 24 deletions(-) diff --git a/dbms/tests/performance/array_element.xml b/dbms/tests/performance/array_element.xml index 672683fe146..2d05f9dcf51 100644 --- a/dbms/tests/performance/array_element.xml +++ b/dbms/tests/performance/array_element.xml @@ -1,5 +1,5 @@ - once + loop @@ -9,7 +9,7 @@ - SELECT count() FROM system.numbers WHERE NOT ignore([[1], [2]][number % 2 + 2]) - SELECT count() FROM system.numbers WHERE NOT ignore([[], [2]][number % 2 + 2]) - SELECT count() FROM system.numbers WHERE NOT ignore([[], []][number % 2 + 2]) + SELECT count() FROM numbers(1000000) WHERE NOT ignore([[1], [2]][number % 2 + 2]) + SELECT count() FROM numbers(1000000) WHERE NOT ignore([[], [2]][number % 2 + 2]) + SELECT count() FROM numbers(1000000) WHERE NOT ignore([[], []][number % 2 + 2]) diff --git a/dbms/tests/performance/array_join.xml b/dbms/tests/performance/array_join.xml index d2eb213ce03..c30039588e0 100644 --- a/dbms/tests/performance/array_join.xml +++ b/dbms/tests/performance/array_join.xml @@ -1,5 +1,5 @@ - once + loop @@ -9,10 +9,10 @@ - SELECT count() FROM (SELECT [number] a, [number * 2] b FROM system.numbers) AS t ARRAY JOIN a, b WHERE NOT ignore(a + b) - SELECT count() FROM (SELECT [number] a, [number * 2] b FROM system.numbers) AS t LEFT ARRAY JOIN a, b WHERE NOT ignore(a + b) - SELECT count() FROM (SELECT [number] a, [number * 2] b FROM system.numbers) AS t ARRAY JOIN a, b WHERE NOT ignore(a + b) SETTINGS enable_unaligned_array_join = 1 - SELECT count() FROM (SELECT [number] a, [number * 2] b FROM system.numbers) AS t LEFT ARRAY JOIN a, b WHERE NOT ignore(a + b) SETTINGS enable_unaligned_array_join = 1 - SELECT count() FROM (SELECT [number] a, [number * 2, number] b FROM system.numbers) AS t ARRAY JOIN a, b WHERE NOT ignore(a + b) SETTINGS enable_unaligned_array_join = 1 - SELECT count() FROM (SELECT [number] a, [number * 2, number] b FROM system.numbers) AS t LEFT ARRAY JOIN a, b WHERE NOT ignore(a + b) SETTINGS enable_unaligned_array_join = 1 + SELECT count() FROM (SELECT [number] a, [number * 2] b FROM numbers(1000000)) AS t ARRAY JOIN a, b WHERE NOT ignore(a + b) + SELECT count() FROM (SELECT [number] a, [number * 2] b FROM numbers(1000000)) AS t LEFT ARRAY JOIN a, b WHERE NOT ignore(a + b) + SELECT count() FROM (SELECT [number] a, [number * 2] b FROM numbers(1000000)) AS t ARRAY JOIN a, b WHERE NOT ignore(a + b) SETTINGS enable_unaligned_array_join = 1 + SELECT count() FROM (SELECT [number] a, [number * 2] b FROM numbers(1000000)) AS t LEFT ARRAY JOIN a, b WHERE NOT ignore(a + b) SETTINGS enable_unaligned_array_join = 1 + SELECT count() FROM (SELECT [number] a, [number * 2, number] b FROM numbers(1000000)) AS t ARRAY JOIN a, b WHERE NOT ignore(a + b) SETTINGS enable_unaligned_array_join = 1 + SELECT count() FROM (SELECT [number] a, [number * 2, number] b FROM numbers(1000000)) AS t LEFT ARRAY JOIN a, b WHERE NOT ignore(a + b) SETTINGS enable_unaligned_array_join = 1 diff --git a/dbms/tests/performance/base64.xml b/dbms/tests/performance/base64.xml index 651412c2752..3175c7811bf 100644 --- a/dbms/tests/performance/base64.xml +++ b/dbms/tests/performance/base64.xml @@ -1,5 +1,5 @@ - once + loop @@ -24,13 +24,13 @@ table - numbers - numbers_mt + numbers(1000000) + numbers_mt(10000000) - SELECT count() FROM system.{table} WHERE NOT ignore(base64Encode({string})) - SELECT count() FROM system.{table} WHERE base64Decode(base64Encode({string})) != {string} - SELECT count() FROM system.{table} WHERE tryBase64Decode(base64Encode({string})) != {string} + SELECT count() FROM {table} WHERE NOT ignore(base64Encode({string})) + SELECT count() FROM {table} WHERE base64Decode(base64Encode({string})) != {string} + SELECT count() FROM {table} WHERE tryBase64Decode(base64Encode({string})) != {string} diff --git a/dbms/tests/performance/base64_hits.xml b/dbms/tests/performance/base64_hits.xml index 7b07f3badb7..63916dcee4e 100644 --- a/dbms/tests/performance/base64_hits.xml +++ b/dbms/tests/performance/base64_hits.xml @@ -1,8 +1,8 @@ - loop + once - hits_100m_single + hits_10m_single @@ -28,7 +28,7 @@ - SELECT count() FROM hits_100m_single WHERE NOT ignore(base64Encode({string})) - SELECT count() FROM hits_100m_single WHERE base64Decode(base64Encode({string})) != {string} - SELECT count() FROM hits_100m_single WHERE tryBase64Decode(base64Encode({string})) != {string} + SELECT count() FROM hits_10m_single WHERE NOT ignore(base64Encode({string})) + SELECT count() FROM hits_10m_single WHERE base64Decode(base64Encode({string})) != {string} + SELECT count() FROM hits_10m_single WHERE tryBase64Decode(base64Encode({string})) != {string} diff --git a/dbms/tests/performance/bitCount.xml b/dbms/tests/performance/bitCount.xml index 8936f700b51..8a58f501180 100644 --- a/dbms/tests/performance/bitCount.xml +++ b/dbms/tests/performance/bitCount.xml @@ -1,5 +1,5 @@ - once + loop @@ -23,5 +23,5 @@ - SELECT bitCount({expr}) FROM system.numbers + SELECT bitCount({expr}) FROM numbers(1000000) From fa6ff1aa27308d6fddbd72cc1522b08e163ee90c Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 28 Feb 2020 19:50:13 +0300 Subject: [PATCH 034/712] Change some endless performance tests (2) --- dbms/tests/performance/bounding_ratio.xml | 7 +++---- .../performance/complex_array_creation.xml | 7 +++---- dbms/tests/performance/conditional.xml | 19 +++++++++---------- dbms/tests/performance/consistent_hashes.xml | 7 +++---- dbms/tests/performance/count.xml | 4 ---- .../performance/cryptographic_hashes.xml | 9 ++++----- 6 files changed, 22 insertions(+), 31 deletions(-) diff --git a/dbms/tests/performance/bounding_ratio.xml b/dbms/tests/performance/bounding_ratio.xml index efa2c10d989..4bf50f57290 100644 --- a/dbms/tests/performance/bounding_ratio.xml +++ b/dbms/tests/performance/bounding_ratio.xml @@ -1,14 +1,13 @@ - once + loop - 2000 10000 - SELECT boundingRatio(number, number) FROM system.numbers - SELECT (argMax(number, number) - argMin(number, number)) / (max(number) - min(number)) FROM system.numbers + SELECT boundingRatio(number, number) FROM numbers(1000000) + SELECT (argMax(number, number) - argMin(number, number)) / (max(number) - min(number)) FROM numbers(1000000) diff --git a/dbms/tests/performance/complex_array_creation.xml b/dbms/tests/performance/complex_array_creation.xml index a5ff824d6de..76e4910a1d7 100644 --- a/dbms/tests/performance/complex_array_creation.xml +++ b/dbms/tests/performance/complex_array_creation.xml @@ -1,14 +1,13 @@ - once + loop - 1000 10000 - SELECT count() FROM system.numbers WHERE NOT ignore([[number], [number]]) - SELECT count() FROM system.numbers WHERE NOT ignore([[], [number]]) + SELECT count() FROM numbers(1000000) WHERE NOT ignore([[number], [number]]) + SELECT count() FROM numbers(1000000) WHERE NOT ignore([[], [number]]) diff --git a/dbms/tests/performance/conditional.xml b/dbms/tests/performance/conditional.xml index 96f48fb401a..eea43d6556a 100644 --- a/dbms/tests/performance/conditional.xml +++ b/dbms/tests/performance/conditional.xml @@ -1,20 +1,19 @@ - once + loop - 3000 10000 - SELECT count() FROM system.numbers WHERE NOT ignore(if(rand() % 2, toDateTime('2019-02-04 01:24:31'), toDate('2019-02-04'))) - SELECT count() FROM system.numbers WHERE NOT ignore(multiIf(rand() % 2, toDateTime('2019-02-04 01:24:31'), toDate('2019-02-04'))) - SELECT count() FROM system.numbers WHERE NOT ignore(if(rand() % 2, [toDateTime('2019-02-04 01:24:31')], [toDate('2019-02-04')])) - SELECT count() FROM system.numbers WHERE NOT ignore(multiIf(rand() % 2, [toDateTime('2019-02-04 01:24:31')], [toDate('2019-02-04')])) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(if(rand() % 2, toDateTime('2019-02-04 01:24:31'), toDate('2019-02-04'))) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(multiIf(rand() % 2, toDateTime('2019-02-04 01:24:31'), toDate('2019-02-04'))) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(if(rand() % 2, [toDateTime('2019-02-04 01:24:31')], [toDate('2019-02-04')])) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(multiIf(rand() % 2, [toDateTime('2019-02-04 01:24:31')], [toDate('2019-02-04')])) - SELECT count() FROM system.numbers WHERE NOT ignore(if(rand() % 2, toDateTime(rand()), toDate(rand()))) - SELECT count() FROM system.numbers WHERE NOT ignore(multiIf(rand() % 2, toDateTime(rand()), toDate(rand()))) - SELECT count() FROM system.numbers WHERE NOT ignore(if(rand() % 2, [toDateTime(rand())], [toDate(rand())])) - SELECT count() FROM system.numbers WHERE NOT ignore(multiIf(rand() % 2, [toDateTime(rand())], [toDate(rand())])) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(if(rand() % 2, toDateTime(rand()), toDate(rand()))) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(multiIf(rand() % 2, toDateTime(rand()), toDate(rand()))) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(if(rand() % 2, [toDateTime(rand())], [toDate(rand())])) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(multiIf(rand() % 2, [toDateTime(rand())], [toDate(rand())])) diff --git a/dbms/tests/performance/consistent_hashes.xml b/dbms/tests/performance/consistent_hashes.xml index 7219aa00c1a..aee232ddce6 100644 --- a/dbms/tests/performance/consistent_hashes.xml +++ b/dbms/tests/performance/consistent_hashes.xml @@ -1,9 +1,8 @@ - once + loop - 6000 15000 @@ -27,6 +26,6 @@ - SELECT {hash_func}(number, {buckets}) FROM system.numbers - SELECT sumburConsistentHash(toUInt32(number), {buckets}) FROM system.numbers + SELECT {hash_func}(number, {buckets}) FROM numbers(1000000) + SELECT sumburConsistentHash(toUInt32(number), {buckets}) FROM numbers(1000000) diff --git a/dbms/tests/performance/count.xml b/dbms/tests/performance/count.xml index 0244adf4b38..3bb4a0d2cd5 100644 --- a/dbms/tests/performance/count.xml +++ b/dbms/tests/performance/count.xml @@ -6,10 +6,6 @@ 30000 - - 6000 - 60000 - diff --git a/dbms/tests/performance/cryptographic_hashes.xml b/dbms/tests/performance/cryptographic_hashes.xml index 7840a7b382a..71c37ed9f8d 100644 --- a/dbms/tests/performance/cryptographic_hashes.xml +++ b/dbms/tests/performance/cryptographic_hashes.xml @@ -1,12 +1,11 @@ - once + loop 10000 - 5000 20000 @@ -36,11 +35,11 @@ table - numbers - numbers_mt + numbers(1000000) + numbers_mt(10000000) - SELECT ignore({crypto_hash_func}({string})) FROM system.{table} + SELECT ignore({crypto_hash_func}({string})) FROM {table} From adb615405281b74471b46c9a61301ae57b67da9c Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 28 Feb 2020 19:56:34 +0300 Subject: [PATCH 035/712] Change endless performance tests (3) --- dbms/tests/performance/date_time.xml | 13 ++++++------- dbms/tests/performance/entropy.xml | 1 - dbms/tests/performance/float_formatting.xml | 5 ++--- dbms/tests/performance/float_parsing.xml | 5 ++--- dbms/tests/performance/format_date_time.xml | 5 ++--- dbms/tests/performance/functions_coding.xml | 11 +++++------ 6 files changed, 17 insertions(+), 23 deletions(-) diff --git a/dbms/tests/performance/date_time.xml b/dbms/tests/performance/date_time.xml index 77a6c634b34..e099b468560 100644 --- a/dbms/tests/performance/date_time.xml +++ b/dbms/tests/performance/date_time.xml @@ -1,6 +1,6 @@ - once + loop long @@ -8,7 +8,6 @@ - 1000 10000 @@ -133,11 +132,11 @@ - SELECT count() FROM system.numbers WHERE NOT ignore(toDateTime('2017-01-01 00:00:00') + number % 100000000 + rand() % 100000 AS t, {datetime_transform}(t, '{time_zone}')) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(toDateTime('2017-01-01 00:00:00') + number % 100000000 + rand() % 100000 AS t, {datetime_transform}(t, '{time_zone}')) - SELECT count() FROM system.numbers WHERE NOT ignore(toDate('2017-01-01') + number % 1000 + rand() % 10 AS t, {date_transform}(t)) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(toDate('2017-01-01') + number % 1000 + rand() % 10 AS t, {date_transform}(t)) - SELECT count() FROM system.numbers WHERE NOT ignore(toDateTime('2017-01-01 00:00:00') + number % 100000000 + rand() % 100000 AS t, {binary_function}(t, 1)) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(toDateTime('2017-01-01 00:00:00') + number % 100000000 + rand() % 100000 AS t, {binary_function}(t, 1)) - SELECT count() FROM system.numbers WHERE NOT ignore(toDateTime('2017-01-01 00:00:00') + number % 100000000 + rand() % 100000 AS t, toStartOfInterval(t, INTERVAL 1 month)) - \ No newline at end of file + SELECT count() FROM numbers(1000000) WHERE NOT ignore(toDateTime('2017-01-01 00:00:00') + number % 100000000 + rand() % 100000 AS t, toStartOfInterval(t, INTERVAL 1 month)) + diff --git a/dbms/tests/performance/entropy.xml b/dbms/tests/performance/entropy.xml index dcede345792..45c9ccb840d 100644 --- a/dbms/tests/performance/entropy.xml +++ b/dbms/tests/performance/entropy.xml @@ -10,7 +10,6 @@ 10000 - 5000 20000 diff --git a/dbms/tests/performance/float_formatting.xml b/dbms/tests/performance/float_formatting.xml index 0216e524735..aaf2fad0c93 100644 --- a/dbms/tests/performance/float_formatting.xml +++ b/dbms/tests/performance/float_formatting.xml @@ -1,5 +1,5 @@ - once + loop long @@ -9,7 +9,6 @@ 10000 - 5000 20000 @@ -54,5 +53,5 @@ - SELECT count() FROM system.numbers WHERE NOT ignore(toString({expr})) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(toString({expr})) diff --git a/dbms/tests/performance/float_parsing.xml b/dbms/tests/performance/float_parsing.xml index 81f30540dd1..e7779751fa4 100644 --- a/dbms/tests/performance/float_parsing.xml +++ b/dbms/tests/performance/float_parsing.xml @@ -1,5 +1,5 @@ - once + loop long @@ -9,7 +9,6 @@ 10000 - 5000 20000 @@ -33,5 +32,5 @@ - SELECT count() FROM system.numbers WHERE NOT ignore(toFloat64({expr})) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(toFloat64({expr})) diff --git a/dbms/tests/performance/format_date_time.xml b/dbms/tests/performance/format_date_time.xml index 0ecdb37734d..aa070c40ec5 100644 --- a/dbms/tests/performance/format_date_time.xml +++ b/dbms/tests/performance/format_date_time.xml @@ -1,12 +1,11 @@ - once + loop - 1000 10000 @@ -25,5 +24,5 @@ - SELECT count() FROM system.numbers WHERE NOT ignore(toDateTime('2017-01-01 00:00:00') + number % 100000000 + rand() % 100000 AS t, formatDateTime(t, '{format}')) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(toDateTime('2017-01-01 00:00:00') + number % 100000000 + rand() % 100000 AS t, formatDateTime(t, '{format}')) diff --git a/dbms/tests/performance/functions_coding.xml b/dbms/tests/performance/functions_coding.xml index 552f88be7fa..93e16a8a221 100644 --- a/dbms/tests/performance/functions_coding.xml +++ b/dbms/tests/performance/functions_coding.xml @@ -1,15 +1,14 @@ - once + loop - 5000 20000 - SELECT count() FROM system.numbers WHERE NOT ignore(MACNumToString(number)) - SELECT count() FROM system.numbers WHERE NOT ignore(MACStringToNum(MACNumToString(number))) - SELECT count() FROM system.numbers_mt WHERE NOT ignore(MACNumToString(rand64())) - SELECT count() FROM system.numbers_mt WHERE NOT ignore(MACStringToNum(MACNumToString(rand64()))) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(MACNumToString(number)) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(MACStringToNum(MACNumToString(number))) + SELECT count() FROM numbers_mt(10000000) WHERE NOT ignore(MACNumToString(rand64())) + SELECT count() FROM numbers_mt(10000000) WHERE NOT ignore(MACStringToNum(MACNumToString(rand64()))) From 420a7097689c17dd163b1ffa5541ccdfcf899a1f Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 28 Feb 2020 20:02:43 +0300 Subject: [PATCH 036/712] Change endless performance tests (4) --- dbms/tests/performance/functions_geo.xml | 15 +++++++-------- dbms/tests/performance/great_circle_dist.xml | 7 +++---- dbms/tests/performance/group_array_moving_sum.xml | 1 - dbms/tests/performance/h3.xml | 5 ++--- dbms/tests/performance/if_array_num.xml | 15 +++++++-------- dbms/tests/performance/if_array_string.xml | 15 +++++++-------- 6 files changed, 26 insertions(+), 32 deletions(-) diff --git a/dbms/tests/performance/functions_geo.xml b/dbms/tests/performance/functions_geo.xml index 1a13262d52f..a4233b2fe57 100644 --- a/dbms/tests/performance/functions_geo.xml +++ b/dbms/tests/performance/functions_geo.xml @@ -1,21 +1,20 @@ - once + loop - 1000 5000 - SELECT count() FROM system.numbers WHERE NOT ignore(geohashEncode((number % 150)*1.1 - 75, (number * 3.14 % 300)*1.1 - 150)) - SELECT count() FROM system.numbers WHERE NOT ignore(geohashDecode(toString(number % 1000000))) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(geohashEncode((number % 150)*1.1 - 75, (number * 3.14 % 300)*1.1 - 150)) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(geohashDecode(toString(number % 1000000))) - SELECT count() FROM system.numbers WHERE NOT ignore(geohashEncode(1.0/rand(), 2.0/rand())) - SELECT count() FROM system.numbers WHERE NOT ignore(geohashDecode(toString(rand() % 1000000))) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(geohashEncode(1.0/rand(), 2.0/rand())) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(geohashDecode(toString(rand() % 1000000))) - SELECT count() FROM system.numbers WHERE NOT ignore(geohashEncode(number + 91.0, number + 181.0)) - SELECT count() FROM system.numbers WHERE NOT ignore(geohashDecode(hex(number))) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(geohashEncode(number + 91.0, number + 181.0)) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(geohashDecode(hex(number))) diff --git a/dbms/tests/performance/great_circle_dist.xml b/dbms/tests/performance/great_circle_dist.xml index 99382543d60..3edfc2c8008 100644 --- a/dbms/tests/performance/great_circle_dist.xml +++ b/dbms/tests/performance/great_circle_dist.xml @@ -1,16 +1,15 @@ - once + loop - 1000 10000 - SELECT count() FROM system.numbers WHERE NOT ignore(greatCircleDistance((rand(1) % 360) * 1. - 180, (number % 150) * 1.2 - 90, (number % 360) + toFloat64(rand(2)) / 4294967296 - 180, (rand(3) % 180) * 1. - 90)) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(greatCircleDistance((rand(1) % 360) * 1. - 180, (number % 150) * 1.2 - 90, (number % 360) + toFloat64(rand(2)) / 4294967296 - 180, (rand(3) % 180) * 1. - 90)) - SELECT count() FROM system.numbers WHERE NOT ignore(greatCircleDistance(55. + toFloat64(rand(1)) / 4294967296, 37. + toFloat64(rand(2)) / 4294967296, 55. + toFloat64(rand(3)) / 4294967296, 37. + toFloat64(rand(4)) / 4294967296)) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(greatCircleDistance(55. + toFloat64(rand(1)) / 4294967296, 37. + toFloat64(rand(2)) / 4294967296, 55. + toFloat64(rand(3)) / 4294967296, 37. + toFloat64(rand(4)) / 4294967296)) diff --git a/dbms/tests/performance/group_array_moving_sum.xml b/dbms/tests/performance/group_array_moving_sum.xml index 504a8b133a1..45878d43e0e 100644 --- a/dbms/tests/performance/group_array_moving_sum.xml +++ b/dbms/tests/performance/group_array_moving_sum.xml @@ -7,7 +7,6 @@ 30000 - 6000 60000 diff --git a/dbms/tests/performance/h3.xml b/dbms/tests/performance/h3.xml index 7381f559a0f..3a6d5940d0d 100644 --- a/dbms/tests/performance/h3.xml +++ b/dbms/tests/performance/h3.xml @@ -1,14 +1,13 @@ - once + loop - 2000 10000 - SELECT count() FROM system.numbers WHERE NOT ignore(geoToH3(37.62 + rand(1) / 0x100000000, 55.75 + rand(2) / 0x100000000, 15)) + SELECT count() FROM numbers(100000) WHERE NOT ignore(geoToH3(37.62 + rand(1) / 0x100000000, 55.75 + rand(2) / 0x100000000, 15)) diff --git a/dbms/tests/performance/if_array_num.xml b/dbms/tests/performance/if_array_num.xml index 417b82a9d0c..375290e635c 100644 --- a/dbms/tests/performance/if_array_num.xml +++ b/dbms/tests/performance/if_array_num.xml @@ -1,18 +1,17 @@ - once + loop - 1000 10000 - SELECT count() FROM system.numbers WHERE NOT ignore(rand() % 2 ? [1, 2, 3] : [4, 5]) - SELECT count() FROM system.numbers WHERE NOT ignore(rand() % 2 ? [1, 2, 3] : materialize([4, 5])) - SELECT count() FROM system.numbers WHERE NOT ignore(rand() % 2 ? materialize([1, 2, 3]) : materialize([4, 5])) - SELECT count() FROM system.numbers WHERE NOT ignore(rand() % 2 ? [1, 2, 3] : [400, 500]) - SELECT count() FROM system.numbers WHERE NOT ignore(rand() % 2 ? [1, 2, 3] : materialize([400, 500])) - SELECT count() FROM system.numbers WHERE NOT ignore(rand() % 2 ? materialize([1, 2, 3]) : materialize([400, 500])) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(rand() % 2 ? [1, 2, 3] : [4, 5]) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(rand() % 2 ? [1, 2, 3] : materialize([4, 5])) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(rand() % 2 ? materialize([1, 2, 3]) : materialize([4, 5])) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(rand() % 2 ? [1, 2, 3] : [400, 500]) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(rand() % 2 ? [1, 2, 3] : materialize([400, 500])) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(rand() % 2 ? materialize([1, 2, 3]) : materialize([400, 500])) diff --git a/dbms/tests/performance/if_array_string.xml b/dbms/tests/performance/if_array_string.xml index e1d8485adc2..1f14393ee16 100644 --- a/dbms/tests/performance/if_array_string.xml +++ b/dbms/tests/performance/if_array_string.xml @@ -1,18 +1,17 @@ - once + loop - 1000 10000 - SELECT count() FROM system.numbers WHERE NOT ignore(rand() % 2 ? ['Hello', 'World'] : ['a', 'b', 'c']) - SELECT count() FROM system.numbers WHERE NOT ignore(rand() % 2 ? materialize(['Hello', 'World']) : ['a', 'b', 'c']) - SELECT count() FROM system.numbers WHERE NOT ignore(rand() % 2 ? ['Hello', 'World'] : materialize(['a', 'b', 'c'])) - SELECT count() FROM system.numbers WHERE NOT ignore(rand() % 2 ? materialize(['Hello', 'World']) : materialize(['a', 'b', 'c'])) - SELECT count() FROM system.numbers WHERE NOT ignore(rand() % 2 ? materialize(['', '']) : emptyArrayString()) - SELECT count() FROM system.numbers WHERE NOT ignore(rand() % 2 ? materialize(['https://github.com/ClickHouse/ClickHouse/pull/1070', 'https://www.google.ru/search?newwindow=1&site=&source=hp&q=zookeeper+wire+protocol+exists&oq=zookeeper+wire+protocol+exists&gs_l=psy-ab.3...330.6300.0.6687.33.28.0.0.0.0.386.4838.0j5j9j5.19.0....0...1.1.64.psy-ab..14.17.4448.0..0j35i39k1j0i131k1j0i22i30k1j0i19k1j33i21k1.r_3uFoNOrSU']) : emptyArrayString()) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(rand() % 2 ? ['Hello', 'World'] : ['a', 'b', 'c']) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(rand() % 2 ? materialize(['Hello', 'World']) : ['a', 'b', 'c']) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(rand() % 2 ? ['Hello', 'World'] : materialize(['a', 'b', 'c'])) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(rand() % 2 ? materialize(['Hello', 'World']) : materialize(['a', 'b', 'c'])) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(rand() % 2 ? materialize(['', '']) : emptyArrayString()) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(rand() % 2 ? materialize(['https://github.com/ClickHouse/ClickHouse/pull/1070', 'https://www.google.ru/search?newwindow=1&site=&source=hp&q=zookeeper+wire+protocol+exists&oq=zookeeper+wire+protocol+exists&gs_l=psy-ab.3...330.6300.0.6687.33.28.0.0.0.0.386.4838.0j5j9j5.19.0....0...1.1.64.psy-ab..14.17.4448.0..0j35i39k1j0i131k1j0i22i30k1j0i19k1j33i21k1.r_3uFoNOrSU']) : emptyArrayString()) From 98cd70ced382577f792109ad58128a95df0fb954 Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Fri, 28 Feb 2020 20:14:55 +0300 Subject: [PATCH 037/712] MergeTree for S3 more coverage. --- dbms/src/Disks/DiskLocal.cpp | 5 +++ dbms/src/Disks/DiskLocal.h | 2 + dbms/src/Disks/DiskMemory.cpp | 6 +++ dbms/src/Disks/DiskMemory.h | 2 + dbms/src/Disks/DiskS3.cpp | 6 +++ dbms/src/Disks/DiskS3.h | 2 + dbms/src/Disks/IDisk.h | 3 ++ .../Storages/MergeTree/DataPartsExchange.cpp | 9 +++-- .../Storages/MergeTree/IMergeTreeDataPart.cpp | 22 +++++----- .../Storages/MergeTree/IMergeTreeDataPart.h | 2 +- .../MergeTree/MergeTreeDataMergerMutator.cpp | 2 +- .../MergeTree/MergeTreeDataPartCompact.cpp | 40 +++++++++---------- .../MergeTree/MergeTreeDataPartWide.cpp | 14 +++---- .../MergeTree/MergeTreeDataWriter.cpp | 11 +++-- dbms/src/Storages/MergeTree/checkDataPart.cpp | 40 ++++++++++--------- dbms/src/Storages/MergeTree/checkDataPart.h | 1 + dbms/src/Storages/StorageMergeTree.cpp | 22 +++++----- 17 files changed, 108 insertions(+), 81 deletions(-) diff --git a/dbms/src/Disks/DiskLocal.cpp b/dbms/src/Disks/DiskLocal.cpp index 6ac305cd943..8c8bffc0f5a 100644 --- a/dbms/src/Disks/DiskLocal.cpp +++ b/dbms/src/Disks/DiskLocal.cpp @@ -239,6 +239,11 @@ void DiskLocal::removeRecursive(const String & path) Poco::File(disk_path + path).remove(true); } +void DiskLocal::listFiles(const String & path, std::vector & file_names) +{ + Poco::File(disk_path + path).list(file_names); +} + void DiskLocalReservation::update(UInt64 new_size) { diff --git a/dbms/src/Disks/DiskLocal.h b/dbms/src/Disks/DiskLocal.h index abccd3db232..03000f9e9a8 100644 --- a/dbms/src/Disks/DiskLocal.h +++ b/dbms/src/Disks/DiskLocal.h @@ -67,6 +67,8 @@ public: void copyFile(const String & from_path, const String & to_path) override; + void listFiles(const String & path, std::vector & file_names) override; + std::unique_ptr readFile( const String & path, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, diff --git a/dbms/src/Disks/DiskMemory.cpp b/dbms/src/Disks/DiskMemory.cpp index b939a78f230..660f67bb7ec 100644 --- a/dbms/src/Disks/DiskMemory.cpp +++ b/dbms/src/Disks/DiskMemory.cpp @@ -384,6 +384,12 @@ void DiskMemory::removeRecursive(const String & path) } } +void DiskMemory::listFiles(const String & path, std::vector & file_names) +{ + for (auto it = iterateDirectory(path); it->isValid(); it->next()) + file_names.push_back(it->name()); +} + using DiskMemoryPtr = std::shared_ptr; diff --git a/dbms/src/Disks/DiskMemory.h b/dbms/src/Disks/DiskMemory.h index 82a96bba5c8..2b9ca65de70 100644 --- a/dbms/src/Disks/DiskMemory.h +++ b/dbms/src/Disks/DiskMemory.h @@ -60,6 +60,8 @@ public: void copyFile(const String & from_path, const String & to_path) override; + void listFiles(const String & path, std::vector & file_names) override; + std::unique_ptr readFile( const String & path, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, diff --git a/dbms/src/Disks/DiskS3.cpp b/dbms/src/Disks/DiskS3.cpp index eefa15c4e71..68e7beab9c6 100644 --- a/dbms/src/Disks/DiskS3.cpp +++ b/dbms/src/Disks/DiskS3.cpp @@ -608,6 +608,12 @@ bool DiskS3::tryReserve(UInt64 bytes) return false; } +void DiskS3::listFiles(const String & path, std::vector & file_names) +{ + for (auto it = iterateDirectory(path); it->isValid(); it->next()) + file_names.push_back(it->name()); +} + DiskS3Reservation::~DiskS3Reservation() { diff --git a/dbms/src/Disks/DiskS3.h b/dbms/src/Disks/DiskS3.h index 85746ddd6bb..984544b018e 100644 --- a/dbms/src/Disks/DiskS3.h +++ b/dbms/src/Disks/DiskS3.h @@ -67,6 +67,8 @@ public: void copyFile(const String & from_path, const String & to_path) override; + void listFiles(const String & path, std::vector & file_names) override; + std::unique_ptr readFile( const String & path, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, diff --git a/dbms/src/Disks/IDisk.h b/dbms/src/Disks/IDisk.h index 1f8abe2af79..0f0447a7169 100644 --- a/dbms/src/Disks/IDisk.h +++ b/dbms/src/Disks/IDisk.h @@ -121,6 +121,9 @@ public: /// Copy the file from `from_path` to `to_path`. virtual void copyFile(const String & from_path, const String & to_path) = 0; + /// List files at `path` and add their names to `file_names` + virtual void listFiles(const String & path, std::vector & file_names) = 0; + /// Open the file for read and return ReadBufferFromFileBase object. virtual std::unique_ptr readFile( const String & path, diff --git a/dbms/src/Storages/MergeTree/DataPartsExchange.cpp b/dbms/src/Storages/MergeTree/DataPartsExchange.cpp index f648b7f3fb7..57c95584eb1 100644 --- a/dbms/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/dbms/src/Storages/MergeTree/DataPartsExchange.cpp @@ -122,16 +122,17 @@ void Service::processQuery(const Poco::Net::HTMLForm & params, ReadBuffer & /*bo { String file_name = it.first; - String path = part->getFullPath() + file_name; + auto disk = part->disk; + String path = part->getFullRelativePath() + file_name; - UInt64 size = Poco::File(path).getSize(); + UInt64 size = disk->getFileSize(path); writeStringBinary(it.first, out); writeBinary(size, out); - ReadBufferFromFile file_in(path); + auto file_in = disk->readFile(path); HashingWriteBuffer hashing_out(out); - copyData(file_in, hashing_out, blocker.getCounter()); + copyData(*file_in, hashing_out, blocker.getCounter()); if (blocker.isCancelled()) throw Exception("Transferring part to replica was cancelled", ErrorCodes::ABORTED); diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 4145dd1ff9d..3cf4fede415 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -244,10 +244,9 @@ void IMergeTreeDataPart::removeIfNeeded() { try { - std::string path = getFullPath(); + auto path = getFullRelativePath(); - Poco::File dir(path); - if (!dir.exists()) + if (!disk->exists(path)) return; if (is_temp) @@ -508,14 +507,14 @@ void IMergeTreeDataPart::loadChecksums(bool require) bytes_on_disk = checksums.getTotalSizeOnDisk(); } else - bytes_on_disk = calculateTotalSizeOnDisk(getFullPath()); + bytes_on_disk = calculateTotalSizeOnDisk(disk, getFullRelativePath()); } else { if (require) throw Exception("No checksums.txt in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); - bytes_on_disk = calculateTotalSizeOnDisk(getFullPath()); + bytes_on_disk = calculateTotalSizeOnDisk(disk, getFullRelativePath()); } } @@ -633,16 +632,15 @@ void IMergeTreeDataPart::loadColumns(bool require) column_name_to_position.emplace(column.name, pos++); } -UInt64 IMergeTreeDataPart::calculateTotalSizeOnDisk(const String & from) +UInt64 IMergeTreeDataPart::calculateTotalSizeOnDisk(const DiskPtr & disk_, const String & from) { - Poco::File cur(from); - if (cur.isFile()) - return cur.getSize(); + if (disk_->isFile(from)) + return disk_->getFileSize(from); std::vector files; - cur.list(files); + disk_->listFiles(from, files); UInt64 res = 0; for (const auto & file : files) - res += calculateTotalSizeOnDisk(from + file); + res += calculateTotalSizeOnDisk(disk_, from + file); return res; } @@ -661,8 +659,8 @@ void IMergeTreeDataPart::renameTo(const String & new_relative_path, bool remove_ { if (remove_new_dir_if_exists) { - /// TODO: List files. Names files; + disk->listFiles(to, files); LOG_WARNING(storage.log, "Part directory " << fullPath(disk, to) << " already exists" << " and contains " << files.size() << " files. Removing it."); diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h index 48ae37edb87..1a3b88bc212 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -306,7 +306,7 @@ public: /// Checks that .bin and .mrk files exist virtual bool hasColumnFiles(const String & /* column */, const IDataType & /* type */) const{ return false; } - static UInt64 calculateTotalSizeOnDisk(const String & from); + static UInt64 calculateTotalSizeOnDisk(const DiskPtr & disk_, const String & from); protected: /// Columns description. diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index f526f317a98..fe05d01947f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -1236,7 +1236,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor new_data_part->partition.assign(source_part->partition); new_data_part->minmax_idx = source_part->minmax_idx; new_data_part->modification_time = time(nullptr); - new_data_part->bytes_on_disk = MergeTreeData::DataPart::calculateTotalSizeOnDisk(new_data_part->getFullPath()); + new_data_part->bytes_on_disk = MergeTreeData::DataPart::calculateTotalSizeOnDisk(new_data_part->disk, new_data_part->getFullRelativePath()); } return new_data_part; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index 09947fce7d3..7e04f62a0f0 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -92,7 +92,7 @@ ColumnSize MergeTreeDataPartCompact::getTotalColumnsSize() const void MergeTreeDataPartCompact::loadIndexGranularity() { - String full_path = getFullPath(); + String full_path = getFullRelativePath(); if (columns.empty()) throw Exception("No columns in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); @@ -100,19 +100,19 @@ void MergeTreeDataPartCompact::loadIndexGranularity() if (!index_granularity_info.is_adaptive) throw Exception("MergeTreeDataPartCompact cannot be created with non-adaptive granulary.", ErrorCodes::NOT_IMPLEMENTED); - std::string marks_file_path = index_granularity_info.getMarksFilePath(full_path + "data"); - if (!Poco::File(marks_file_path).exists()) - throw Exception("Marks file '" + marks_file_path + "' doesn't exist", ErrorCodes::NO_FILE_IN_DATA_PART); + auto marks_file_path = index_granularity_info.getMarksFilePath(full_path + "data"); + if (!disk->exists(marks_file_path)) + throw Exception("Marks file '" + fullPath(disk, marks_file_path) + "' doesn't exist", ErrorCodes::NO_FILE_IN_DATA_PART); - size_t marks_file_size = Poco::File(marks_file_path).getSize(); + size_t marks_file_size = disk->getFileSize(marks_file_path); - ReadBufferFromFile buffer(marks_file_path, marks_file_size); - while (!buffer.eof()) + auto buffer = disk->readFile(marks_file_path, marks_file_size); + while (!buffer->eof()) { /// Skip offsets for columns - buffer.seek(columns.size() * sizeof(MarkInCompressedFile), SEEK_CUR); + buffer->seek(columns.size() * sizeof(MarkInCompressedFile), SEEK_CUR); size_t granularity; - readIntBinary(granularity, buffer); + readIntBinary(granularity, *buffer); index_granularity.appendMark(granularity); } @@ -187,7 +187,7 @@ NameToNameMap MergeTreeDataPartCompact::createRenameMapForAlter( void MergeTreeDataPartCompact::checkConsistency(bool require_part_metadata) const { checkConsistencyBase(); - String path = getFullPath(); + String path = getFullRelativePath(); String mrk_file_name = DATA_FILE_NAME + index_granularity_info.marks_file_extension; if (!checksums.empty()) @@ -199,34 +199,34 @@ void MergeTreeDataPartCompact::checkConsistency(bool require_part_metadata) cons if (require_part_metadata) { if (!checksums.files.count(mrk_file_name)) - throw Exception("No marks file checksum for column in part " + path, ErrorCodes::NO_FILE_IN_DATA_PART); + throw Exception("No marks file checksum for column in part " + fullPath(disk, path), ErrorCodes::NO_FILE_IN_DATA_PART); if (!checksums.files.count(DATA_FILE_NAME_WITH_EXTENSION)) - throw Exception("No data file checksum for in part " + path, ErrorCodes::NO_FILE_IN_DATA_PART); + throw Exception("No data file checksum for in part " + fullPath(disk, path), ErrorCodes::NO_FILE_IN_DATA_PART); } } else { { /// count.txt should be present even in non custom-partitioned parts - Poco::File file(path + "count.txt"); - if (!file.exists() || file.getSize() == 0) - throw Exception("Part " + path + " is broken: " + file.path() + " is empty", ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); + auto file_path = path + "count.txt"; + if (!disk->exists(file_path) || disk->getFileSize(file_path) == 0) + throw Exception("Part " + path + " is broken: " + fullPath(disk, file_path) + " is empty", ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); } /// Check that marks are nonempty and have the consistent size with columns number. - Poco::File file(path + mrk_file_name); + auto mrk_file_path = path + mrk_file_name; - if (file.exists()) + if (disk->exists(mrk_file_name)) { - UInt64 file_size = file.getSize(); + UInt64 file_size = disk->getFileSize(mrk_file_name); if (!file_size) - throw Exception("Part " + path + " is broken: " + file.path() + " is empty.", + throw Exception("Part " + path + " is broken: " + fullPath(disk, mrk_file_name) + " is empty.", ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); UInt64 expected_file_size = index_granularity_info.getMarkSizeInBytes(columns.size()) * index_granularity.getMarksCount(); if (expected_file_size != file_size) throw Exception( - "Part " + path + " is broken: bad size of marks file '" + file.path() + "': " + std::to_string(file_size) + ", must be: " + std::to_string(expected_file_size), + "Part " + path + " is broken: bad size of marks file '" + fullPath(disk, mrk_file_name) + "': " + std::to_string(file_size) + ", must be: " + std::to_string(expected_file_size), ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); } } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index 44d8715f3d1..29e2fc1a4cc 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -176,7 +176,7 @@ void MergeTreeDataPartWide::accumulateColumnSizes(ColumnToSize & column_to_size) void MergeTreeDataPartWide::checkConsistency(bool require_part_metadata) const { checkConsistencyBase(); - String path = getFullPath(); + String path = getFullRelativePath(); if (!checksums.empty()) { @@ -191,10 +191,10 @@ void MergeTreeDataPartWide::checkConsistency(bool require_part_metadata) const String mrk_file_name = file_name + index_granularity_info.marks_file_extension; String bin_file_name = file_name + ".bin"; if (!checksums.files.count(mrk_file_name)) - throw Exception("No " + mrk_file_name + " file checksum for column " + name_type.name + " in part " + path, + throw Exception("No " + mrk_file_name + " file checksum for column " + name_type.name + " in part " + fullPath(disk, path), ErrorCodes::NO_FILE_IN_DATA_PART); if (!checksums.files.count(bin_file_name)) - throw Exception("No " + bin_file_name + " file checksum for column " + name_type.name + " in part " + path, + throw Exception("No " + bin_file_name + " file checksum for column " + name_type.name + " in part " + fullPath(disk, path), ErrorCodes::NO_FILE_IN_DATA_PART); }, stream_path); } @@ -209,15 +209,15 @@ void MergeTreeDataPartWide::checkConsistency(bool require_part_metadata) const { name_type.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path) { - Poco::File file(IDataType::getFileNameForStream(name_type.name, substream_path) + index_granularity_info.marks_file_extension); + auto file_path = path + IDataType::getFileNameForStream(name_type.name, substream_path) + index_granularity_info.marks_file_extension; /// Missing file is Ok for case when new column was added. - if (file.exists()) + if (disk->exists(file_path)) { - UInt64 file_size = file.getSize(); + UInt64 file_size = disk->getFileSize(file_path); if (!file_size) - throw Exception("Part " + path + " is broken: " + file.path() + " is empty.", + throw Exception("Part " + path + " is broken: " + fullPath(disk, file_path) + " is empty.", ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); if (!marks_size) diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp index 48b011da28d..9e8d8f55fb4 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -244,16 +244,15 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa new_data_part->is_temp = true; /// The name could be non-unique in case of stale files from previous runs. - String full_path = new_data_part->getFullPath(); - Poco::File dir(full_path); + String full_path = new_data_part->getFullRelativePath(); - if (dir.exists()) + if (new_data_part->disk->exists(full_path)) { - LOG_WARNING(log, "Removing old temporary directory " + full_path); - dir.remove(true); + LOG_WARNING(log, "Removing old temporary directory " + fullPath(new_data_part->disk, full_path)); + new_data_part->disk->removeRecursive(full_path); } - dir.createDirectories(); + new_data_part->disk->createDirectories(full_path); /// If we need to calculate some columns to sort. if (data.hasSortingKey() || data.hasSkipIndices()) diff --git a/dbms/src/Storages/MergeTree/checkDataPart.cpp b/dbms/src/Storages/MergeTree/checkDataPart.cpp index f88732768b9..6195facc914 100644 --- a/dbms/src/Storages/MergeTree/checkDataPart.cpp +++ b/dbms/src/Storages/MergeTree/checkDataPart.cpp @@ -28,7 +28,8 @@ namespace ErrorCodes IMergeTreeDataPart::Checksums checkDataPart( - const String & full_path, + const DiskPtr & disk, + const String & full_relative_path, const NamesAndTypesList & columns_list, const MergeTreeDataPartType & part_type, bool require_checksums, @@ -42,16 +43,16 @@ IMergeTreeDataPart::Checksums checkDataPart( CurrentMetrics::Increment metric_increment{CurrentMetrics::ReplicatedChecks}; - String path = full_path; + String path = full_relative_path; if (!path.empty() && path.back() != '/') path += "/"; NamesAndTypesList columns_txt; { - ReadBufferFromFile buf(path + "columns.txt"); - columns_txt.readText(buf); - assertEOF(buf); + auto buf = disk->readFile(path + "columns.txt"); + columns_txt.readText(*buf); + assertEOF(*buf); } if (columns_txt != columns_list) @@ -62,10 +63,10 @@ IMergeTreeDataPart::Checksums checkDataPart( /// Real checksums based on contents of data. Must correspond to checksums.txt. If not - it means the data is broken. IMergeTreeDataPart::Checksums checksums_data; - auto checksum_compressed_file = [](const String & file_path) + auto checksum_compressed_file = [](const DiskPtr & disk_, const String & file_path) { - ReadBufferFromFile file_buf(file_path); - HashingReadBuffer compressed_hashing_buf(file_buf); + auto file_buf = disk_->readFile(file_path); + HashingReadBuffer compressed_hashing_buf(*file_buf); CompressedReadBuffer uncompressing_buf(compressed_hashing_buf); HashingReadBuffer uncompressed_hashing_buf(uncompressing_buf); @@ -80,7 +81,7 @@ IMergeTreeDataPart::Checksums checkDataPart( if (part_type == MergeTreeDataPartType::COMPACT) { const auto & file_name = MergeTreeDataPartCompact::DATA_FILE_NAME_WITH_EXTENSION; - checksums_data.files[file_name] = checksum_compressed_file(path + file_name); + checksums_data.files[file_name] = checksum_compressed_file(disk, path + file_name); } else if (part_type == MergeTreeDataPartType::WIDE) { @@ -89,7 +90,7 @@ IMergeTreeDataPart::Checksums checkDataPart( column.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path) { String file_name = IDataType::getFileNameForStream(column.name, substream_path) + ".bin"; - checksums_data.files[file_name] = checksum_compressed_file(path + file_name); + checksums_data.files[file_name] = checksum_compressed_file(disk, path + file_name); }, {}); } } @@ -99,14 +100,14 @@ IMergeTreeDataPart::Checksums checkDataPart( } Poco::DirectoryIterator dir_end; - for (Poco::DirectoryIterator dir_it(path); dir_it != dir_end; ++dir_it) + for (auto it = disk->iterateDirectory(path); it->isValid(); it->next()) { - const String & file_name = dir_it.name(); + const String & file_name = it->name(); auto checksum_it = checksums_data.files.find(file_name); if (checksum_it == checksums_data.files.end() && file_name != "checksums.txt" && file_name != "columns.txt") { - ReadBufferFromFile file_buf(dir_it->path()); - HashingReadBuffer hashing_buf(file_buf); + auto file_buf = disk->readFile(it->path()); + HashingReadBuffer hashing_buf(*file_buf); hashing_buf.tryIgnore(std::numeric_limits::max()); checksums_data.files[file_name] = IMergeTreeDataPart::Checksums::Checksum(hashing_buf.count(), hashing_buf.getHash()); } @@ -115,11 +116,11 @@ IMergeTreeDataPart::Checksums checkDataPart( /// Checksums from file checksums.txt. May be absent. If present, they are subsequently compared with the actual data checksums. IMergeTreeDataPart::Checksums checksums_txt; - if (require_checksums || Poco::File(path + "checksums.txt").exists()) + if (require_checksums || disk->exists(path + "checksums.txt")) { - ReadBufferFromFile buf(path + "checksums.txt"); - checksums_txt.read(buf); - assertEOF(buf); + auto buf = disk->readFile(path + "checksums.txt"); + checksums_txt.read(*buf); + assertEOF(*buf); } if (is_cancelled()) @@ -137,7 +138,8 @@ IMergeTreeDataPart::Checksums checkDataPart( std::function is_cancelled) { return checkDataPart( - data_part->getFullPath(), + data_part->disk, + data_part->getFullRelativePath(), data_part->getColumns(), data_part->getType(), require_checksums, diff --git a/dbms/src/Storages/MergeTree/checkDataPart.h b/dbms/src/Storages/MergeTree/checkDataPart.h index 2f263065b5d..f7483f6e5d9 100644 --- a/dbms/src/Storages/MergeTree/checkDataPart.h +++ b/dbms/src/Storages/MergeTree/checkDataPart.h @@ -13,6 +13,7 @@ IMergeTreeDataPart::Checksums checkDataPart( std::function is_cancelled = []{ return false; }); IMergeTreeDataPart::Checksums checkDataPart( + const DiskPtr & disk, const String & full_path, const NamesAndTypesList & columns_list, const MergeTreeDataPartType & part_type, diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 192b4194b7a..1b1bad37a8e 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -565,7 +565,7 @@ void StorageMergeTree::loadMutations() { if (startsWith(it->name(), "mutation_")) { - MergeTreeMutationEntry entry(disk, it->path(), it->name()); + MergeTreeMutationEntry entry(disk, path, it->name()); Int64 block_number = entry.block_number; auto insertion = current_mutations_by_id.emplace(it->name(), std::move(entry)); current_mutations_by_version.emplace(block_number, insertion.first->second); @@ -1325,26 +1325,26 @@ CheckResults StorageMergeTree::checkData(const ASTPtr & query, const Context & c for (auto & part : data_parts) { - String full_part_path = part->getFullPath(); + auto disk = part->disk; + String part_path = part->getFullRelativePath(); /// If the checksums file is not present, calculate the checksums and write them to disk. - String checksums_path = full_part_path + "checksums.txt"; - String tmp_checksums_path = full_part_path + "checksums.txt.tmp"; - if (!Poco::File(checksums_path).exists()) + String checksums_path = part_path + "checksums.txt"; + String tmp_checksums_path = part_path + "checksums.txt.tmp"; + if (!disk->exists(checksums_path)) { try { auto calculated_checksums = checkDataPart(part, false); calculated_checksums.checkEqual(part->checksums, true); - WriteBufferFromFile out(tmp_checksums_path, 4096); - part->checksums.write(out); - Poco::File(tmp_checksums_path).renameTo(checksums_path); + auto out = disk->writeFile(tmp_checksums_path, 4096); + part->checksums.write(*out); + disk->moveFile(tmp_checksums_path, checksums_path); results.emplace_back(part->name, true, "Checksums recounted and written to disk."); } catch (const Exception & ex) { - Poco::File tmp_file(tmp_checksums_path); - if (tmp_file.exists()) - tmp_file.remove(); + if (disk->exists(tmp_checksums_path)) + disk->remove(tmp_checksums_path); results.emplace_back(part->name, false, "Check of part finished with error: '" + ex.message() + "'"); From 72d2e56c0aea08f6fee4bfded59a453de8c26aa8 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 28 Feb 2020 22:44:15 +0300 Subject: [PATCH 038/712] fixup --- dbms/tests/performance/array_element.xml | 1 - dbms/tests/performance/array_join.xml | 1 - dbms/tests/performance/base64.xml | 1 - dbms/tests/performance/base64_hits.xml | 1 - dbms/tests/performance/bitCount.xml | 1 - 5 files changed, 5 deletions(-) diff --git a/dbms/tests/performance/array_element.xml b/dbms/tests/performance/array_element.xml index 2d05f9dcf51..1040c33ddbf 100644 --- a/dbms/tests/performance/array_element.xml +++ b/dbms/tests/performance/array_element.xml @@ -3,7 +3,6 @@ - 2000 10000 diff --git a/dbms/tests/performance/array_join.xml b/dbms/tests/performance/array_join.xml index c30039588e0..7574c5f8b15 100644 --- a/dbms/tests/performance/array_join.xml +++ b/dbms/tests/performance/array_join.xml @@ -3,7 +3,6 @@ - 5000 10000 diff --git a/dbms/tests/performance/base64.xml b/dbms/tests/performance/base64.xml index 3175c7811bf..8860ec232e8 100644 --- a/dbms/tests/performance/base64.xml +++ b/dbms/tests/performance/base64.xml @@ -6,7 +6,6 @@ 10000 - 5000 20000 diff --git a/dbms/tests/performance/base64_hits.xml b/dbms/tests/performance/base64_hits.xml index 63916dcee4e..5002edccaea 100644 --- a/dbms/tests/performance/base64_hits.xml +++ b/dbms/tests/performance/base64_hits.xml @@ -10,7 +10,6 @@ 10000 - 5000 20000 diff --git a/dbms/tests/performance/bitCount.xml b/dbms/tests/performance/bitCount.xml index 8a58f501180..c714610f351 100644 --- a/dbms/tests/performance/bitCount.xml +++ b/dbms/tests/performance/bitCount.xml @@ -4,7 +4,6 @@ - 2000 10000 From 5068346e59d7658aea4607285eccba25e670e54c Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Mon, 2 Mar 2020 18:08:35 +0300 Subject: [PATCH 039/712] fixup --- .../performance/group_array_moving_sum.xml | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/dbms/tests/performance/group_array_moving_sum.xml b/dbms/tests/performance/group_array_moving_sum.xml index 45878d43e0e..fadd225e1a2 100644 --- a/dbms/tests/performance/group_array_moving_sum.xml +++ b/dbms/tests/performance/group_array_moving_sum.xml @@ -18,19 +18,19 @@ INSERT INTO moving_sum_1m SELECT number%100, rand() from numbers(1000000) INSERT INTO moving_sum_10m SELECT number%100, rand() from numbers(10000000) - SELECT k,groupArrayMovingSum(10)(v) FROM moving_sum_1m GROUP BY k - SELECT k,groupArrayMovingSum(10)(v) FROM moving_sum_1m WHERE k in (49, 50, 51) GROUP BY k - SELECT k,groupArrayMovingSum(1000)(v) FROM moving_sum_1m GROUP BY k - SELECT k,groupArrayMovingSum(1000)(v) FROM moving_sum_1m WHERE k in (49, 50, 51) GROUP BY k - SELECT k,groupArrayMovingSum(10000)(v) FROM moving_sum_1m GROUP BY k - SELECT k,groupArrayMovingSum(10000)(v) FROM moving_sum_1m WHERE k in (49, 50, 51) GROUP BY k + SELECT k,groupArrayMovingSum(10)(v) FROM moving_sum_1m GROUP BY k FORMAT Null + SELECT k,groupArrayMovingSum(10)(v) FROM moving_sum_1m WHERE k in (49, 50, 51) GROUP BY k FORMAT Null + SELECT k,groupArrayMovingSum(1000)(v) FROM moving_sum_1m GROUP BY k FORMAT Null + SELECT k,groupArrayMovingSum(1000)(v) FROM moving_sum_1m WHERE k in (49, 50, 51) GROUP BY k FORMAT Null + SELECT k,groupArrayMovingSum(10000)(v) FROM moving_sum_1m GROUP BY k FORMAT Null + SELECT k,groupArrayMovingSum(10000)(v) FROM moving_sum_1m WHERE k in (49, 50, 51) GROUP BY k FORMAT Null - SELECT k,groupArrayMovingSum(10)(v) FROM moving_sum_10m GROUP BY k - SELECT k,groupArrayMovingSum(10)(v) FROM moving_sum_10m WHERE k in (49, 50, 51) GROUP BY k - SELECT k,groupArrayMovingSum(1000)(v) FROM moving_sum_10m GROUP BY k - SELECT k,groupArrayMovingSum(1000)(v) FROM moving_sum_10m WHERE k in (49, 50, 51) GROUP BY k - SELECT k,groupArrayMovingSum(10000)(v) FROM moving_sum_10m GROUP BY k - SELECT k,groupArrayMovingSum(10000)(v) FROM moving_sum_10m WHERE k in (49, 50, 51) GROUP BY k + SELECT k,groupArrayMovingSum(10)(v) FROM moving_sum_10m GROUP BY k FORMAT Null + SELECT k,groupArrayMovingSum(10)(v) FROM moving_sum_10m WHERE k in (49, 50, 51) GROUP BY k FORMAT Null + SELECT k,groupArrayMovingSum(1000)(v) FROM moving_sum_10m GROUP BY k FORMAT Null + SELECT k,groupArrayMovingSum(1000)(v) FROM moving_sum_10m WHERE k in (49, 50, 51) GROUP BY k FORMAT Null + SELECT k,groupArrayMovingSum(10000)(v) FROM moving_sum_10m GROUP BY k FORMAT Null + SELECT k,groupArrayMovingSum(10000)(v) FROM moving_sum_10m WHERE k in (49, 50, 51) GROUP BY k FORMAT Null DROP TABLE IF EXISTS moving_sum_10m DROP TABLE IF EXISTS moving_sum_1m From 7e542053fc01fc666e4ae77ad31467834e415bf7 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Mon, 2 Mar 2020 18:34:43 +0300 Subject: [PATCH 040/712] fixup --- dbms/tests/performance/consistent_hashes.xml | 4 +++- dbms/tests/performance/cryptographic_hashes.xml | 15 ++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/dbms/tests/performance/consistent_hashes.xml b/dbms/tests/performance/consistent_hashes.xml index aee232ddce6..f3858e16a28 100644 --- a/dbms/tests/performance/consistent_hashes.xml +++ b/dbms/tests/performance/consistent_hashes.xml @@ -27,5 +27,7 @@ SELECT {hash_func}(number, {buckets}) FROM numbers(1000000) - SELECT sumburConsistentHash(toUInt32(number), {buckets}) FROM numbers(1000000) + + + SELECT sumburConsistentHash(toUInt32(number), 2) FROM numbers(1000000) diff --git a/dbms/tests/performance/cryptographic_hashes.xml b/dbms/tests/performance/cryptographic_hashes.xml index 71c37ed9f8d..7bafb25f299 100644 --- a/dbms/tests/performance/cryptographic_hashes.xml +++ b/dbms/tests/performance/cryptographic_hashes.xml @@ -29,6 +29,11 @@ materialize('') toString(1000000000+number) + + + + string_slow + materialize('Lorem ipsum dolor sit amet, consectetur adipiscing elit. Mauris sollicitudin nisi ac erat mollis dapibus. Maecenas leo purus, bibendum eu erat eget, iaculis molestie tortor. Phasellus maximus odio nec mauris ultrices dictum. Morbi efficitur nisl eget congue mollis. Vestibulum pharetra diam vitae urna interdum, eget ultricies justo sollicitudin. Nunc sit amet purus id leo tempus dignissim. Donec ac lacus ut orci tempus scelerisque quis ultricies nibh. Nullam lobortis, erat ac ullamcorper interdum, odio nisl elementum quam, ut malesuada massa nunc eget quam. Nam suscipit neque quis sapien ultricies imperdiet. Maecenas augue libero, finibus tristique sagittis et, semper nec arcu. Morbi non tortor ultrices, sollicitudin justo sed, accumsan ligula. Nullam at ipsum in nibh auctor ullamcorper. Nullam laoreet neque id lorem condimentum tincidunt. Nullam vel orci nibh. Ut sit amet sem faucibus, fringilla orci at, lacinia enim. Mauris imperdiet ex id scelerisque eleifend. Ut tincidunt massa nibh, viverra pharetra metus') @@ -39,7 +44,15 @@ numbers_mt(10000000) + + table_slow + + numbers(100000) + numbers_mt(1000000) + + - SELECT ignore({crypto_hash_func}({string})) FROM {table} + SELECT ignore({crypto_hash_func}({string})) FROM {table} FORMAT Null + SELECT ignore({crypto_hash_func}({string_slow})) FROM {table_slow} FORMAT Null From 42d608317223a87d7b6459c91868fc4fc0a66c30 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Mon, 2 Mar 2020 18:37:44 +0300 Subject: [PATCH 041/712] fixup --- dbms/tests/performance/base64.xml | 4 ++-- dbms/tests/performance/bitCount.xml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dbms/tests/performance/base64.xml b/dbms/tests/performance/base64.xml index 8860ec232e8..dbf8e0dc981 100644 --- a/dbms/tests/performance/base64.xml +++ b/dbms/tests/performance/base64.xml @@ -23,8 +23,8 @@ table - numbers(1000000) - numbers_mt(10000000) + numbers(10000000) + numbers_mt(100000000) diff --git a/dbms/tests/performance/bitCount.xml b/dbms/tests/performance/bitCount.xml index c714610f351..60901885dbd 100644 --- a/dbms/tests/performance/bitCount.xml +++ b/dbms/tests/performance/bitCount.xml @@ -22,5 +22,5 @@ - SELECT bitCount({expr}) FROM numbers(1000000) + SELECT bitCount({expr}) FROM numbers(1000000) FORMAT Null From 0eb47b3415e355add689e56b28478bc6f9490765 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Mon, 2 Mar 2020 19:50:18 +0300 Subject: [PATCH 042/712] Change endless performance tests (5) --- dbms/tests/performance/if_string_const.xml | 12 +++++------- dbms/tests/performance/if_to_multiif.xml | 13 ++++++------- dbms/tests/performance/information_value.xml | 7 +++---- dbms/tests/performance/linear_regression.xml | 1 - .../performance/random_printable_ascii.xml | 17 ++++++++--------- 5 files changed, 22 insertions(+), 28 deletions(-) diff --git a/dbms/tests/performance/if_string_const.xml b/dbms/tests/performance/if_string_const.xml index 15a281685ae..5b06440473f 100644 --- a/dbms/tests/performance/if_string_const.xml +++ b/dbms/tests/performance/if_string_const.xml @@ -1,16 +1,14 @@ - once + loop - 1000 10000 - - SELECT count() FROM system.numbers WHERE NOT ignore(rand() % 2 ? 'hello' : 'world') - SELECT count() FROM system.numbers WHERE NOT ignore(rand() % 2 ? 'hello' : '') - SELECT count() FROM system.numbers WHERE NOT ignore(rand() % 2 ? toFixedString('hello', 5) : toFixedString('world', 5)) - SELECT count() FROM system.numbers WHERE NOT ignore(rand() % 2 ? '' : toFixedString('world', 5)) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(rand() % 2 ? 'hello' : 'world') + SELECT count() FROM numbers(1000000) WHERE NOT ignore(rand() % 2 ? 'hello' : '') + SELECT count() FROM numbers(1000000) WHERE NOT ignore(rand() % 2 ? toFixedString('hello', 5) : toFixedString('world', 5)) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(rand() % 2 ? '' : toFixedString('world', 5)) diff --git a/dbms/tests/performance/if_to_multiif.xml b/dbms/tests/performance/if_to_multiif.xml index 54d4b8ba842..7dee667a1bd 100644 --- a/dbms/tests/performance/if_to_multiif.xml +++ b/dbms/tests/performance/if_to_multiif.xml @@ -1,19 +1,18 @@ - once + loop - 1000 10000 - - - - + + + + diff --git a/dbms/tests/performance/information_value.xml b/dbms/tests/performance/information_value.xml index ed054eda40d..f5b73a18abc 100644 --- a/dbms/tests/performance/information_value.xml +++ b/dbms/tests/performance/information_value.xml @@ -2,7 +2,7 @@ loop - test.hits + hits_100m_single @@ -10,12 +10,11 @@ 10000 - 5000 20000 - SELECT categoricalInformationValue(Age < 15, IsMobile) - SELECT categoricalInformationValue(Age < 15, Age >= 15 and Age < 30, Age >= 30 and Age < 45, Age >= 45 and Age < 60, Age >= 60, IsMobile) + SELECT categoricalInformationValue(Age < 15, IsMobile) from hits_100m_single + SELECT categoricalInformationValue(Age < 15, Age >= 15 and Age < 30, Age >= 30 and Age < 45, Age >= 45 and Age < 60, Age >= 60, IsMobile) from hits_100m_single diff --git a/dbms/tests/performance/linear_regression.xml b/dbms/tests/performance/linear_regression.xml index 50634b6a60a..0b4892f71ec 100644 --- a/dbms/tests/performance/linear_regression.xml +++ b/dbms/tests/performance/linear_regression.xml @@ -3,7 +3,6 @@ - 3000 10000 diff --git a/dbms/tests/performance/random_printable_ascii.xml b/dbms/tests/performance/random_printable_ascii.xml index b37469c0aee..5fca705464e 100644 --- a/dbms/tests/performance/random_printable_ascii.xml +++ b/dbms/tests/performance/random_printable_ascii.xml @@ -1,19 +1,18 @@ - once + loop - 4000 10000 - SELECT count() FROM system.numbers WHERE NOT ignore(randomPrintableASCII(10)) - SELECT count() FROM system.numbers WHERE NOT ignore(randomPrintableASCII(100)) - SELECT count() FROM system.numbers WHERE NOT ignore(randomPrintableASCII(1000)) - SELECT count() FROM system.numbers WHERE NOT ignore(randomPrintableASCII(10000)) - SELECT count() FROM system.numbers WHERE NOT ignore(randomPrintableASCII(rand() % 10)) - SELECT count() FROM system.numbers WHERE NOT ignore(randomPrintableASCII(rand() % 100)) - SELECT count() FROM system.numbers WHERE NOT ignore(randomPrintableASCII(rand() % 1000)) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(randomPrintableASCII(10)) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(randomPrintableASCII(100)) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(randomPrintableASCII(1000)) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(randomPrintableASCII(10000)) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(randomPrintableASCII(rand() % 10)) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(randomPrintableASCII(rand() % 100)) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(randomPrintableASCII(rand() % 1000)) From 55a08a278400fcc4816c74ec27a9d961006ab58c Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Mon, 2 Mar 2020 20:05:41 +0300 Subject: [PATCH 043/712] Change endless performance tests (6) --- dbms/tests/performance/range.xml | 7 ++-- dbms/tests/performance/read_hits_with_aio.xml | 12 +++---- dbms/tests/performance/right.xml | 1 - dbms/tests/performance/round_down.xml | 13 ++++---- dbms/tests/performance/round_methods.xml | 33 +++++++++---------- dbms/tests/performance/set.xml | 7 ++-- dbms/tests/performance/simple_join_query.xml | 1 - 7 files changed, 34 insertions(+), 40 deletions(-) diff --git a/dbms/tests/performance/range.xml b/dbms/tests/performance/range.xml index 48463b535ef..ee61a22b0cf 100644 --- a/dbms/tests/performance/range.xml +++ b/dbms/tests/performance/range.xml @@ -1,14 +1,13 @@ - once + loop - 5000 10000 - SELECT count() FROM (SELECT range(number % 100) FROM system.numbers limit 10000000) - SELECT count() FROM (SELECT range(0, number % 100, 1) FROM system.numbers limit 10000000) + SELECT range(number % 100) FROM numbers(10000000) FORMAT Null + SELECT range(0, number % 100, 1) FROM numbers(10000000) FORMAT Null diff --git a/dbms/tests/performance/read_hits_with_aio.xml b/dbms/tests/performance/read_hits_with_aio.xml index 573da5a4c70..9b70dfaa421 100644 --- a/dbms/tests/performance/read_hits_with_aio.xml +++ b/dbms/tests/performance/read_hits_with_aio.xml @@ -1,20 +1,20 @@ + loop + - 5000 30000 - once - hits_1000m_single + hits_100m_single -SELECT count() FROM hits_1000m_single where UserID=1234567890 SETTINGS max_threads = 1, min_bytes_to_use_direct_io = 1, max_read_buffer_size = 10485760; +SELECT count() FROM hits_100m_single where UserID=1234567890 SETTINGS max_threads = 1, min_bytes_to_use_direct_io = 1, max_read_buffer_size = 10485760; SELECT count() FROM hits_1000m_single where EventDate between toDate('2013-07-10') and toDate('2013-07-16') and UserID=123 SETTINGS max_threads = 1, min_bytes_to_use_direct_io = 1, max_read_buffer_size = 10485760; -SELECT count() FROM hits_1000m_single where UserID=1234567890 SETTINGS max_threads = 1, min_bytes_to_use_direct_io = 0, max_read_buffer_size = 10485760; -SELECT count() FROM hits_1000m_single where EventDate between toDate('2013-07-10') and toDate('2013-07-16') and UserID=123 SETTINGS max_threads = 1, min_bytes_to_use_direct_io = 0, max_read_buffer_size = 10485760; +SELECT count() FROM hits_100m_single where UserID=1234567890 SETTINGS max_threads = 1, min_bytes_to_use_direct_io = 0, max_read_buffer_size = 10485760; +SELECT count() FROM hits_100m_single where EventDate between toDate('2013-07-10') and toDate('2013-07-16') and UserID=123 SETTINGS max_threads = 1, min_bytes_to_use_direct_io = 0, max_read_buffer_size = 10485760; diff --git a/dbms/tests/performance/right.xml b/dbms/tests/performance/right.xml index 06d4bdaa93f..55095d251f7 100644 --- a/dbms/tests/performance/right.xml +++ b/dbms/tests/performance/right.xml @@ -10,7 +10,6 @@ 10000 - 5000 20000 diff --git a/dbms/tests/performance/round_down.xml b/dbms/tests/performance/round_down.xml index 5275d69ad84..b14b5a9fb2a 100644 --- a/dbms/tests/performance/round_down.xml +++ b/dbms/tests/performance/round_down.xml @@ -1,20 +1,19 @@ - once + loop 10000 - 5000 20000 - SELECT count() FROM system.numbers WHERE NOT ignore(roundDuration(rand() % 65536)) - SELECT count() FROM system.numbers WHERE NOT ignore(roundDown(rand() % 65536, [0, 1, 10, 30, 60, 120, 180, 240, 300, 600, 1200, 1800, 3600, 7200, 18000, 36000])) - SELECT count() FROM system.numbers WHERE NOT ignore(roundAge(rand() % 100)) - SELECT count() FROM system.numbers WHERE NOT ignore(roundDown(rand() % 100, [0, 1, 18, 25, 35, 45, 55])) - SELECT count() FROM system.numbers WHERE NOT ignore(roundDown(rand() % 65536, (SELECT groupArray(number) FROM numbers(65536)))) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(roundDuration(rand() % 65536)) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(roundDown(rand() % 65536, [0, 1, 10, 30, 60, 120, 180, 240, 300, 600, 1200, 1800, 3600, 7200, 18000, 36000])) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(roundAge(rand() % 100)) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(roundDown(rand() % 100, [0, 1, 18, 25, 35, 45, 55])) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(roundDown(rand() % 65536, (SELECT groupArray(number) FROM numbers(65536)))) diff --git a/dbms/tests/performance/round_methods.xml b/dbms/tests/performance/round_methods.xml index b80a8977c33..54bd1e4af17 100644 --- a/dbms/tests/performance/round_methods.xml +++ b/dbms/tests/performance/round_methods.xml @@ -1,32 +1,31 @@ - once + loop 10000 - 5000 20000 - SELECT count() FROM system.numbers WHERE NOT ignore(round(toInt64(number), -2)) - SELECT count() FROM system.numbers WHERE NOT ignore(roundBankers(toInt64(number), -2)) - SELECT count() FROM system.numbers WHERE NOT ignore(floor(toInt64(number), -2)) - SELECT count() FROM system.numbers WHERE NOT ignore(ceil(toInt64(number), -2)) - SELECT count() FROM system.numbers WHERE NOT ignore(trunc(toInt64(number), -2)) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(round(toInt64(number), -2)) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(roundBankers(toInt64(number), -2)) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(floor(toInt64(number), -2)) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(ceil(toInt64(number), -2)) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(trunc(toInt64(number), -2)) - SELECT count() FROM system.numbers WHERE NOT ignore(round(toFloat64(number), -2)) - SELECT count() FROM system.numbers WHERE NOT ignore(roundBankers(toFloat64(number), -2)) - SELECT count() FROM system.numbers WHERE NOT ignore(floor(toFloat64(number), -2)) - SELECT count() FROM system.numbers WHERE NOT ignore(ceil(toFloat64(number), -2)) - SELECT count() FROM system.numbers WHERE NOT ignore(trunc(toFloat64(number), -2)) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(round(toFloat64(number), -2)) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(roundBankers(toFloat64(number), -2)) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(floor(toFloat64(number), -2)) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(ceil(toFloat64(number), -2)) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(trunc(toFloat64(number), -2)) - SELECT count() FROM system.numbers WHERE NOT ignore(round(toDecimal128(number, 0), -2)) - SELECT count() FROM system.numbers WHERE NOT ignore(roundBankers(toDecimal128(number, 0), -2)) - SELECT count() FROM system.numbers WHERE NOT ignore(floor(toDecimal128(number, 0), -2)) - SELECT count() FROM system.numbers WHERE NOT ignore(ceil(toDecimal128(number, 0), -2)) - SELECT count() FROM system.numbers WHERE NOT ignore(trunc(toDecimal128(number, 0), -2)) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(round(toDecimal128(number, 0), -2)) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(roundBankers(toDecimal128(number, 0), -2)) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(floor(toDecimal128(number, 0), -2)) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(ceil(toDecimal128(number, 0), -2)) + SELECT count() FROM numbers(1000000) WHERE NOT ignore(trunc(toDecimal128(number, 0), -2)) diff --git a/dbms/tests/performance/set.xml b/dbms/tests/performance/set.xml index 7f3ee4fd4c1..c142730a560 100644 --- a/dbms/tests/performance/set.xml +++ b/dbms/tests/performance/set.xml @@ -1,5 +1,5 @@ - once + loop long @@ -9,7 +9,6 @@ 10000 - 5000 20000 @@ -19,8 +18,8 @@ table - system.numbers - system.numbers_mt + numbers(1000000) + numbers_mt(10000000) diff --git a/dbms/tests/performance/simple_join_query.xml b/dbms/tests/performance/simple_join_query.xml index 1f6d6ba74d6..c13dd70a777 100644 --- a/dbms/tests/performance/simple_join_query.xml +++ b/dbms/tests/performance/simple_join_query.xml @@ -6,7 +6,6 @@ 30000 - 6000 60000 From b9a52f18f1cf115f95f9d4b421146ac1c2802a92 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Mon, 2 Mar 2020 20:13:49 +0300 Subject: [PATCH 044/712] fixup --- dbms/tests/performance/date_time.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dbms/tests/performance/date_time.xml b/dbms/tests/performance/date_time.xml index e099b468560..4e9cd2c4abd 100644 --- a/dbms/tests/performance/date_time.xml +++ b/dbms/tests/performance/date_time.xml @@ -132,11 +132,11 @@ - SELECT count() FROM numbers(1000000) WHERE NOT ignore(toDateTime('2017-01-01 00:00:00') + number % 100000000 + rand() % 100000 AS t, {datetime_transform}(t, '{time_zone}')) + SELECT count() FROM numbers(100000) WHERE NOT ignore(toDateTime('2017-01-01 00:00:00') + number % 100000000 + rand() % 100000 AS t, {datetime_transform}(t, '{time_zone}')) - SELECT count() FROM numbers(1000000) WHERE NOT ignore(toDate('2017-01-01') + number % 1000 + rand() % 10 AS t, {date_transform}(t)) + SELECT count() FROM numbers(100000) WHERE NOT ignore(toDate('2017-01-01') + number % 1000 + rand() % 10 AS t, {date_transform}(t)) - SELECT count() FROM numbers(1000000) WHERE NOT ignore(toDateTime('2017-01-01 00:00:00') + number % 100000000 + rand() % 100000 AS t, {binary_function}(t, 1)) + SELECT count() FROM numbers(100000) WHERE NOT ignore(toDateTime('2017-01-01 00:00:00') + number % 100000000 + rand() % 100000 AS t, {binary_function}(t, 1)) - SELECT count() FROM numbers(1000000) WHERE NOT ignore(toDateTime('2017-01-01 00:00:00') + number % 100000000 + rand() % 100000 AS t, toStartOfInterval(t, INTERVAL 1 month)) + SELECT count() FROM numbers(100000) WHERE NOT ignore(toDateTime('2017-01-01 00:00:00') + number % 100000000 + rand() % 100000 AS t, toStartOfInterval(t, INTERVAL 1 month)) From fa27ecf35388db93648a39df318b4b9916962b19 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 2 Mar 2020 20:25:36 +0300 Subject: [PATCH 045/712] do not allow custom database for system tables --- dbms/src/Interpreters/Context.cpp | 14 +++++++------- dbms/src/Interpreters/SystemLog.cpp | 12 ++++++++++-- dbms/src/Interpreters/SystemLog.h | 3 +-- .../0_stateless/01069_database_memory.reference | 2 +- 4 files changed, 19 insertions(+), 12 deletions(-) diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index d98c6044464..f2bf887ac6a 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -1557,7 +1557,7 @@ std::shared_ptr Context::getQueryLog() { auto lock = getLock(); - if (!shared->system_logs || !shared->system_logs->query_log) + if (!shared->system_logs) return {}; return shared->system_logs->query_log; @@ -1568,7 +1568,7 @@ std::shared_ptr Context::getQueryThreadLog() { auto lock = getLock(); - if (!shared->system_logs || !shared->system_logs->query_thread_log) + if (!shared->system_logs) return {}; return shared->system_logs->query_thread_log; @@ -1580,13 +1580,13 @@ std::shared_ptr Context::getPartLog(const String & part_database) auto lock = getLock(); /// No part log or system logs are shutting down. - if (!shared->system_logs || !shared->system_logs->part_log) + if (!shared->system_logs) return {}; /// Will not log operations on system tables (including part_log itself). /// It doesn't make sense and not allow to destruct PartLog correctly due to infinite logging and flushing, /// and also make troubles on startup. - if (part_database == shared->system_logs->part_log_database) + if (part_database == DatabaseCatalog::SYSTEM_DATABASE) return {}; return shared->system_logs->part_log; @@ -1597,7 +1597,7 @@ std::shared_ptr Context::getTraceLog() { auto lock = getLock(); - if (!shared->system_logs || !shared->system_logs->trace_log) + if (!shared->system_logs) return {}; return shared->system_logs->trace_log; @@ -1608,7 +1608,7 @@ std::shared_ptr Context::getTextLog() { auto lock = getLock(); - if (!shared->system_logs || !shared->system_logs->text_log) + if (!shared->system_logs) return {}; return shared->system_logs->text_log; @@ -1619,7 +1619,7 @@ std::shared_ptr Context::getMetricLog() { auto lock = getLock(); - if (!shared->system_logs || !shared->system_logs->metric_log) + if (!shared->system_logs) return {}; return shared->system_logs->metric_log; diff --git a/dbms/src/Interpreters/SystemLog.cpp b/dbms/src/Interpreters/SystemLog.cpp index d347488ea50..e307394e361 100644 --- a/dbms/src/Interpreters/SystemLog.cpp +++ b/dbms/src/Interpreters/SystemLog.cpp @@ -7,6 +7,7 @@ #include #include +#include namespace DB @@ -37,6 +38,15 @@ std::shared_ptr createSystemLog( String database = config.getString(config_prefix + ".database", default_database_name); String table = config.getString(config_prefix + ".table", default_table_name); + if (database != default_database_name) + { + /// System tables must be loaded before other tables, but loading order is undefined for all databases except `system` + LOG_ERROR(&Logger::get("SystemLog"), "Custom database name for a system table specified in config. " + "Table `" << table << "` will be created in `system` database " + "instead of `" << database << "`"); + database = default_database_name; + } + String engine; if (config.has(config_prefix + ".engine")) { @@ -72,8 +82,6 @@ SystemLogs::SystemLogs(Context & global_context, const Poco::Util::AbstractConfi size_t collect_interval_milliseconds = config.getUInt64("metric_log.collect_interval_milliseconds"); metric_log->startCollectMetric(collect_interval_milliseconds); } - - part_log_database = config.getString("part_log.database", "system"); } diff --git a/dbms/src/Interpreters/SystemLog.h b/dbms/src/Interpreters/SystemLog.h index e0153506a40..e9f3dd816c6 100644 --- a/dbms/src/Interpreters/SystemLog.h +++ b/dbms/src/Interpreters/SystemLog.h @@ -82,8 +82,6 @@ struct SystemLogs std::shared_ptr trace_log; /// Used to log traces from query profiler std::shared_ptr text_log; /// Used to log all text messages. std::shared_ptr metric_log; /// Used to log all metrics. - - String part_log_database; }; @@ -176,6 +174,7 @@ SystemLog::SystemLog(Context & context_, , storage_def(storage_def_), flush_interval_milliseconds(flush_interval_milliseconds_) { + assert(database_name_ == DatabaseCatalog::SYSTEM_DATABASE); log = &Logger::get("SystemLog (" + database_name_ + "." + table_name_ + ")"); saving_thread = ThreadFromGlobalPool([this] { savingThreadFunction(); }); diff --git a/dbms/tests/queries/0_stateless/01069_database_memory.reference b/dbms/tests/queries/0_stateless/01069_database_memory.reference index 35175dd4b62..393c85070b9 100644 --- a/dbms/tests/queries/0_stateless/01069_database_memory.reference +++ b/dbms/tests/queries/0_stateless/01069_database_memory.reference @@ -5,4 +5,4 @@ CREATE DATABASE memory_01069 ENGINE = Memory() 4 3 4 -CREATE TABLE memory_01069.file (`n` UInt8) ENGINE = File(CSV) +CREATE TABLE memory_01069.file (`n` UInt8) ENGINE = File(\'CSV\') From e95ce301f5b1ed73895928e7c259c682d6f4af05 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Mon, 2 Mar 2020 23:01:57 +0300 Subject: [PATCH 046/712] Update base64_hits.xml --- dbms/tests/performance/base64_hits.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dbms/tests/performance/base64_hits.xml b/dbms/tests/performance/base64_hits.xml index 5002edccaea..edf4321fa07 100644 --- a/dbms/tests/performance/base64_hits.xml +++ b/dbms/tests/performance/base64_hits.xml @@ -2,7 +2,7 @@ once - hits_10m_single + hits_100m_single @@ -27,7 +27,7 @@ - SELECT count() FROM hits_10m_single WHERE NOT ignore(base64Encode({string})) - SELECT count() FROM hits_10m_single WHERE base64Decode(base64Encode({string})) != {string} - SELECT count() FROM hits_10m_single WHERE tryBase64Decode(base64Encode({string})) != {string} + SELECT count() FROM hits_100m_single WHERE NOT ignore(base64Encode({string})) + SELECT count() FROM hits_100m_single WHERE base64Decode(base64Encode({string})) != {string} + SELECT count() FROM hits_100m_single WHERE tryBase64Decode(base64Encode({string})) != {string} From 634ca1feec395a0f8bfdf63ee5fd60d842d006e0 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Mon, 2 Mar 2020 23:02:55 +0300 Subject: [PATCH 047/712] Update array_join.xml --- dbms/tests/performance/array_join.xml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dbms/tests/performance/array_join.xml b/dbms/tests/performance/array_join.xml index 7574c5f8b15..e5025695d15 100644 --- a/dbms/tests/performance/array_join.xml +++ b/dbms/tests/performance/array_join.xml @@ -8,10 +8,10 @@ - SELECT count() FROM (SELECT [number] a, [number * 2] b FROM numbers(1000000)) AS t ARRAY JOIN a, b WHERE NOT ignore(a + b) - SELECT count() FROM (SELECT [number] a, [number * 2] b FROM numbers(1000000)) AS t LEFT ARRAY JOIN a, b WHERE NOT ignore(a + b) - SELECT count() FROM (SELECT [number] a, [number * 2] b FROM numbers(1000000)) AS t ARRAY JOIN a, b WHERE NOT ignore(a + b) SETTINGS enable_unaligned_array_join = 1 - SELECT count() FROM (SELECT [number] a, [number * 2] b FROM numbers(1000000)) AS t LEFT ARRAY JOIN a, b WHERE NOT ignore(a + b) SETTINGS enable_unaligned_array_join = 1 - SELECT count() FROM (SELECT [number] a, [number * 2, number] b FROM numbers(1000000)) AS t ARRAY JOIN a, b WHERE NOT ignore(a + b) SETTINGS enable_unaligned_array_join = 1 - SELECT count() FROM (SELECT [number] a, [number * 2, number] b FROM numbers(1000000)) AS t LEFT ARRAY JOIN a, b WHERE NOT ignore(a + b) SETTINGS enable_unaligned_array_join = 1 + SELECT count() FROM (SELECT [number] a, [number * 2] b FROM numbers(10000000)) AS t ARRAY JOIN a, b WHERE NOT ignore(a + b) + SELECT count() FROM (SELECT [number] a, [number * 2] b FROM numbers(10000000)) AS t LEFT ARRAY JOIN a, b WHERE NOT ignore(a + b) + SELECT count() FROM (SELECT [number] a, [number * 2] b FROM numbers(10000000)) AS t ARRAY JOIN a, b WHERE NOT ignore(a + b) SETTINGS enable_unaligned_array_join = 1 + SELECT count() FROM (SELECT [number] a, [number * 2] b FROM numbers(10000000)) AS t LEFT ARRAY JOIN a, b WHERE NOT ignore(a + b) SETTINGS enable_unaligned_array_join = 1 + SELECT count() FROM (SELECT [number] a, [number * 2, number] b FROM numbers(10000000)) AS t ARRAY JOIN a, b WHERE NOT ignore(a + b) SETTINGS enable_unaligned_array_join = 1 + SELECT count() FROM (SELECT [number] a, [number * 2, number] b FROM numbers(10000000)) AS t LEFT ARRAY JOIN a, b WHERE NOT ignore(a + b) SETTINGS enable_unaligned_array_join = 1 From 70dc82f9aa4e40972264535707f3e4d76404c7b8 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 2 Mar 2020 23:23:58 +0300 Subject: [PATCH 048/712] add StorageID to ASTInsertQuery --- dbms/programs/server/MySQLHandler.cpp | 2 +- dbms/programs/server/TCPHandler.cpp | 6 +++--- .../CheckConstraintsBlockOutputStream.cpp | 6 +++--- .../CheckConstraintsBlockOutputStream.h | 5 +++-- .../InputStreamFromASTInsertQuery.cpp | 4 ++-- .../PushingToViewsBlockOutputStream.cpp | 3 +-- dbms/src/Interpreters/Context.h | 7 ++++--- .../Interpreters/InterpreterCreateQuery.cpp | 6 +----- .../Interpreters/InterpreterInsertQuery.cpp | 19 ++++++++----------- .../src/Interpreters/InterpreterInsertQuery.h | 2 +- dbms/src/Interpreters/SystemLog.h | 3 +-- dbms/src/Interpreters/executeQuery.cpp | 6 +++--- dbms/src/Parsers/ASTInsertQuery.cpp | 4 +++- dbms/src/Parsers/ASTInsertQuery.h | 7 +++---- dbms/src/Parsers/ParserInsertQuery.cpp | 4 ++-- dbms/src/Storages/Kafka/StorageKafka.cpp | 3 +-- dbms/src/Storages/StorageBuffer.cpp | 4 +--- dbms/src/Storages/StorageDistributed.cpp | 3 +-- 18 files changed, 42 insertions(+), 52 deletions(-) diff --git a/dbms/programs/server/MySQLHandler.cpp b/dbms/programs/server/MySQLHandler.cpp index 3d4461a2317..1356b99ff2d 100644 --- a/dbms/programs/server/MySQLHandler.cpp +++ b/dbms/programs/server/MySQLHandler.cpp @@ -252,7 +252,7 @@ void MySQLHandler::comFieldList(ReadBuffer & payload) ComFieldList packet; packet.readPayload(payload); String database = connection_context.getCurrentDatabase(); - StoragePtr tablePtr = connection_context.getTable(database, packet.table); + StoragePtr tablePtr = DatabaseCatalog::instance().getTable({database, packet.table}); for (const NameAndTypePair & column: tablePtr->getColumns().getAll()) { ColumnDefinition column_definition( diff --git a/dbms/programs/server/TCPHandler.cpp b/dbms/programs/server/TCPHandler.cpp index 997381c3a93..661b92bcc12 100644 --- a/dbms/programs/server/TCPHandler.cpp +++ b/dbms/programs/server/TCPHandler.cpp @@ -463,11 +463,11 @@ void TCPHandler::processInsertQuery(const Settings & connection_settings) /// Send ColumnsDescription for insertion table if (client_revision >= DBMS_MIN_REVISION_WITH_COLUMN_DEFAULTS_METADATA) { - const auto & db_and_table = query_context->getInsertionTable(); + const auto & table_id = query_context->getInsertionTable(); if (query_context->getSettingsRef().input_format_defaults_for_omitted_fields) { - if (!db_and_table.second.empty()) - sendTableColumns(query_context->getTable(db_and_table.first, db_and_table.second)->getColumns()); + if (!table_id.empty()) + sendTableColumns(DatabaseCatalog::instance().getTable(table_id)->getColumns()); } } diff --git a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp index b9f4412c034..25e4340f655 100644 --- a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp +++ b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp @@ -19,12 +19,12 @@ namespace ErrorCodes CheckConstraintsBlockOutputStream::CheckConstraintsBlockOutputStream( - const String & table_, + const StorageID & table_id_, const BlockOutputStreamPtr & output_, const Block & header_, const ConstraintsDescription & constraints_, const Context & context_) - : table(table_), + : table_id(table_id_), output(output_), header(header_), constraints(constraints_), @@ -62,7 +62,7 @@ void CheckConstraintsBlockOutputStream::write(const Block & block) std::stringstream exception_message; exception_message << "Constraint " << backQuote(constraints.constraints[i]->name) - << " for table " << backQuote(table) + << " for table " << table_id.getNameForLogs() << " is violated at row " << (rows_written + row_idx + 1) << ". Expression: (" << serializeAST(*(constraints.constraints[i]->expr), true) << ")" << ". Column values"; diff --git a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h index 17b30a0ec4b..2e5f1464536 100644 --- a/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h +++ b/dbms/src/DataStreams/CheckConstraintsBlockOutputStream.h @@ -2,6 +2,7 @@ #include #include +#include namespace DB @@ -15,7 +16,7 @@ class CheckConstraintsBlockOutputStream : public IBlockOutputStream { public: CheckConstraintsBlockOutputStream( - const String & table_, + const StorageID & table_, const BlockOutputStreamPtr & output_, const Block & header_, const ConstraintsDescription & constraints_, @@ -30,7 +31,7 @@ public: void writeSuffix() override; private: - String table; + StorageID table_id; BlockOutputStreamPtr output; Block header; const ConstraintsDescription constraints; diff --git a/dbms/src/DataStreams/InputStreamFromASTInsertQuery.cpp b/dbms/src/DataStreams/InputStreamFromASTInsertQuery.cpp index bf31f55c610..8d8f64b4c83 100644 --- a/dbms/src/DataStreams/InputStreamFromASTInsertQuery.cpp +++ b/dbms/src/DataStreams/InputStreamFromASTInsertQuery.cpp @@ -56,9 +56,9 @@ InputStreamFromASTInsertQuery::InputStreamFromASTInsertQuery( res_stream = context.getInputFormat(format, *input_buffer_contacenated, header, context.getSettings().max_insert_block_size); - if (context.getSettingsRef().input_format_defaults_for_omitted_fields && !ast_insert_query->table.empty() && !input_function) + if (context.getSettingsRef().input_format_defaults_for_omitted_fields && !ast_insert_query->table_id.empty() && !input_function) { - StoragePtr storage = context.getTable(ast_insert_query->database, ast_insert_query->table); + StoragePtr storage = DatabaseCatalog::instance().getTable(ast_insert_query->table_id); auto column_defaults = storage->getColumns().getDefaults(); if (!column_defaults.empty()) res_stream = std::make_shared(res_stream, column_defaults, context); diff --git a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp index 01a2be34f2f..f6dbf0b6c0b 100644 --- a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp +++ b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp @@ -61,8 +61,7 @@ PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream( query = materialized_view->getInnerQuery(); std::unique_ptr insert = std::make_unique(); - insert->database = inner_table_id.database_name; - insert->table = inner_table_id.table_name; + insert->table_id = inner_table_id; /// Get list of columns we get from select query. auto header = InterpreterSelectQuery(query, *views_context, SelectQueryOptions().analyze()) diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index bbd199cfd4d..ef9e1cf646a 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -160,7 +160,7 @@ private: using ProgressCallback = std::function; ProgressCallback progress_callback; /// Callback for tracking progress of query execution. QueryStatus * process_list_elem = nullptr; /// For tracking total resource usage for query. - std::pair insertion_table; /// Saved insertion table in query context + StorageID insertion_table = StorageID::createEmpty(); /// Saved insertion table in query context String default_format; /// Format, used when server formats data by itself and if query does not have FORMAT specification. /// Thus, used in HTTP interface. If not specified - then some globally default format is used. @@ -297,6 +297,7 @@ public: ResolveCurrentDatabase = 2u, /// Use current database ResolveOrdinary = ResolveGlobal | ResolveCurrentDatabase, /// If database name is not specified, use current database ResolveExternal = 4u, /// Try get external table + ResolveExternalOrGlobal = ResolveGlobal | ResolveExternal, /// If external table doesn't exist, database name must be specifies ResolveAll = ResolveExternal | ResolveOrdinary /// If database name is not specified, try get external table, /// if external table not found use current database. }; @@ -332,8 +333,8 @@ public: void killCurrentQuery(); - void setInsertionTable(std::pair && db_and_table) { insertion_table = db_and_table; } - const std::pair & getInsertionTable() const { return insertion_table; } + void setInsertionTable(StorageID db_and_table) { insertion_table = std::move(db_and_table); } + const StorageID & getInsertionTable() const { return insertion_table; } String getDefaultFormat() const; /// If default_format is not specified, some global default format is returned. void setDefaultFormat(const String & name); diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp index 31daae6af90..4d2579cd360 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp @@ -662,11 +662,7 @@ BlockIO InterpreterCreateQuery::fillTableIfNeeded(const ASTCreateQuery & create) && !create.is_view && !create.is_live_view && (!create.is_materialized_view || create.is_populate)) { auto insert = std::make_shared(); - - if (!create.temporary) - insert->database = create.database; - - insert->table = create.table; + insert->table_id = context.getSessionContext().resolveStorageID({create.database, create.table, create.uuid}, Context::ResolveExternalOrGlobal); insert->select = create.select->clone(); if (create.temporary && !context.getSessionContext().hasQueryContext()) diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.cpp b/dbms/src/Interpreters/InterpreterInsertQuery.cpp index bab361ab52d..ce3a90fdf81 100644 --- a/dbms/src/Interpreters/InterpreterInsertQuery.cpp +++ b/dbms/src/Interpreters/InterpreterInsertQuery.cpp @@ -56,10 +56,8 @@ StoragePtr InterpreterInsertQuery::getTable(ASTInsertQuery & query) return table_function_ptr->execute(query.table_function, context, table_function_ptr->getName()); } - /// Into what table to write. - if (query.database.empty() && !context.isExternalTableExist(query.table)) - query.database = context.getCurrentDatabase(); - return context.getTable(query.database, query.table); + query.table_id = context.resolveStorageID(query.table_id); + return DatabaseCatalog::instance().getTable(query.table_id); } Block InterpreterInsertQuery::getSampleBlock(const ASTInsertQuery & query, const StoragePtr & table) @@ -83,7 +81,7 @@ Block InterpreterInsertQuery::getSampleBlock(const ASTInsertQuery & query, const /// The table does not have a column with that name if (!table_sample.has(current_name)) - throw Exception("No such column " + current_name + " in table " + query.table, ErrorCodes::NO_SUCH_COLUMN_IN_TABLE); + throw Exception("No such column " + current_name + " in table " + query.table_id.getNameForLogs(), ErrorCodes::NO_SUCH_COLUMN_IN_TABLE); if (!allow_materialized && !table_sample_non_materialized.has(current_name)) throw Exception("Cannot insert column " + current_name + ", because it is MATERIALIZED column.", ErrorCodes::ILLEGAL_COLUMN); @@ -107,8 +105,8 @@ BlockIO InterpreterInsertQuery::execute() auto table_lock = table->lockStructureForShare(true, context.getInitialQueryId()); auto query_sample_block = getSampleBlock(query, table); - if (!query.table.empty() && !query.database.empty() /* always allow access to temporary tables */) - context.checkAccess(AccessType::INSERT, query.database, query.table, query_sample_block.getNames()); + if (!query.table_function) + context.checkAccess(AccessType::INSERT, query.table_id.database_name, query.table_id.table_name, query_sample_block.getNames()); BlockInputStreams in_streams; size_t out_streams_size = 1; @@ -159,7 +157,7 @@ BlockIO InterpreterInsertQuery::execute() out, query_sample_block, out->getHeader(), table->getColumns().getDefaults(), context); if (const auto & constraints = table->getConstraints(); !constraints.empty()) - out = std::make_shared(query.table, + out = std::make_shared(query.table_id, out, query_sample_block, table->getConstraints(), context); auto out_wrapper = std::make_shared(out); @@ -207,10 +205,9 @@ BlockIO InterpreterInsertQuery::execute() } -std::pair InterpreterInsertQuery::getDatabaseTable() const +StorageID InterpreterInsertQuery::getDatabaseTable() const { - const auto & query = query_ptr->as(); - return {query.database, query.table}; + return query_ptr->as().table_id; } } diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.h b/dbms/src/Interpreters/InterpreterInsertQuery.h index 03317aaef86..cead826c48a 100644 --- a/dbms/src/Interpreters/InterpreterInsertQuery.h +++ b/dbms/src/Interpreters/InterpreterInsertQuery.h @@ -29,7 +29,7 @@ public: */ BlockIO execute() override; - std::pair getDatabaseTable() const; + StorageID getDatabaseTable() const; private: StoragePtr getTable(ASTInsertQuery & query); diff --git a/dbms/src/Interpreters/SystemLog.h b/dbms/src/Interpreters/SystemLog.h index e9f3dd816c6..83a2f666cca 100644 --- a/dbms/src/Interpreters/SystemLog.h +++ b/dbms/src/Interpreters/SystemLog.h @@ -339,8 +339,7 @@ void SystemLog::flushImpl(const std::vector & to_flush, /// This is needed to support DEFAULT-columns in table. std::unique_ptr insert = std::make_unique(); - insert->database = table_id.database_name; - insert->table = table_id.table_name; + insert->table_id = table_id; ASTPtr query_ptr(insert.release()); InterpreterInsertQuery interpreter(query_ptr, context); diff --git a/dbms/src/Interpreters/executeQuery.cpp b/dbms/src/Interpreters/executeQuery.cpp index 8cd7edc85b8..4deeee60885 100644 --- a/dbms/src/Interpreters/executeQuery.cpp +++ b/dbms/src/Interpreters/executeQuery.cpp @@ -330,9 +330,9 @@ static std::tuple executeQueryImpl( if (auto * insert_interpreter = typeid_cast(&*interpreter)) { /// Save insertion table (not table function). TODO: support remote() table function. - auto db_table = insert_interpreter->getDatabaseTable(); - if (!db_table.second.empty()) - context.setInsertionTable(std::move(db_table)); + auto table_id = insert_interpreter->getDatabaseTable(); + if (!table_id.empty()) + context.setInsertionTable(std::move(table_id)); } if (process_list_entry) diff --git a/dbms/src/Parsers/ASTInsertQuery.cpp b/dbms/src/Parsers/ASTInsertQuery.cpp index 1bd3f98751a..37e94c49668 100644 --- a/dbms/src/Parsers/ASTInsertQuery.cpp +++ b/dbms/src/Parsers/ASTInsertQuery.cpp @@ -2,6 +2,7 @@ #include #include #include +#include namespace DB @@ -25,7 +26,8 @@ void ASTInsertQuery::formatImpl(const FormatSettings & settings, FormatState & s } else settings.ostr << (settings.hilite ? hilite_none : "") - << (!database.empty() ? backQuoteIfNeed(database) + "." : "") << backQuoteIfNeed(table); + << (!table_id.database_name.empty() ? backQuoteIfNeed(table_id.database_name) + "." : "") << backQuoteIfNeed(table_id.table_name) + << (table_id.hasUUID() ? " UUID " : "") << (table_id.hasUUID() ? quoteString(toString(table_id.uuid)) : ""); if (columns) { diff --git a/dbms/src/Parsers/ASTInsertQuery.h b/dbms/src/Parsers/ASTInsertQuery.h index 7eab80ca54d..93a99012be6 100644 --- a/dbms/src/Parsers/ASTInsertQuery.h +++ b/dbms/src/Parsers/ASTInsertQuery.h @@ -1,7 +1,7 @@ #pragma once #include - +#include namespace DB { @@ -12,8 +12,7 @@ namespace DB class ASTInsertQuery : public IAST { public: - String database; - String table; + StorageID table_id = StorageID::createEmpty(); ASTPtr columns; String format; ASTPtr select; @@ -31,7 +30,7 @@ public: void tryFindInputFunction(ASTPtr & input_function) const; /** Get the text that identifies this element. */ - String getID(char delim) const override { return "InsertQuery" + (delim + database) + delim + table; } + String getID(char delim) const override { return "InsertQuery" + (delim + table_id.database_name) + delim + table_id.table_name; } ASTPtr clone() const override { diff --git a/dbms/src/Parsers/ParserInsertQuery.cpp b/dbms/src/Parsers/ParserInsertQuery.cpp index 62f9f57930c..26219bc82cb 100644 --- a/dbms/src/Parsers/ParserInsertQuery.cpp +++ b/dbms/src/Parsers/ParserInsertQuery.cpp @@ -151,8 +151,8 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) } else { - tryGetIdentifierNameInto(database, query->database); - tryGetIdentifierNameInto(table, query->table); + tryGetIdentifierNameInto(database, query->table_id.database_name); + tryGetIdentifierNameInto(table, query->table_id.table_name); } tryGetIdentifierNameInto(format, query->format); diff --git a/dbms/src/Storages/Kafka/StorageKafka.cpp b/dbms/src/Storages/Kafka/StorageKafka.cpp index b937f9e52f6..d058932eacc 100644 --- a/dbms/src/Storages/Kafka/StorageKafka.cpp +++ b/dbms/src/Storages/Kafka/StorageKafka.cpp @@ -353,8 +353,7 @@ bool StorageKafka::streamToViews() // Create an INSERT query for streaming data auto insert = std::make_shared(); - insert->database = table_id.database_name; - insert->table = table_id.table_name; + insert->table_id = table_id; const Settings & settings = global_context.getSettingsRef(); size_t block_size = max_block_size; diff --git a/dbms/src/Storages/StorageBuffer.cpp b/dbms/src/Storages/StorageBuffer.cpp index a5c71e86b7b..217d474defe 100644 --- a/dbms/src/Storages/StorageBuffer.cpp +++ b/dbms/src/Storages/StorageBuffer.cpp @@ -643,9 +643,7 @@ void StorageBuffer::writeBlockToDestination(const Block & block, StoragePtr tabl auto temporarily_disable_memory_tracker = getCurrentMemoryTrackerActionLock(); auto insert = std::make_shared(); - - insert->database = destination_id.database_name; - insert->table = destination_id.table_name; + insert->table_id = destination_id; /** We will insert columns that are the intersection set of columns of the buffer table and the subordinate table. * This will support some of the cases (but not all) when the table structure does not match. diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp index 3aa6d24c7fd..41c00f6c7b3 100644 --- a/dbms/src/Storages/StorageDistributed.cpp +++ b/dbms/src/Storages/StorageDistributed.cpp @@ -110,8 +110,7 @@ ASTPtr rewriteSelectQuery(const ASTPtr & query, const std::string & database, co ASTPtr createInsertToRemoteTableQuery(const std::string & database, const std::string & table, const Block & sample_block_non_materialized) { auto query = std::make_shared(); - query->database = database; - query->table = table; + query->table_id = StorageID(database, table); auto columns = std::make_shared(); query->columns = columns; From 3b54c5eae794b2494624a4b812d7d91dd5c32537 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Mon, 2 Mar 2020 23:36:44 +0300 Subject: [PATCH 049/712] Update group_array_moving_sum.xml --- .../performance/group_array_moving_sum.xml | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/dbms/tests/performance/group_array_moving_sum.xml b/dbms/tests/performance/group_array_moving_sum.xml index fadd225e1a2..d5a0030b623 100644 --- a/dbms/tests/performance/group_array_moving_sum.xml +++ b/dbms/tests/performance/group_array_moving_sum.xml @@ -12,26 +12,26 @@ - CREATE TABLE moving_sum_1m(k UInt64, v UInt64) ENGINE = MergeTree ORDER BY k CREATE TABLE moving_sum_10m(k UInt64, v UInt64) ENGINE = MergeTree ORDER BY k + CREATE TABLE moving_sum_100m(k UInt64, v UInt64) ENGINE = MergeTree ORDER BY k - INSERT INTO moving_sum_1m SELECT number%100, rand() from numbers(1000000) INSERT INTO moving_sum_10m SELECT number%100, rand() from numbers(10000000) + INSERT INTO moving_sum_100m SELECT number%100, rand() from numbers(100000000) - SELECT k,groupArrayMovingSum(10)(v) FROM moving_sum_1m GROUP BY k FORMAT Null - SELECT k,groupArrayMovingSum(10)(v) FROM moving_sum_1m WHERE k in (49, 50, 51) GROUP BY k FORMAT Null - SELECT k,groupArrayMovingSum(1000)(v) FROM moving_sum_1m GROUP BY k FORMAT Null - SELECT k,groupArrayMovingSum(1000)(v) FROM moving_sum_1m WHERE k in (49, 50, 51) GROUP BY k FORMAT Null - SELECT k,groupArrayMovingSum(10000)(v) FROM moving_sum_1m GROUP BY k FORMAT Null - SELECT k,groupArrayMovingSum(10000)(v) FROM moving_sum_1m WHERE k in (49, 50, 51) GROUP BY k FORMAT Null + SELECT k,groupArrayMovingSum(10)(v) FROM moving_sum_10m GROUP BY k FORMAT Null + SELECT k,groupArrayMovingSum(10)(v) FROM moving_sum_10m WHERE k in (49, 50, 51) GROUP BY k FORMAT Null + SELECT k,groupArrayMovingSum(1000)(v) FROM moving_sum_10m GROUP BY k FORMAT Null + SELECT k,groupArrayMovingSum(1000)(v) FROM moving_sum_10m WHERE k in (49, 50, 51) GROUP BY k FORMAT Null + SELECT k,groupArrayMovingSum(10000)(v) FROM moving_sum_10m GROUP BY k FORMAT Null + SELECT k,groupArrayMovingSum(10000)(v) FROM moving_sum_10m WHERE k in (49, 50, 51) GROUP BY k FORMAT Null - SELECT k,groupArrayMovingSum(10)(v) FROM moving_sum_10m GROUP BY k FORMAT Null - SELECT k,groupArrayMovingSum(10)(v) FROM moving_sum_10m WHERE k in (49, 50, 51) GROUP BY k FORMAT Null - SELECT k,groupArrayMovingSum(1000)(v) FROM moving_sum_10m GROUP BY k FORMAT Null - SELECT k,groupArrayMovingSum(1000)(v) FROM moving_sum_10m WHERE k in (49, 50, 51) GROUP BY k FORMAT Null - SELECT k,groupArrayMovingSum(10000)(v) FROM moving_sum_10m GROUP BY k FORMAT Null - SELECT k,groupArrayMovingSum(10000)(v) FROM moving_sum_10m WHERE k in (49, 50, 51) GROUP BY k FORMAT Null + SELECT k,groupArrayMovingSum(10)(v) FROM moving_sum_100m GROUP BY k FORMAT Null + SELECT k,groupArrayMovingSum(10)(v) FROM moving_sum_100m WHERE k in (49, 50, 51) GROUP BY k FORMAT Null + SELECT k,groupArrayMovingSum(1000)(v) FROM moving_sum_100m GROUP BY k FORMAT Null + SELECT k,groupArrayMovingSum(1000)(v) FROM moving_sum_100m WHERE k in (49, 50, 51) GROUP BY k FORMAT Null + SELECT k,groupArrayMovingSum(10000)(v) FROM moving_sum_100m GROUP BY k FORMAT Null + SELECT k,groupArrayMovingSum(10000)(v) FROM moving_sum_100m WHERE k in (49, 50, 51) GROUP BY k FORMAT Null + DROP TABLE IF EXISTS moving_sum_100m DROP TABLE IF EXISTS moving_sum_10m - DROP TABLE IF EXISTS moving_sum_1m From 493354968bc3efc032c135ff1deb1d6e40e63693 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Mon, 2 Mar 2020 23:40:17 +0300 Subject: [PATCH 050/712] Update consistent_hashes.xml --- dbms/tests/performance/consistent_hashes.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbms/tests/performance/consistent_hashes.xml b/dbms/tests/performance/consistent_hashes.xml index f3858e16a28..5929c6388d5 100644 --- a/dbms/tests/performance/consistent_hashes.xml +++ b/dbms/tests/performance/consistent_hashes.xml @@ -26,8 +26,8 @@ - SELECT {hash_func}(number, {buckets}) FROM numbers(1000000) + SELECT {hash_func}(number, {buckets}) FROM numbers(1000000) FORMAT Null - SELECT sumburConsistentHash(toUInt32(number), 2) FROM numbers(1000000) + SELECT sumburConsistentHash(toUInt32(number), 2) FROM numbers(1000000) FORMAT Null From e78ceebf6ee317308f952ba98b70c0e4c92c92ef Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 3 Mar 2020 13:00:51 +0300 Subject: [PATCH 051/712] fixup --- dbms/tests/performance/read_hits_with_aio.xml | 2 +- dbms/tests/performance/round_down.xml | 10 +++++----- dbms/tests/performance/set.xml | 4 ++-- dbms/tests/performance/simple_join_query.xml | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/dbms/tests/performance/read_hits_with_aio.xml b/dbms/tests/performance/read_hits_with_aio.xml index 9b70dfaa421..5fa3f70ed86 100644 --- a/dbms/tests/performance/read_hits_with_aio.xml +++ b/dbms/tests/performance/read_hits_with_aio.xml @@ -12,7 +12,7 @@ SELECT count() FROM hits_100m_single where UserID=1234567890 SETTINGS max_threads = 1, min_bytes_to_use_direct_io = 1, max_read_buffer_size = 10485760; -SELECT count() FROM hits_1000m_single where EventDate between toDate('2013-07-10') and toDate('2013-07-16') and UserID=123 SETTINGS max_threads = 1, min_bytes_to_use_direct_io = 1, max_read_buffer_size = 10485760; +SELECT count() FROM hits_100m_single where EventDate between toDate('2013-07-10') and toDate('2013-07-16') and UserID=123 SETTINGS max_threads = 1, min_bytes_to_use_direct_io = 1, max_read_buffer_size = 10485760; SELECT count() FROM hits_100m_single where UserID=1234567890 SETTINGS max_threads = 1, min_bytes_to_use_direct_io = 0, max_read_buffer_size = 10485760; SELECT count() FROM hits_100m_single where EventDate between toDate('2013-07-10') and toDate('2013-07-16') and UserID=123 SETTINGS max_threads = 1, min_bytes_to_use_direct_io = 0, max_read_buffer_size = 10485760; diff --git a/dbms/tests/performance/round_down.xml b/dbms/tests/performance/round_down.xml index b14b5a9fb2a..880b625af28 100644 --- a/dbms/tests/performance/round_down.xml +++ b/dbms/tests/performance/round_down.xml @@ -11,9 +11,9 @@ - SELECT count() FROM numbers(1000000) WHERE NOT ignore(roundDuration(rand() % 65536)) - SELECT count() FROM numbers(1000000) WHERE NOT ignore(roundDown(rand() % 65536, [0, 1, 10, 30, 60, 120, 180, 240, 300, 600, 1200, 1800, 3600, 7200, 18000, 36000])) - SELECT count() FROM numbers(1000000) WHERE NOT ignore(roundAge(rand() % 100)) - SELECT count() FROM numbers(1000000) WHERE NOT ignore(roundDown(rand() % 100, [0, 1, 18, 25, 35, 45, 55])) - SELECT count() FROM numbers(1000000) WHERE NOT ignore(roundDown(rand() % 65536, (SELECT groupArray(number) FROM numbers(65536)))) + SELECT count() FROM numbers(10000000) WHERE NOT ignore(roundDuration(rand() % 65536)) + SELECT count() FROM numbers(10000000) WHERE NOT ignore(roundDown(rand() % 65536, [0, 1, 10, 30, 60, 120, 180, 240, 300, 600, 1200, 1800, 3600, 7200, 18000, 36000])) + SELECT count() FROM numbers(10000000) WHERE NOT ignore(roundAge(rand() % 100)) + SELECT count() FROM numbers(10000000) WHERE NOT ignore(roundDown(rand() % 100, [0, 1, 18, 25, 35, 45, 55])) + SELECT count() FROM numbers(10000000) WHERE NOT ignore(roundDown(rand() % 65536, (SELECT groupArray(number) FROM numbers(65536)))) diff --git a/dbms/tests/performance/set.xml b/dbms/tests/performance/set.xml index c142730a560..75b87d38abe 100644 --- a/dbms/tests/performance/set.xml +++ b/dbms/tests/performance/set.xml @@ -18,8 +18,8 @@ table - numbers(1000000) - numbers_mt(10000000) + numbers(10000000) + numbers_mt(100000000) diff --git a/dbms/tests/performance/simple_join_query.xml b/dbms/tests/performance/simple_join_query.xml index c13dd70a777..919cce33be6 100644 --- a/dbms/tests/performance/simple_join_query.xml +++ b/dbms/tests/performance/simple_join_query.xml @@ -11,12 +11,12 @@ - CREATE TABLE join_table(A Int64, S0 String, S1 String, S2 String, S3 String)ENGINE = MergeTree ORDER BY A + CREATE TABLE join_table(A Int64, S0 String, S1 String, S2 String, S3 String) ENGINE = MergeTree ORDER BY A INSERT INTO join_table SELECT number AS A, toString(arrayMap(x->x, range(100))) S0, S0 AS S1, S0 AS S2, S0 AS S3 from numbers(500000) SELECT COUNT() FROM join_table LEFT JOIN join_table USING A - SELECT COUNT() FROM join_table LEFT JOIN (SELECT A FROM join_table) USING A + SELECT COUNT() FROM join_table LEFT JOIN (SELECT A FROM join_table) right USING A SELECT COUNT() FROM join_table AS left LEFT JOIN join_table AS right ON left.A = right.A SELECT COUNT() FROM join_table AS left LEFT JOIN (SELECT A FROM join_table) AS right ON left.A = right.A From 825f86759171c411cf950af09b8b6dd23794fd3c Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 3 Mar 2020 13:09:47 +0300 Subject: [PATCH 052/712] fixup --- dbms/tests/performance/if_to_multiif.xml | 6 ++++++ dbms/tests/performance/linear_regression.xml | 11 ++++++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/dbms/tests/performance/if_to_multiif.xml b/dbms/tests/performance/if_to_multiif.xml index 7dee667a1bd..03f528d6349 100644 --- a/dbms/tests/performance/if_to_multiif.xml +++ b/dbms/tests/performance/if_to_multiif.xml @@ -7,6 +7,12 @@ + + + nonexistent_table_if_multiif + + diff --git a/dbms/tests/performance/linear_regression.xml b/dbms/tests/performance/linear_regression.xml index 0b4892f71ec..c358e21af05 100644 --- a/dbms/tests/performance/linear_regression.xml +++ b/dbms/tests/performance/linear_regression.xml @@ -9,6 +9,7 @@ test.hits + hits_100m_single @@ -16,17 +17,17 @@ CREATE TABLE test_model engine = Memory as select stochasticLinearRegressionState(0.0001)(Age, Income, ParamPrice, Robotness, RefererHash) as state from test.hits - WITH (SELECT stochasticLinearRegressionState(0.0001, 0, 15)(Age, Income, ParamPrice, Robotness, RefererHash) FROM test.hits) AS model SELECT 1 - SELECT stochasticLinearRegression(Age, Income, ParamPrice, Robotness, RefererHash) FROM test.hits + SELECT stochasticLinearRegressionState(0.0001, 0, 15)(Age, Income, ParamPrice, Robotness, RefererHash) FROM test.hits FORMAT Null + SELECT stochasticLinearRegression(Age, Income, ParamPrice, Robotness, RefererHash) FROM test.hits FORMAT Null - WITH (SELECT stochasticLinearRegressionState(0.0001, 0, 15, 'Momentum')(Age, Income, ParamPrice, Robotness, RefererHash) FROM test.hits) AS model SELECT 1 + SELECT stochasticLinearRegressionState(0.0001, 0, 15, 'Momentum')(Age, Income, ParamPrice, Robotness, RefererHash) FROM hits_100m_single FORMAT Null - WITH (SELECT stochasticLinearRegressionState(0.0001, 0, 15, 'Nesterov')(Age, Income, ParamPrice, Robotness, RefererHash) FROM test.hits) AS model SELECT 1 + SELECT stochasticLinearRegressionState(0.0001, 0, 15, 'Nesterov')(Age, Income, ParamPrice, Robotness, RefererHash) FROM hits_100m_single FORMAT Null - with (SELECT state FROM test_model) as model select evalMLMethod(model, Income, ParamPrice, Robotness, RefererHash) from test.hits + WITH (SELECT state FROM test_model) AS model SELECT evalMLMethod(model, Income, ParamPrice, Robotness, RefererHash) FROM hits_100m_single FORMAT Null DROP TABLE IF EXISTS test_model From 66fbc39357ac7433103ea7df6794b6900d04bf0b Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 3 Mar 2020 13:17:23 +0300 Subject: [PATCH 053/712] fixup --- .../performance/group_array_moving_sum.xml | 30 +++++++++---------- dbms/tests/performance/if_array_num.xml | 12 ++++---- dbms/tests/performance/if_array_string.xml | 12 ++++---- 3 files changed, 27 insertions(+), 27 deletions(-) diff --git a/dbms/tests/performance/group_array_moving_sum.xml b/dbms/tests/performance/group_array_moving_sum.xml index d5a0030b623..6939989c5b4 100644 --- a/dbms/tests/performance/group_array_moving_sum.xml +++ b/dbms/tests/performance/group_array_moving_sum.xml @@ -12,26 +12,26 @@ - CREATE TABLE moving_sum_10m(k UInt64, v UInt64) ENGINE = MergeTree ORDER BY k CREATE TABLE moving_sum_100m(k UInt64, v UInt64) ENGINE = MergeTree ORDER BY k + CREATE TABLE moving_sum_1000m(k UInt64, v UInt64) ENGINE = MergeTree ORDER BY k - INSERT INTO moving_sum_10m SELECT number%100, rand() from numbers(10000000) INSERT INTO moving_sum_100m SELECT number%100, rand() from numbers(100000000) + INSERT INTO moving_sum_1000m SELECT number%100, rand() from numbers(1000000000) - SELECT k,groupArrayMovingSum(10)(v) FROM moving_sum_10m GROUP BY k FORMAT Null - SELECT k,groupArrayMovingSum(10)(v) FROM moving_sum_10m WHERE k in (49, 50, 51) GROUP BY k FORMAT Null - SELECT k,groupArrayMovingSum(1000)(v) FROM moving_sum_10m GROUP BY k FORMAT Null - SELECT k,groupArrayMovingSum(1000)(v) FROM moving_sum_10m WHERE k in (49, 50, 51) GROUP BY k FORMAT Null - SELECT k,groupArrayMovingSum(10000)(v) FROM moving_sum_10m GROUP BY k FORMAT Null - SELECT k,groupArrayMovingSum(10000)(v) FROM moving_sum_10m WHERE k in (49, 50, 51) GROUP BY k FORMAT Null + SELECT k,groupArrayMovingSum(10)(v) FROM moving_sum_100m GROUP BY k FORMAT Null + SELECT k,groupArrayMovingSum(10)(v) FROM moving_sum_100m WHERE k in (49, 50, 51) GROUP BY k FORMAT Null + SELECT k,groupArrayMovingSum(1000)(v) FROM moving_sum_100m GROUP BY k FORMAT Null + SELECT k,groupArrayMovingSum(1000)(v) FROM moving_sum_100m WHERE k in (49, 50, 51) GROUP BY k FORMAT Null + SELECT k,groupArrayMovingSum(10000)(v) FROM moving_sum_100m GROUP BY k FORMAT Null + SELECT k,groupArrayMovingSum(10000)(v) FROM moving_sum_100m WHERE k in (49, 50, 51) GROUP BY k FORMAT Null - SELECT k,groupArrayMovingSum(10)(v) FROM moving_sum_100m GROUP BY k FORMAT Null - SELECT k,groupArrayMovingSum(10)(v) FROM moving_sum_100m WHERE k in (49, 50, 51) GROUP BY k FORMAT Null - SELECT k,groupArrayMovingSum(1000)(v) FROM moving_sum_100m GROUP BY k FORMAT Null - SELECT k,groupArrayMovingSum(1000)(v) FROM moving_sum_100m WHERE k in (49, 50, 51) GROUP BY k FORMAT Null - SELECT k,groupArrayMovingSum(10000)(v) FROM moving_sum_100m GROUP BY k FORMAT Null - SELECT k,groupArrayMovingSum(10000)(v) FROM moving_sum_100m WHERE k in (49, 50, 51) GROUP BY k FORMAT Null + SELECT k,groupArrayMovingSum(10)(v) FROM moving_sum_1000m GROUP BY k FORMAT Null + SELECT k,groupArrayMovingSum(10)(v) FROM moving_sum_1000m WHERE k in (49, 50, 51) GROUP BY k FORMAT Null + SELECT k,groupArrayMovingSum(1000)(v) FROM moving_sum_1000m GROUP BY k FORMAT Null + SELECT k,groupArrayMovingSum(1000)(v) FROM moving_sum_1000m WHERE k in (49, 50, 51) GROUP BY k FORMAT Null + SELECT k,groupArrayMovingSum(10000)(v) FROM moving_sum_1000m GROUP BY k FORMAT Null + SELECT k,groupArrayMovingSum(10000)(v) FROM moving_sum_1000m WHERE k in (49, 50, 51) GROUP BY k FORMAT Null + DROP TABLE IF EXISTS moving_sum_1000m DROP TABLE IF EXISTS moving_sum_100m - DROP TABLE IF EXISTS moving_sum_10m diff --git a/dbms/tests/performance/if_array_num.xml b/dbms/tests/performance/if_array_num.xml index 375290e635c..d4c9c29dd99 100644 --- a/dbms/tests/performance/if_array_num.xml +++ b/dbms/tests/performance/if_array_num.xml @@ -8,10 +8,10 @@ - SELECT count() FROM numbers(1000000) WHERE NOT ignore(rand() % 2 ? [1, 2, 3] : [4, 5]) - SELECT count() FROM numbers(1000000) WHERE NOT ignore(rand() % 2 ? [1, 2, 3] : materialize([4, 5])) - SELECT count() FROM numbers(1000000) WHERE NOT ignore(rand() % 2 ? materialize([1, 2, 3]) : materialize([4, 5])) - SELECT count() FROM numbers(1000000) WHERE NOT ignore(rand() % 2 ? [1, 2, 3] : [400, 500]) - SELECT count() FROM numbers(1000000) WHERE NOT ignore(rand() % 2 ? [1, 2, 3] : materialize([400, 500])) - SELECT count() FROM numbers(1000000) WHERE NOT ignore(rand() % 2 ? materialize([1, 2, 3]) : materialize([400, 500])) + SELECT count() FROM numbers(10000000) WHERE NOT ignore(rand() % 2 ? [1, 2, 3] : [4, 5]) + SELECT count() FROM numbers(10000000) WHERE NOT ignore(rand() % 2 ? [1, 2, 3] : materialize([4, 5])) + SELECT count() FROM numbers(10000000) WHERE NOT ignore(rand() % 2 ? materialize([1, 2, 3]) : materialize([4, 5])) + SELECT count() FROM numbers(10000000) WHERE NOT ignore(rand() % 2 ? [1, 2, 3] : [400, 500]) + SELECT count() FROM numbers(10000000) WHERE NOT ignore(rand() % 2 ? [1, 2, 3] : materialize([400, 500])) + SELECT count() FROM numbers(10000000) WHERE NOT ignore(rand() % 2 ? materialize([1, 2, 3]) : materialize([400, 500])) diff --git a/dbms/tests/performance/if_array_string.xml b/dbms/tests/performance/if_array_string.xml index 1f14393ee16..235051fc905 100644 --- a/dbms/tests/performance/if_array_string.xml +++ b/dbms/tests/performance/if_array_string.xml @@ -8,10 +8,10 @@ - SELECT count() FROM numbers(1000000) WHERE NOT ignore(rand() % 2 ? ['Hello', 'World'] : ['a', 'b', 'c']) - SELECT count() FROM numbers(1000000) WHERE NOT ignore(rand() % 2 ? materialize(['Hello', 'World']) : ['a', 'b', 'c']) - SELECT count() FROM numbers(1000000) WHERE NOT ignore(rand() % 2 ? ['Hello', 'World'] : materialize(['a', 'b', 'c'])) - SELECT count() FROM numbers(1000000) WHERE NOT ignore(rand() % 2 ? materialize(['Hello', 'World']) : materialize(['a', 'b', 'c'])) - SELECT count() FROM numbers(1000000) WHERE NOT ignore(rand() % 2 ? materialize(['', '']) : emptyArrayString()) - SELECT count() FROM numbers(1000000) WHERE NOT ignore(rand() % 2 ? materialize(['https://github.com/ClickHouse/ClickHouse/pull/1070', 'https://www.google.ru/search?newwindow=1&site=&source=hp&q=zookeeper+wire+protocol+exists&oq=zookeeper+wire+protocol+exists&gs_l=psy-ab.3...330.6300.0.6687.33.28.0.0.0.0.386.4838.0j5j9j5.19.0....0...1.1.64.psy-ab..14.17.4448.0..0j35i39k1j0i131k1j0i22i30k1j0i19k1j33i21k1.r_3uFoNOrSU']) : emptyArrayString()) + SELECT count() FROM numbers(10000000) WHERE NOT ignore(rand() % 2 ? ['Hello', 'World'] : ['a', 'b', 'c']) + SELECT count() FROM numbers(10000000) WHERE NOT ignore(rand() % 2 ? materialize(['Hello', 'World']) : ['a', 'b', 'c']) + SELECT count() FROM numbers(10000000) WHERE NOT ignore(rand() % 2 ? ['Hello', 'World'] : materialize(['a', 'b', 'c'])) + SELECT count() FROM numbers(10000000) WHERE NOT ignore(rand() % 2 ? materialize(['Hello', 'World']) : materialize(['a', 'b', 'c'])) + SELECT count() FROM numbers(10000000) WHERE NOT ignore(rand() % 2 ? materialize(['', '']) : emptyArrayString()) + SELECT count() FROM numbers(10000000) WHERE NOT ignore(rand() % 2 ? materialize(['https://github.com/ClickHouse/ClickHouse/pull/1070', 'https://www.google.ru/search?newwindow=1&site=&source=hp&q=zookeeper+wire+protocol+exists&oq=zookeeper+wire+protocol+exists&gs_l=psy-ab.3...330.6300.0.6687.33.28.0.0.0.0.386.4838.0j5j9j5.19.0....0...1.1.64.psy-ab..14.17.4448.0..0j35i39k1j0i131k1j0i22i30k1j0i19k1j33i21k1.r_3uFoNOrSU']) : emptyArrayString()) From e594955618c0f8f6f40ce73c57447bb55046367c Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 3 Mar 2020 13:49:16 +0300 Subject: [PATCH 054/712] boop From add201401da7ee7f90929ba383971da7bd93b0f6 Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Tue, 3 Mar 2020 11:50:09 +0100 Subject: [PATCH 055/712] Try newer version of odbc driver --- .../{bugs => 0_stateless}/01086_odbc_roundtrip.reference | 0 .../queries/{bugs => 0_stateless}/01086_odbc_roundtrip.sql | 0 docker/test/stateless/Dockerfile | 2 +- 3 files changed, 1 insertion(+), 1 deletion(-) rename dbms/tests/queries/{bugs => 0_stateless}/01086_odbc_roundtrip.reference (100%) rename dbms/tests/queries/{bugs => 0_stateless}/01086_odbc_roundtrip.sql (100%) diff --git a/dbms/tests/queries/bugs/01086_odbc_roundtrip.reference b/dbms/tests/queries/0_stateless/01086_odbc_roundtrip.reference similarity index 100% rename from dbms/tests/queries/bugs/01086_odbc_roundtrip.reference rename to dbms/tests/queries/0_stateless/01086_odbc_roundtrip.reference diff --git a/dbms/tests/queries/bugs/01086_odbc_roundtrip.sql b/dbms/tests/queries/0_stateless/01086_odbc_roundtrip.sql similarity index 100% rename from dbms/tests/queries/bugs/01086_odbc_roundtrip.sql rename to dbms/tests/queries/0_stateless/01086_odbc_roundtrip.sql diff --git a/docker/test/stateless/Dockerfile b/docker/test/stateless/Dockerfile index 3e0b346184b..b631352d1d8 100644 --- a/docker/test/stateless/Dockerfile +++ b/docker/test/stateless/Dockerfile @@ -1,7 +1,7 @@ # docker build -t yandex/clickhouse-stateless-test . FROM yandex/clickhouse-deb-builder -ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.3.20200115/clickhouse-odbc-1.1.3-Linux.tar.gz" +ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.4.20200302/clickhouse-odbc-1.1.4-Linux.tar.gz" RUN apt-get update -y \ && env DEBIAN_FRONTEND=noninteractive \ From 42fc5efb52182a23772d5d5785bd51291189082b Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 3 Mar 2020 13:50:47 +0300 Subject: [PATCH 056/712] boop From 74d7d95d3414c35c944ba3ccc50643f9ac2c8f73 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 3 Mar 2020 15:01:25 +0300 Subject: [PATCH 057/712] fixup --- dbms/tests/performance/array_element.xml | 6 +++--- dbms/tests/performance/base64.xml | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/dbms/tests/performance/array_element.xml b/dbms/tests/performance/array_element.xml index 1040c33ddbf..f4a33810fdd 100644 --- a/dbms/tests/performance/array_element.xml +++ b/dbms/tests/performance/array_element.xml @@ -8,7 +8,7 @@ - SELECT count() FROM numbers(1000000) WHERE NOT ignore([[1], [2]][number % 2 + 2]) - SELECT count() FROM numbers(1000000) WHERE NOT ignore([[], [2]][number % 2 + 2]) - SELECT count() FROM numbers(1000000) WHERE NOT ignore([[], []][number % 2 + 2]) + SELECT count() FROM numbers(10000000) WHERE NOT ignore([[1], [2]][number % 2 + 2]) + SELECT count() FROM numbers(10000000) WHERE NOT ignore([[], [2]][number % 2 + 2]) + SELECT count() FROM numbers(10000000) WHERE NOT ignore([[], []][number % 2 + 2]) diff --git a/dbms/tests/performance/base64.xml b/dbms/tests/performance/base64.xml index dbf8e0dc981..232b4d20ba3 100644 --- a/dbms/tests/performance/base64.xml +++ b/dbms/tests/performance/base64.xml @@ -24,7 +24,6 @@ table numbers(10000000) - numbers_mt(100000000) From f1336592857ae651973db5207569a81086107b8d Mon Sep 17 00:00:00 2001 From: Sergei Shtykov Date: Tue, 3 Mar 2020 17:20:08 +0300 Subject: [PATCH 058/712] CLICKHOUSEDOCS-548: Added some adopters to the list. --- docs/en/introduction/adopters.md | 139 +++++++++++++++++- .../query_language/agg_functions/reference.md | 4 +- .../query_language/agg_functions/reference.md | 22 +-- 3 files changed, 151 insertions(+), 14 deletions(-) diff --git a/docs/en/introduction/adopters.md b/docs/en/introduction/adopters.md index 1cc85c3f881..edfc774f7b6 100644 --- a/docs/en/introduction/adopters.md +++ b/docs/en/introduction/adopters.md @@ -10,22 +10,159 @@ | [Appsflyer](https://www.appsflyer.com) | Mobile analytics | Main product | — | — | [Talk in Russian, July 2019](https://www.youtube.com/watch?v=M3wbRlcpBbY) | | [Badoo](https://badoo.com) | Dating | Timeseries | — | — | [Slides in Russian, December 2019](https://presentations.clickhouse.tech/meetup38/forecast.pdf) | | [Bloomberg](https://www.bloomberg.com/) | Finance, Media | Monitoring | 102 servers | — | [Slides, May 2018](https://www.slideshare.net/Altinity/http-analytics-for-6m-requests-per-second-using-clickhouse-by-alexander-bocharov) | +| [CARTO](https://carto.com/) | Business Intelligence | Geo analytics | — | — | [Geospatial processing with Clickhouse](https://carto.com/blog/geospatial-processing-with-clickhouse/) | | [CERN](http://public.web.cern.ch/public/) | Research | Experiment | — | — | [Press release, April 2012](https://www.yandex.com/company/press_center/press_releases/2012/2012-04-10/) | -| [Cisco](http://public.web.cern.ch/public/) | Networking | Traffic analysis | — | — | [Lightning talk, October 2019](https://youtu.be/-hI1vDR2oPY?t=5057) | +| [Cisco](http://cisco.com/) | Networking | Traffic analysis | — | — | [Lightning talk, October 2019](https://youtu.be/-hI1vDR2oPY?t=5057) | | [Citadel Securities](https://www.citadelsecurities.com/) | Finance | — | — | — | [Contribution, March 2019](https://github.com/ClickHouse/ClickHouse/pull/4774) | +| [Citymobil](https://city-mobil.ru) | Taxi | Analytics | — | — | [Blog Post in Russian, March 2020](https://habr.com/en/company/citymobil/blog/490660/) | | [ContentSquare](https://contentsquare.com) | Web analytics | Main product | — | — | [Blog post in French, November 2018](http://souslecapot.net/2018/11/21/patrick-chatain-vp-engineering-chez-contentsquare-penser-davantage-amelioration-continue-que-revolution-constante/) | | [Cloudflare](https://cloudflare.com) | CDN | Traffic analysis | 36 servers | — | [Blog post, May 2017](https://blog.cloudflare.com/how-cloudflare-analyzes-1m-dns-queries-per-second/), [Blog post, March 2018](https://blog.cloudflare.com/http-analytics-for-6m-requests-per-second-using-clickhouse/) | +| [Corunet](https://coru.net/) | Analytics | Main product | — | — | [Slides in English, April 2019 ](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup21/predictive_models.pdf) | +| [Deutsche Bank](db.com) | Finance | BI Analytics | — | — | [Slides in English, October 2019](https://bigdatadays.ru/wp-content/uploads/2019/10/D2-H3-3_Yakunin-Goihburg.pdf) | | [Exness](https://www.exness.com) | Trading | Metrics, Logging | — | — | [Talk in Russian, May 2019](https://youtu.be/_rpU-TvSfZ8?t=3215) | | [Geniee](https://geniee.co.jp) | Ad network | Main product | — | — | [Blog post in Japanese, July 2017](https://tech.geniee.co.jp/entry/2017/07/20/160100) | +| [Idealista](www.idealista.com) | Real Estate | Analytics | — | — | [Blog Post in English, April 2019](https://clickhouse.yandex/blog/en/clickhouse-meetup-in-madrid-on-april-2-2019) | +| [Kontur](https://kontur.ru) | Software Development | Metrics | — | — | [Talk in Russian, November 2018](https://www.youtube.com/watch?v=U4u4Bd0FtrY) | | [LifeStreet](https://cloudflare.com) | Ad network | Main product | — | — | [Blog post in Russian, February 2017](https://habr.com/en/post/322620/) | +| [Mail.ru Cloud Solutions](https://mcs.mail.ru/) | Cloud services | Main product | — | — | [Running ClickHouse Instance, in Russian](https://mcs.mail.ru/help/db-create/clickhouse#) | +| [MGID](https://www.mgid.com/) | Ad network | Web-analytics | --- | --- | [Our experience in implementing analytical DBMS ClickHouse, in Russian](http://gs-studio.com/news-about-it/32777----clickhouse---c) | | [Qrator](https://qrator.net) | DDoS protection | Main product | — | — | [Blog Post, March 2019](https://blog.qrator.net/en/clickhouse-ddos-mitigation_37/) | | [Tencent](https://www.tencent.com) | Messaging | Logging | — | — | [Talk in Chinese, November 2019](https://youtu.be/T-iVQRuw-QY?t=5050) | | [S7 Airlines](https://www.s7.ru) | Airlines | Metrics, Logging | — | — | [Talk in Russian, March 2019](https://www.youtube.com/watch?v=nwG68klRpPg&t=15s) | +| [scireum GmbH](https://www.scireum.de/) | e-Commerce | ??? | — | — | [Talk in German, February 2020](https://www.youtube.com/watch?v=7QWAn5RbyR4) | | [Spotify](https://www.spotify.com) | Music | Experimentation | — | — | [Slides, July 2018](https://www.slideshare.net/glebus/using-clickhouse-for-experimentation-104247173) | | [Uber](https://www.uber.com) | Taxi | Logging | — | — | [Slides, February 2020](https://presentations.clickhouse.tech/meetup40/ml.pdf) | | [Yandex Cloud](https://cloud.yandex.ru/services/managed-clickhouse) | Public Cloud | Main product | — | — | [Talk in Russian, December 2019](https://www.youtube.com/watch?v=pgnak9e_E0o) | | [Yandex DataLens](https://cloud.yandex.ru/services/datalens) | Business Intelligence | Main product | — | — | [Slides in Russian, December 2019](https://presentations.clickhouse.tech/meetup38/datalens.pdf) | +| [Yandex Market](https://market.yandex.ru/) | e-Commerce | Metrics, Logging | — | — | [Talk in Russian, January 2019](https://youtu.be/_l1qP0DyBcA?t=478) | | [Yandex Metrica](https://metrica.yandex.com) | Web analytics | Main product | 360 servers in one cluster, 1862 servers in one department | 66.41 PiB / 5.68 PiB | [Slides, February 2020](https://presentations.clickhouse.tech/meetup40/introduction/#13) | +| [ЦВТ](https://htc-cs.ru/) | Software Development | Metrics, Logging | — | — | [Blog Post, March 2019, in Russian](https://vc.ru/dev/62715-kak-my-stroili-monitoring-na-prometheus-clickhouse-i-elk) | +| --- | --- | --- | --- | --- | --- | +| --- | --- | --- | --- | --- | --- | + +### Not checked mentions + +- Bioinformatics - evolutionary genetics: +https://github.com/msestak/FindOrigin + +"We are exploring evolution of novel genes in genomes because if seems that genomes are far from being static as previously believed and what actually happens is that new genes are constantly being added and old genes are lost." + +- Search engine and analytics for Bitcoin transactions: +https://blockchair.com/ + +"We have quite large tables on just single server and everything works really fast &mdash with any filters and sorting everything is processed just instantly." + +- https://www.octonica.ru/ презентация https://github.com/ClickHouse/clickhouse-presentations/blob/master/database_saturday_2018_2/octonica/meetup.pptx + + + +### Stash + +- [СБИС, Тензор](https://sbis.ru/about), Analytics. В своих вакансиях регулярно указывают ClickHouse + есть задание для студентов 2018 https://tensor.ru/stipend2018/projects Active Manager - Анализатор активности на странице, где подразумевается использование ClickHouse. + +- Компания Republer https://republer.com/ О том, что они используют ClickHouse видно из вакансии https://telegram.me/nodejs_jobs_feed/607 + +- Kaspersky заявлен в нескольких презентациях, Однако живых публичных источников найти не получается. Поисковики приводят на сайт вакансий Касперского, но в явном виде ClickHouse там нигде не появляется. Есть https://careers.kaspersky.ru/tech/, поиск, на котором приводит на вакансию девопса, при этом в описании вакансии ClickHouse не упоминается. Вот такая штука есть https://sudonull.com/post/76060-The-second-mitap-of-the-Rust-community-in-Kaspersky-Lab-Kaspersky-Labs-blog + +- Мегафон. На сайтах вакансий от него есть несколько вакансий с ClickHouse типа https://ekaterinburg.hh.ru/vacancy/35891497?utm_source=jooble.org&utm_medium=meta&utm_campaign=RU_paid_cpc_applicant_feed_magic1 и есть вот такая хрень https://newprolab.com/ru/dataengineer-megafon, с которой непонятно, что делать. Вакансии не кажутся хоть сколько-нибудь надёжным источником, поскольку сейчас есть - завтра нет. Как у касперского. + + +- [Quantrum.Team](https://quantrum.me). Непонятное комьюнити трейдеров, один из которых решил попробовать ClickHouse. https://quantrum.me/1709-clickhouse-i-python-dlya-xraneniya-istorii-cen/ + +- https://severalnines.com/database-blog/hybrid-oltpanalytics-database-workloads-replicating-mysql-data-clickhouse какая-то компания типа альтинити видимо. + +- В презентациях есть Wikimedia Foundation, но реальных упоминаний об этом не нашел. Есть какой-то ответ в блоге https://www.mail-archive.com/analytics@lists.wikimedia.org/msg03619.html, но он не указывает прямо на использование ClickHouse. Вот этот вот указывает, но я с ходу не разобрал, что за источник вообще такой https://phabricator.wikimedia.org/T150343 + +- [Mercadona](mercadona.com) не нашел ни единой связи с ClickHouse. + +- [Zara](zara.com) не нашел связи с ClickHouse. + +- ByteDance, единственное прямое доказательство было ссылкой на уже не существующую вакансию. + [Original article](https://clickhouse.tech/docs/en/introduction/adopters/) + + + + +- bdtc_2019 ++ cern (Mail.ru, MGID,) +- cpp_russia_2019 +- cpp_sprint_2019 +- data_at_scale +- database_saturday_2018 +- database_saturday_2018_2/pipeline ++ database_saturday_2018_2/octonica +- database_saturday_2019 ++ dataops_2019 (CARTO, Mercadona, Zara, Idealista, Corunet, ... Cloudflare, Spotify, Amadeus, Bloomberg, Cisco, Deutsche Bank, Tencent, ByteDance) +drafts yandex/ClickHouse -> ClickHouse/ClickHouse, part 2 5 months ago +evolution Correction on KuaiShou company name 3 months ago +group_by Changed tabs to spaces in code [#CLICKHOUSE-3]. 3 years ago +hash_tables yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago +highload2016 Added historical presentation from HighLoad 2016 11 days ago +highload2017 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago +highload2018 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago +highload2019 Added presentation from HighLoad++ 2019 4 months ago +highload_siberia_2018 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago +highload_siberia_2019 Added another presentation from HighLoad Siberia 2019 8 months ago +highload_spb_2019 Added presentation from Saint Highload 2019 11 months ago +hse_2019 Added one more presentation from HSE 8 months ago +internals add sources for the internals presentation 2 years ago +it_rzn_2019 Updated presentation from IT Rzn 3 months ago +meetup10 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago +meetup11 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago +meetup12 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago +meetup13 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago +meetup14 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago +meetup15 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago +meetup16 Added all presentations from Summer Berlin Meetup 2 years ago +meetup17 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago +meetup18 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago +meetup19 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago +meetup20 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago +meetup21 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago +meetup22 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago +meetup23 Added presentations from ClickHouse Meetup in San Francisco 9 months ago +meetup24 yandex/ClickHouse -> ClickHouse/ClickHouse, part 2 5 months ago +meetup25 Add lost slides from Novosibirsk 8 months ago +meetup26 add one more slide deck from Minsk meetup 8 months ago +meetup27 Added more presentations from 27th meetup. 7 months ago +meetup28 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago +meetup29 Merge branch 'master' of github.com:ClickHouse/clickhouse-presentations 5 months ago +meetup3 Presentation: added analysts part [#CLICKHOUSE-2]. 3 years ago +meetup30 Add more slides from Paris Meetup 4 months ago +meetup31 Correction on KuaiShou company name 3 months ago +meetup32 Correction on KuaiShou company name 3 months ago +meetup33 Correction on KuaiShou company name 3 months ago +meetup34 Add tokyo meetup 3 months ago +meetup35 Added half of presentations from ClickHouse Instanbul Meetup 3 months ago +meetup36/new_features Added a presentation from 36th ClickHouse Meetup 3 months ago +meetup37 Moved presentation from 37th ClickHouse Meetup 3 months ago +meetup38 Remaining Moscow meetup slides 3 months ago +meetup39 Remaining slides from SF meetup 14 days ago +meetup4 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago +meetup40 More slides from NYC meetup 19 days ago +meetup5 Added presentation from 5th meetup [#CLICKHOUSE-3]. 3 years ago +meetup6 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago +meetup7 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago +meetup9 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago +misc Added resized images for "evolution" article. 3 years ago +percona2017 percona2017 3 years ago +percona2018 Added part of presentations from Percona 2018 and Sunnyvale Meetup 2 years ago +percona2019 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago +percona_europe_2017 add presentation from Percona Europe 2017 2 years ago +percona_europe_2018 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago +percona_europe_2019 Merge branch 'master' of github.com:ClickHouse/clickhouse-presentations 5 months ago +pgday2017 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago +rit2017 add sources for RIT++2017 3 years ago +rit2018 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago +rit2019 Add rit2019 presentation 8 months ago +roadmap2018 Added roadmap for 2018..2019 15 months ago +shad2017 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago +tbd yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago +tutorials Create catboost_with_clickhouse_en.md 2 years ago +unknown_developers_reissue Added slightly modified version of "Unknown developers" presentation 15 months ago +uwdc yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago +yatalks_2019 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago +yatalks_2019_moscow Fixed error in presentation from YaTalks 2019 in Moscow 3 months ago diff --git a/docs/en/query_language/agg_functions/reference.md b/docs/en/query_language/agg_functions/reference.md index 725ef3a4f62..7c099c26580 100644 --- a/docs/en/query_language/agg_functions/reference.md +++ b/docs/en/query_language/agg_functions/reference.md @@ -1034,7 +1034,7 @@ Alias: `medianExactWeighted`. - `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` value in the range of `[0.01, 0.99]`. Default value: 0.5. At `level=0.5` the function calculates [median](https://en.wikipedia.org/wiki/Median). - `expr` — Expression over the column values resulting in numeric [data types](../../data_types/index.md#data_types), [Date](../../data_types/date.md) or [DateTime](../../data_types/datetime.md). -- `weight` — Column with weights of sequence elements. Weight is a number of value occurrences. +- `weight` — Column with weights of sequence members. Weight is a number of value occurrences. **Returned value** @@ -1300,7 +1300,7 @@ Result: ## quantileTDigestWeighted {#quantiletdigestweighted} -Computes an approximate [quantile](https://en.wikipedia.org/wiki/Quantile) of a numeric data sequence using the [t-digest](https://github.com/tdunning/t-digest/blob/master/docs/t-digest-paper/histo.pdf) algorithm. The function takes into account the weight of each sequence number. The maximum error is 1%. Memory consumption is `log(n)`, where `n` is a number of values. +Computes an approximate [quantile](https://en.wikipedia.org/wiki/Quantile) of a numeric data sequence using the [t-digest](https://github.com/tdunning/t-digest/blob/master/docs/t-digest-paper/histo.pdf) algorithm. The function takes into account the weight of each sequence member. The maximum error is 1%. Memory consumption is `log(n)`, where `n` is a number of values. The performance of the function is lower than performance of [quantile](#quantile) or [quantileTiming](#quantiletiming). In terms of the ratio of State size to precision, this function is much better than `quantile`. diff --git a/docs/ru/query_language/agg_functions/reference.md b/docs/ru/query_language/agg_functions/reference.md index 0367e15ba3a..f4ee9cffad2 100644 --- a/docs/ru/query_language/agg_functions/reference.md +++ b/docs/ru/query_language/agg_functions/reference.md @@ -860,7 +860,7 @@ FROM t quantile(level)(expr) ``` -Альяс: `median`. +Алиас: `median`. **Параметры** @@ -870,7 +870,7 @@ quantile(level)(expr) **Возвращаемое значение** -- Приблизительную квантиль заданного уровня. +- Приблизительный квантиль заданного уровня. Тип: @@ -926,7 +926,7 @@ SELECT quantile(val) FROM t quantileDeterministic(level)(expr, determinator) ``` -Альяс: `medianDeterministic`. +Алиас: `medianDeterministic`. **Параметры** @@ -936,7 +936,7 @@ quantileDeterministic(level)(expr, determinator) **Возвращаемое значение** -- Приблизительную квантиль заданного уровня. +- Приблизительный квантиль заданного уровня. Тип: @@ -993,7 +993,7 @@ SELECT quantileDeterministic(val, 1) FROM t quantileExact(level)(expr) ``` -Альяс: `medianExact`. +Алиас: `medianExact`. **Параметры** @@ -1046,7 +1046,7 @@ SELECT quantileExact(number) FROM numbers(10) quantileExactWeighted(level)(expr, weight) ``` -Альяс: `medianExactWeighted`. +Алиас: `medianExactWeighted`. **Параметры** @@ -1110,7 +1110,7 @@ SELECT quantileExactWeighted(n, val) FROM t quantileTiming(level)(expr) ``` -Альяс: `medianTiming`. +Алиас: `medianTiming`. **Параметры** @@ -1192,7 +1192,7 @@ SELECT quantileTiming(response_time) FROM t quantileTimingWeighted(level)(expr, weight) ``` -Альяс: `medianTimingWeighted`. +Алиас: `medianTimingWeighted`. **Параметры** @@ -1276,7 +1276,7 @@ SELECT quantileTimingWeighted(response_time, weight) FROM t quantileTDigest(level)(expr) ``` -Альяс: `medianTDigest`. +Алиас: `medianTDigest`. **Параметры** @@ -1333,7 +1333,7 @@ SELECT quantileTDigest(number) FROM numbers(10) quantileTDigestWeighted(level)(expr, weight) ``` -Альяс: `medianTDigest`. +Алиас: `medianTDigest`. **Параметры** @@ -1343,7 +1343,7 @@ quantileTDigestWeighted(level)(expr, weight) **Возвращаемое значение** -- Приблизительную квантиль заданного уровня. +- Приблизительный квантиль заданного уровня. Тип: From 273333b437628e79eb75971e252c9a6904ec3324 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 3 Mar 2020 22:53:18 +0300 Subject: [PATCH 059/712] fixes --- dbms/programs/client/Client.cpp | 5 +-- dbms/programs/copier/ClusterCopierApp.cpp | 1 - dbms/programs/local/LocalServer.cpp | 1 - dbms/programs/server/Server.cpp | 1 - dbms/src/Access/AccessRightsContext.cpp | 14 ++++++-- dbms/src/Interpreters/Context.cpp | 32 +++++++++++++++---- dbms/src/Interpreters/Context.h | 8 +++-- dbms/src/Interpreters/DatabaseCatalog.cpp | 8 +++++ .../Interpreters/InterpreterCreateQuery.cpp | 4 +-- dbms/src/Parsers/ASTQueryWithTableAndOutput.h | 2 +- .../src/Storages/LiveView/StorageLiveView.cpp | 1 + ..._transform_query_for_external_database.cpp | 1 - 12 files changed, 57 insertions(+), 21 deletions(-) diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp index a3b0ef97d08..873fc92aac6 100644 --- a/dbms/programs/client/Client.cpp +++ b/dbms/programs/client/Client.cpp @@ -219,13 +219,9 @@ private: configReadClient(config(), home_path); - context.makeGlobalContext(); context.setApplicationType(Context::ApplicationType::CLIENT); context.setQueryParameters(query_parameters); - /// There is no database catalog on client, but we have to initialized this singleton, because context may access it. - DatabaseCatalog::init(&context); - /// settings and limits could be specified in config file, but passed settings has higher priority for (auto && setting : context.getSettingsRef()) { @@ -1710,6 +1706,7 @@ public: ("server_logs_file", po::value(), "put server logs into specified file") ; + context.makeGlobalContext(); context.getSettingsRef().addProgramOptions(main_description); /// Commandline options related to external tables. diff --git a/dbms/programs/copier/ClusterCopierApp.cpp b/dbms/programs/copier/ClusterCopierApp.cpp index b4460413259..c70e79313b9 100644 --- a/dbms/programs/copier/ClusterCopierApp.cpp +++ b/dbms/programs/copier/ClusterCopierApp.cpp @@ -92,7 +92,6 @@ void ClusterCopierApp::mainImpl() auto context = std::make_unique(Context::createGlobal()); context->makeGlobalContext(); - DatabaseCatalog::init(context.get()); SCOPE_EXIT(context->shutdown()); context->setConfig(loaded_config.configuration); diff --git a/dbms/programs/local/LocalServer.cpp b/dbms/programs/local/LocalServer.cpp index eda7a45ab12..d5e08382424 100644 --- a/dbms/programs/local/LocalServer.cpp +++ b/dbms/programs/local/LocalServer.cpp @@ -187,7 +187,6 @@ try * Otherwise, metadata of temporary File(format, EXPLICIT_PATH) tables will pollute metadata/ directory; * if such tables will not be dropped, clickhouse-server will not be able to load them due to security reasons. */ - DatabaseCatalog::init(context.get()); std::string default_database = config().getString("default_database", "_local"); DatabaseCatalog::instance().attachDatabase(default_database, std::make_shared(default_database)); context->setCurrentDatabase(default_database); diff --git a/dbms/programs/server/Server.cpp b/dbms/programs/server/Server.cpp index f1e750c3d61..9d5ecb7c608 100644 --- a/dbms/programs/server/Server.cpp +++ b/dbms/programs/server/Server.cpp @@ -550,7 +550,6 @@ int Server::main(const std::vector & /*args*/) try { - DatabaseCatalog::init(global_context.get()); loadMetadataSystem(*global_context); /// After attaching system databases we can initialize system log. global_context->initializeSystemLogs(); diff --git a/dbms/src/Access/AccessRightsContext.cpp b/dbms/src/Access/AccessRightsContext.cpp index 471db279751..9e781cbe280 100644 --- a/dbms/src/Access/AccessRightsContext.cpp +++ b/dbms/src/Access/AccessRightsContext.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -435,8 +436,12 @@ boost::shared_ptr AccessRightsContext::calculateResultAccess | AccessType::DROP_ROLE | AccessType::DROP_POLICY | AccessType::DROP_QUOTA | AccessType::ROLE_ADMIN; /// Anyone has access to the "system" database. - if (!result.isGranted(AccessType::SELECT, "system")) - result.grant(AccessType::SELECT, "system"); + if (!result.isGranted(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE)) + result.grant(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE); + + /// User has access to temporary or external table if such table was resolved in session or query context + if (!result.isGranted(AccessType::SELECT, DatabaseCatalog::TEMPORARY_DATABASE)) + result.grant(AccessType::SELECT, DatabaseCatalog::TEMPORARY_DATABASE); if (readonly_) result.fullRevoke(write_table_access | all_dcl | AccessType::SYSTEM | AccessType::KILL); @@ -453,6 +458,11 @@ boost::shared_ptr AccessRightsContext::calculateResultAccess /// For example, for readonly = 2 - allowed. result.fullRevoke(AccessType::CREATE_TEMPORARY_TABLE | AccessType::TABLE_FUNCTIONS); } + else if (readonly_ == 2) + { + /// Allow INSERT into temporary tables + result.grant(AccessType::INSERT, DatabaseCatalog::TEMPORARY_DATABASE); + } if (!allow_introspection_) result.fullRevoke(AccessType::INTROSPECTION); diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index f2bf887ac6a..aad9957d401 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -106,16 +106,23 @@ struct TemporaryTableHolder : boost::noncopyable const ASTPtr & query = {}) : context(context_), external_tables(external_tables_) { - if (query) + ASTCreateQuery * create = dynamic_cast(query.get()); + if (create) { - ASTCreateQuery & create = dynamic_cast(*query); - if (create.uuid == UUIDHelpers::Nil) - create.uuid = UUIDHelpers::generateV4(); - id = create.uuid; + if (create->uuid == UUIDHelpers::Nil) + create->uuid = UUIDHelpers::generateV4(); + id = create->uuid; } else id = UUIDHelpers::generateV4(); - external_tables.createTable(context, "_data_" + toString(id), table, query); + String global_name = "_data_" + toString(id); + external_tables.createTable(context, global_name, table, query ? query->clone() : query); + if (create) + { + create->database = DatabaseCatalog::TEMPORARY_DATABASE; + create->table = global_name; + create->uuid = id; + } } TemporaryTableHolder(TemporaryTableHolder && other) @@ -2056,9 +2063,22 @@ StorageID Context::resolveStorageIDImpl(StorageID storage_id, StorageNamespace w if (look_for_external_table) { + /// Global context should not contain temporary tables + assert(global_context != this); + + /// Firstly look for temporary table in current context auto it = external_tables_mapping.find(storage_id.getTableName()); if (it != external_tables_mapping.end()) return it->second->getGlobalTableID(); + + /// If not found and current context was created from some session context, look for temporay table in session context + if (session_context && session_context != this) + { + const auto & external_tables = session_context->external_tables_mapping; + it = external_tables.find(storage_id.getTableName()); + if (it != external_tables.end()) + return it->second->getGlobalTableID(); + } } if (in_current_database) diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index ef9e1cf646a..80c14bf73f1 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -297,7 +297,7 @@ public: ResolveCurrentDatabase = 2u, /// Use current database ResolveOrdinary = ResolveGlobal | ResolveCurrentDatabase, /// If database name is not specified, use current database ResolveExternal = 4u, /// Try get external table - ResolveExternalOrGlobal = ResolveGlobal | ResolveExternal, /// If external table doesn't exist, database name must be specifies + //ResolveExternalOrGlobal = ResolveGlobal | ResolveExternal, /// If external table doesn't exist, database name must be specifies ResolveAll = ResolveExternal | ResolveOrdinary /// If database name is not specified, try get external table, /// if external table not found use current database. }; @@ -422,7 +422,11 @@ public: void makeQueryContext() { query_context = this; } void makeSessionContext() { session_context = this; } - void makeGlobalContext() { global_context = this; } + void makeGlobalContext() + { + global_context = this; + DatabaseCatalog::init(this); + } const Settings & getSettingsRef() const { return settings; } Settings & getSettingsRef() { return settings; } diff --git a/dbms/src/Interpreters/DatabaseCatalog.cpp b/dbms/src/Interpreters/DatabaseCatalog.cpp index 05551a88d1c..9e9f40a70fe 100644 --- a/dbms/src/Interpreters/DatabaseCatalog.cpp +++ b/dbms/src/Interpreters/DatabaseCatalog.cpp @@ -19,6 +19,7 @@ namespace ErrorCodes extern const int DATABASE_ALREADY_EXISTS; extern const int DDL_GUARD_IS_ACTIVE; extern const int DATABASE_NOT_EMPTY; + extern const int DATABASE_ACCESS_DENIED; } @@ -80,6 +81,13 @@ StoragePtr DatabaseCatalog::getTableImpl(const StorageID & table_id, const Conte return {}; } + //if (table_id.database_name == TEMPORARY_DATABASE && !table_id.hasUUID()) + //{ + // if (exception) + // exception->emplace("Direct access to `" + String(TEMPORARY_DATABASE) + "` database is not allowed.", ErrorCodes::DATABASE_ACCESS_DENIED); + // return {}; + //} + //if (table_id.hasUUID()) //{ // auto db_and_table = tryGetByUUID(table_id.uuid); diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp index 4d2579cd360..778325e8961 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp @@ -583,7 +583,7 @@ bool InterpreterCreateQuery::doCreateTable(const ASTCreateQuery & create, DatabasePtr database; - const String & table_name = create.table; + String table_name = create.table; bool need_add_to_database = !create.temporary; if (need_add_to_database) { @@ -662,7 +662,7 @@ BlockIO InterpreterCreateQuery::fillTableIfNeeded(const ASTCreateQuery & create) && !create.is_view && !create.is_live_view && (!create.is_materialized_view || create.is_populate)) { auto insert = std::make_shared(); - insert->table_id = context.getSessionContext().resolveStorageID({create.database, create.table, create.uuid}, Context::ResolveExternalOrGlobal); + insert->table_id = {create.database, create.table, create.uuid}; insert->select = create.select->clone(); if (create.temporary && !context.getSessionContext().hasQueryContext()) diff --git a/dbms/src/Parsers/ASTQueryWithTableAndOutput.h b/dbms/src/Parsers/ASTQueryWithTableAndOutput.h index 975d55e3e65..03f5fa7bf22 100644 --- a/dbms/src/Parsers/ASTQueryWithTableAndOutput.h +++ b/dbms/src/Parsers/ASTQueryWithTableAndOutput.h @@ -16,7 +16,7 @@ class ASTQueryWithTableAndOutput : public ASTQueryWithOutput public: String database; String table; - UUID uuid; + UUID uuid = UUIDHelpers::Nil; bool temporary{false}; protected: diff --git a/dbms/src/Storages/LiveView/StorageLiveView.cpp b/dbms/src/Storages/LiveView/StorageLiveView.cpp index 34a9ec65408..d7567eaafe0 100644 --- a/dbms/src/Storages/LiveView/StorageLiveView.cpp +++ b/dbms/src/Storages/LiveView/StorageLiveView.cpp @@ -136,6 +136,7 @@ Pipes StorageLiveView::blocksToPipes(BlocksPtrs blocks, Block & sample_block) /// Complete query using input streams from mergeable blocks BlockInputStreamPtr StorageLiveView::completeQuery(Pipes pipes) { + //FIXME it's dangerous to create Context on stack auto block_context = std::make_unique(global_context); block_context->makeQueryContext(); diff --git a/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp b/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp index 1fd4dc6fb41..856de3d89ca 100644 --- a/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp +++ b/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp @@ -33,7 +33,6 @@ struct State DatabasePtr database = std::make_shared("test"); database->attachTable("table", StorageMemory::create(StorageID("test", "table"), ColumnsDescription{columns}, ConstraintsDescription{})); context.makeGlobalContext(); - DatabaseCatalog::init(&context); DatabaseCatalog::instance().attachDatabase("test", database); context.setCurrentDatabase("test"); } From 2d5ed7832babe343b24a9466c26b5c7de1673e4a Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 4 Mar 2020 23:29:52 +0300 Subject: [PATCH 060/712] remove tryGetTable --- dbms/programs/server/TCPHandler.cpp | 3 +- dbms/src/Core/QualifiedTableName.h | 1 + .../DataStreams/RemoteBlockInputStream.cpp | 2 +- dbms/src/DataStreams/RemoteBlockInputStream.h | 4 +- dbms/src/Interpreters/ActionLocksManager.cpp | 8 +- dbms/src/Interpreters/ActionLocksManager.h | 5 +- dbms/src/Interpreters/ActionsVisitor.cpp | 4 +- .../ClusterProxy/SelectStreamFactory.cpp | 11 +- .../ClusterProxy/SelectStreamFactory.h | 4 +- dbms/src/Interpreters/Context.cpp | 10 +- dbms/src/Interpreters/Context.h | 3 +- dbms/src/Interpreters/DatabaseCatalog.cpp | 21 ++- dbms/src/Interpreters/DatabaseCatalog.h | 4 +- dbms/src/Interpreters/ExpressionAnalyzer.cpp | 4 +- .../InJoinSubqueriesPreprocessor.cpp | 4 +- .../Interpreters/InterpreterSystemQuery.cpp | 139 ++++++++---------- .../src/Interpreters/InterpreterSystemQuery.h | 6 +- dbms/src/Storages/LiveView/StorageLiveView.h | 1 - dbms/src/Storages/StorageDistributed.cpp | 2 +- dbms/src/Storages/StorageID.cpp | 7 + dbms/src/Storages/StorageID.h | 7 + .../Storages/getStructureOfRemoteTable.cpp | 2 +- 22 files changed, 134 insertions(+), 118 deletions(-) diff --git a/dbms/programs/server/TCPHandler.cpp b/dbms/programs/server/TCPHandler.cpp index 1b1855b5f3b..3743d921d66 100644 --- a/dbms/programs/server/TCPHandler.cpp +++ b/dbms/programs/server/TCPHandler.cpp @@ -669,7 +669,8 @@ void TCPHandler::processTablesStatusRequest() TablesStatusResponse response; for (const QualifiedTableName & table_name: request.tables) { - StoragePtr table = connection_context.tryGetTable(table_name.database, table_name.table); + auto resolved_id = connection_context.tryResolveStorageID({table_name.database, table_name.table}); + StoragePtr table = DatabaseCatalog::instance().tryGetTable(resolved_id); if (!table) continue; diff --git a/dbms/src/Core/QualifiedTableName.h b/dbms/src/Core/QualifiedTableName.h index 6337cc1ccbf..453d55d85c7 100644 --- a/dbms/src/Core/QualifiedTableName.h +++ b/dbms/src/Core/QualifiedTableName.h @@ -7,6 +7,7 @@ namespace DB { +//TODO replace with StorageID struct QualifiedTableName { std::string database; diff --git a/dbms/src/DataStreams/RemoteBlockInputStream.cpp b/dbms/src/DataStreams/RemoteBlockInputStream.cpp index 50392692461..9d9f629d463 100644 --- a/dbms/src/DataStreams/RemoteBlockInputStream.cpp +++ b/dbms/src/DataStreams/RemoteBlockInputStream.cpp @@ -71,7 +71,7 @@ RemoteBlockInputStream::RemoteBlockInputStream( std::vector connections; if (main_table) { - auto try_results = pool->getManyChecked(timeouts, ¤t_settings, pool_mode, *main_table); + auto try_results = pool->getManyChecked(timeouts, ¤t_settings, pool_mode, main_table.getQualifiedName()); connections.reserve(try_results.size()); for (auto & try_result : try_results) connections.emplace_back(std::move(try_result.entry)); diff --git a/dbms/src/DataStreams/RemoteBlockInputStream.h b/dbms/src/DataStreams/RemoteBlockInputStream.h index 89f4e84f080..783811f2521 100644 --- a/dbms/src/DataStreams/RemoteBlockInputStream.h +++ b/dbms/src/DataStreams/RemoteBlockInputStream.h @@ -54,7 +54,7 @@ public: /// Specify how we allocate connections on a shard. void setPoolMode(PoolMode pool_mode_) { pool_mode = pool_mode_; } - void setMainTable(QualifiedTableName main_table_) { main_table = std::move(main_table_); } + void setMainTable(StorageID main_table_) { main_table = std::move(main_table_); } /// Sends query (initiates calculation) before read() void readPrefix() override; @@ -148,7 +148,7 @@ private: std::atomic got_unknown_packet_from_replica { false }; PoolMode pool_mode = PoolMode::GET_MANY; - std::optional main_table; + StorageID main_table = StorageID::createEmpty(); Logger * log = &Logger::get("RemoteBlockInputStream"); }; diff --git a/dbms/src/Interpreters/ActionLocksManager.cpp b/dbms/src/Interpreters/ActionLocksManager.cpp index 023129f7cdc..9ab220d0a84 100644 --- a/dbms/src/Interpreters/ActionLocksManager.cpp +++ b/dbms/src/Interpreters/ActionLocksManager.cpp @@ -33,9 +33,9 @@ void ActionLocksManager::add(StorageActionBlockType action_type) forEachTable(global_context, [&](const StoragePtr & table) { add(table, action_type); }); } -void ActionLocksManager::add(const String & database_name, const String & table_name, StorageActionBlockType action_type) +void ActionLocksManager::add(const StorageID & table_id, StorageActionBlockType action_type) { - if (auto table = global_context.tryGetTable(database_name, table_name)) + if (auto table = DatabaseCatalog::instance().tryGetTable(table_id)) add(table, action_type); } @@ -58,9 +58,9 @@ void ActionLocksManager::remove(StorageActionBlockType action_type) storage_elem.second.erase(action_type); } -void ActionLocksManager::remove(const String & database_name, const String & table_name, StorageActionBlockType action_type) +void ActionLocksManager::remove(const StorageID & table_id, StorageActionBlockType action_type) { - if (auto table = global_context.tryGetTable(database_name, table_name)) + if (auto table = DatabaseCatalog::instance().tryGetTable(table_id)) remove(table, action_type); } diff --git a/dbms/src/Interpreters/ActionLocksManager.h b/dbms/src/Interpreters/ActionLocksManager.h index 0547ac02ed2..1e26da0e95e 100644 --- a/dbms/src/Interpreters/ActionLocksManager.h +++ b/dbms/src/Interpreters/ActionLocksManager.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -23,13 +24,13 @@ public: /// Adds new locks for each table void add(StorageActionBlockType action_type); /// Add new lock for a table if it has not been already added - void add(const String & database_name, const String & table_name, StorageActionBlockType action_type); + void add(const StorageID & table_id, StorageActionBlockType action_type); void add(const StoragePtr & table, StorageActionBlockType action_type); /// Remove locks for all tables void remove(StorageActionBlockType action_type); /// Removes a lock for a table if it exists - void remove(const String & database_name, const String & table_name, StorageActionBlockType action_type); + void remove(const StorageID & table_id, StorageActionBlockType action_type); void remove(const StoragePtr & table, StorageActionBlockType action_type); /// Removes all locks of non-existing tables diff --git a/dbms/src/Interpreters/ActionsVisitor.cpp b/dbms/src/Interpreters/ActionsVisitor.cpp index ea13bb57b14..8eec593d663 100644 --- a/dbms/src/Interpreters/ActionsVisitor.cpp +++ b/dbms/src/Interpreters/ActionsVisitor.cpp @@ -642,8 +642,8 @@ SetPtr ActionsMatcher::makeSet(const ASTFunction & node, Data & data, bool no_su /// and the table has the type Set (a previously prepared set). if (identifier) { - DatabaseAndTableWithAlias database_table(*identifier); - StoragePtr table = data.context.tryGetTable(database_table.database, database_table.table); + auto table_id = StorageID::resolveFromAST(right_in_operand, data.context); + StoragePtr table = DatabaseCatalog::instance().tryGetTable(table_id); if (table) { diff --git a/dbms/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/dbms/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp index b6ece8d0176..b6fdee02eb1 100644 --- a/dbms/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp +++ b/dbms/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp @@ -38,7 +38,7 @@ namespace ClusterProxy SelectStreamFactory::SelectStreamFactory( const Block & header_, QueryProcessingStage::Enum processed_stage_, - QualifiedTableName main_table_, + StorageID main_table_, const Scalars & scalars_, bool has_virtual_shard_num_column_, const Tables & external_tables_) @@ -172,7 +172,10 @@ void SelectStreamFactory::createForShard( main_table_storage = table_function_ptr->execute(table_func_ptr, context, table_function_ptr->getName()); } else - main_table_storage = context.tryGetTable(main_table.database, main_table.table); + { + auto resolved_id = context.resolveStorageID(main_table); + main_table_storage = DatabaseCatalog::instance().tryGetTable(resolved_id); + } if (!main_table_storage) /// Table is absent on a local server. @@ -182,7 +185,7 @@ void SelectStreamFactory::createForShard( { LOG_WARNING( &Logger::get("ClusterProxy::SelectStreamFactory"), - "There is no table " << main_table.database << "." << main_table.table + "There is no table " << main_table.getNameForLogs() << " on local replica of shard " << shard_info.shard_num << ", will try remote replicas."); emplace_remote_stream(); } @@ -264,7 +267,7 @@ void SelectStreamFactory::createForShard( if (table_func_ptr) try_results = pool->getManyForTableFunction(timeouts, ¤t_settings, PoolMode::GET_MANY); else - try_results = pool->getManyChecked(timeouts, ¤t_settings, PoolMode::GET_MANY, main_table); + try_results = pool->getManyChecked(timeouts, ¤t_settings, PoolMode::GET_MANY, main_table.getQualifiedName()); } catch (const Exception & ex) { diff --git a/dbms/src/Interpreters/ClusterProxy/SelectStreamFactory.h b/dbms/src/Interpreters/ClusterProxy/SelectStreamFactory.h index ed5afd9f758..daf4dd48b4d 100644 --- a/dbms/src/Interpreters/ClusterProxy/SelectStreamFactory.h +++ b/dbms/src/Interpreters/ClusterProxy/SelectStreamFactory.h @@ -17,7 +17,7 @@ public: SelectStreamFactory( const Block & header_, QueryProcessingStage::Enum processed_stage_, - QualifiedTableName main_table_, + StorageID main_table_, const Scalars & scalars_, bool has_virtual_shard_num_column_, const Tables & external_tables); @@ -41,7 +41,7 @@ public: private: const Block header; QueryProcessingStage::Enum processed_stage; - QualifiedTableName main_table; + StorageID main_table = StorageID::createEmpty(); ASTPtr table_func_ptr; Scalars scalars; bool has_virtual_shard_num_column = false; diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index d96e8fe15fd..d3b2ff25322 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -738,6 +738,8 @@ void Context::checkAccess(const AccessFlags & access, const std::string_view & d void Context::checkAccess(const AccessRightsElement & access) const { return checkAccessImpl(access); } void Context::checkAccess(const AccessRightsElements & access) const { return checkAccessImpl(access); } +void Context::checkAccess(const AccessFlags & access, const StorageID & table_id) const { checkAccessImpl(access, table_id.getDatabaseName(), table_id.getTableName()); } + AccessRightsContextPtr Context::getAccessRights() const { auto lock = getLock(); @@ -857,18 +859,12 @@ StoragePtr Context::getTable(const String & database_name, const String & table_ { auto resolved_id = resolveStorageID(StorageID(database_name, table_name)); std::optional exc; - auto res = DatabaseCatalog::instance().getTableImpl(resolved_id, *this, &exc); + auto res = DatabaseCatalog::instance().getTableImpl(resolved_id, *this, &exc).second; if (!res) throw *exc; return res; } -StoragePtr Context::tryGetTable(const String & database_name, const String & table_name) const -{ - auto resolved_id = tryResolveStorageID(StorageID(database_name, table_name)); - return DatabaseCatalog::instance().getTableImpl(resolved_id, *this, nullptr); -} - void Context::addExternalTable(const String & table_name, const StoragePtr & storage, const ASTPtr & ast) { diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index 4b78aff69a4..6b7c8305741 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -258,6 +258,8 @@ public: void checkAccess(const AccessRightsElement & access) const; void checkAccess(const AccessRightsElements & access) const; + void checkAccess(const AccessFlags & access, const StorageID & table_id) const; + AccessRightsContextPtr getAccessRights() const; RowPolicyContextPtr getRowPolicy() const; @@ -311,7 +313,6 @@ public: const Block & getScalar(const String & name) const; Tables getExternalTables() const; StoragePtr getTable(const String & database_name, const String & table_name) const; - StoragePtr tryGetTable(const String & database_name, const String & table_name) const; void addExternalTable(const String & table_name, const StoragePtr & storage, const ASTPtr & ast = {}); void addScalar(const String & name, const Block & block); bool hasScalar(const String & name) const; diff --git a/dbms/src/Interpreters/DatabaseCatalog.cpp b/dbms/src/Interpreters/DatabaseCatalog.cpp index 9e9f40a70fe..4cca73143f5 100644 --- a/dbms/src/Interpreters/DatabaseCatalog.cpp +++ b/dbms/src/Interpreters/DatabaseCatalog.cpp @@ -72,7 +72,7 @@ DatabaseAndTable DatabaseCatalog::tryGetByUUID(const UUID & uuid) const } -StoragePtr DatabaseCatalog::getTableImpl(const StorageID & table_id, const Context & local_context, std::optional * exception) const +DatabaseAndTable DatabaseCatalog::getTableImpl(const StorageID & table_id, const Context & local_context, std::optional * exception) const { if (!table_id) { @@ -120,7 +120,7 @@ StoragePtr DatabaseCatalog::getTableImpl(const StorageID & table_id, const Conte if (!table && exception) exception->emplace("Table " + table_id.getNameForLogs() + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE); - return table; + return {database, table}; } void DatabaseCatalog::assertDatabaseExists(const String & database_name) const @@ -343,16 +343,21 @@ bool DatabaseCatalog::isDictionaryExist(const StorageID & table_id, const Contex StoragePtr DatabaseCatalog::getTable(const StorageID & table_id) const { - std::optional exc; - auto res = getTableImpl(table_id, *global_context, &exc); - if (!res) - throw *exc; - return res; + return tryGetDatabaseAndTable(table_id).second; } StoragePtr DatabaseCatalog::tryGetTable(const StorageID & table_id) const { - return getTableImpl(table_id, *global_context, nullptr); + return getTableImpl(table_id, *global_context, nullptr).second; +} + +DatabaseAndTable DatabaseCatalog::tryGetDatabaseAndTable(const StorageID & table_id) const +{ + std::optional exc; + auto res = getTableImpl(table_id, *global_context, &exc); + if (!res.second) + throw *exc; + return res; } diff --git a/dbms/src/Interpreters/DatabaseCatalog.h b/dbms/src/Interpreters/DatabaseCatalog.h index 0439e98198e..aadb51c37ca 100644 --- a/dbms/src/Interpreters/DatabaseCatalog.h +++ b/dbms/src/Interpreters/DatabaseCatalog.h @@ -101,7 +101,9 @@ public: StoragePtr getTable(const StorageID & table_id) const; StoragePtr tryGetTable(const StorageID & table_id) const; - StoragePtr getTableImpl(const StorageID & table_id, const Context & local_context, std::optional * exception = nullptr) const; + DatabaseAndTable getDatabaseAndTable(const StorageID & table_id) const { return getTableImpl(table_id, *global_context); } + DatabaseAndTable tryGetDatabaseAndTable(const StorageID & table_id) const; + DatabaseAndTable getTableImpl(const StorageID & table_id, const Context & local_context, std::optional * exception = nullptr) const; void addDependency(const StorageID & from, const StorageID & where); diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp index 4dee6565b52..3339f14ff6c 100644 --- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp +++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp @@ -519,8 +519,8 @@ static JoinPtr tryGetStorageJoin(const ASTTablesInSelectQueryElement & join_elem /// TODO This syntax does not support specifying a database name. if (table_to_join.database_and_table_name) { - DatabaseAndTableWithAlias database_table(table_to_join.database_and_table_name); - StoragePtr table = context.tryGetTable(database_table.database, database_table.table); + auto table_id = StorageID::resolveFromAST(table_to_join.database_and_table_name, context); + StoragePtr table = DatabaseCatalog::instance().tryGetTable(table_id); if (table) { diff --git a/dbms/src/Interpreters/InJoinSubqueriesPreprocessor.cpp b/dbms/src/Interpreters/InJoinSubqueriesPreprocessor.cpp index 149565beadc..14b97390b09 100644 --- a/dbms/src/Interpreters/InJoinSubqueriesPreprocessor.cpp +++ b/dbms/src/Interpreters/InJoinSubqueriesPreprocessor.cpp @@ -26,8 +26,8 @@ namespace StoragePtr tryGetTable(const ASTPtr & database_and_table, const Context & context) { - DatabaseAndTableWithAlias db_and_table(database_and_table); - return context.tryGetTable(db_and_table.database, db_and_table.table); + auto table_id = StorageID::resolveFromAST(database_and_table, context); + return DatabaseCatalog::instance().tryGetTable(table_id); } using CheckShardsAndTables = InJoinSubqueriesPreprocessor::CheckShardsAndTables; diff --git a/dbms/src/Interpreters/InterpreterSystemQuery.cpp b/dbms/src/Interpreters/InterpreterSystemQuery.cpp index b0564fb94d8..4779dc538bd 100644 --- a/dbms/src/Interpreters/InterpreterSystemQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSystemQuery.cpp @@ -119,21 +119,21 @@ AccessType getRequiredAccessType(StorageActionBlockType action_type) throw Exception("Unknown action type: " + std::to_string(action_type), ErrorCodes::LOGICAL_ERROR); } +} /// Implements SYSTEM [START|STOP] -void startStopAction(Context & context, Poco::Logger * log, ASTSystemQuery & query, StorageActionBlockType action_type, bool start) +void InterpreterSystemQuery::startStopAction(StorageActionBlockType action_type, bool start) { auto manager = context.getActionLocksManager(); manager->cleanExpired(); - if (!query.table.empty()) + if (table_id) { - String database = !query.database.empty() ? query.database : context.getCurrentDatabase(); - context.checkAccess(getRequiredAccessType(action_type), database, query.table); + context.checkAccess(getRequiredAccessType(action_type), table_id); if (start) - manager->remove(database, query.table, action_type); + manager->remove(table_id, action_type); else - manager->add(database, query.table, action_type); + manager->add(table_id, action_type); } else { @@ -153,7 +153,7 @@ void startStopAction(Context & context, Poco::Logger * log, ASTSystemQuery & que } } -} + InterpreterSystemQuery::InterpreterSystemQuery(const ASTPtr & query_ptr_, Context & context_) @@ -176,8 +176,8 @@ BlockIO InterpreterSystemQuery::execute() system_context.setSetting("profile", context.getSystemProfileName()); /// Make canonical query for simpler processing - if (!query.table.empty() && query.database.empty()) - query.database = context.getCurrentDatabase(); + if (!query.table.empty()) + table_id = context.resolveStorageID(StorageID(query.database, query.table), Context::ResolveOrdinary); if (!query.target_dictionary.empty() && !query.database.empty()) query.target_dictionary = query.database + "." + query.target_dictionary; @@ -234,46 +234,46 @@ BlockIO InterpreterSystemQuery::execute() system_context.reloadConfig(); break; case Type::STOP_MERGES: - startStopAction(context, log, query, ActionLocks::PartsMerge, false); + startStopAction(ActionLocks::PartsMerge, false); break; case Type::START_MERGES: - startStopAction(context, log, query, ActionLocks::PartsMerge, true); + startStopAction(ActionLocks::PartsMerge, true); break; case Type::STOP_TTL_MERGES: - startStopAction(context, log, query, ActionLocks::PartsTTLMerge, false); + startStopAction(ActionLocks::PartsTTLMerge, false); break; case Type::START_TTL_MERGES: - startStopAction(context, log, query, ActionLocks::PartsTTLMerge, true); + startStopAction(ActionLocks::PartsTTLMerge, true); break; case Type::STOP_MOVES: - startStopAction(context, log, query, ActionLocks::PartsMove, false); + startStopAction(ActionLocks::PartsMove, false); break; case Type::START_MOVES: - startStopAction(context, log, query, ActionLocks::PartsMove, true); + startStopAction(ActionLocks::PartsMove, true); break; case Type::STOP_FETCHES: - startStopAction(context, log, query, ActionLocks::PartsFetch, false); + startStopAction(ActionLocks::PartsFetch, false); break; case Type::START_FETCHES: - startStopAction(context, log, query, ActionLocks::PartsFetch, true); + startStopAction(ActionLocks::PartsFetch, true); break; case Type::STOP_REPLICATED_SENDS: - startStopAction(context, log, query, ActionLocks::PartsSend, false); + startStopAction(ActionLocks::PartsSend, false); break; case Type::START_REPLICATED_SENDS: - startStopAction(context, log, query, ActionLocks::PartsSend, true); + startStopAction(ActionLocks::PartsSend, true); break; case Type::STOP_REPLICATION_QUEUES: - startStopAction(context, log, query, ActionLocks::ReplicationQueue, false); + startStopAction(ActionLocks::ReplicationQueue, false); break; case Type::START_REPLICATION_QUEUES: - startStopAction(context, log, query, ActionLocks::ReplicationQueue, true); + startStopAction(ActionLocks::ReplicationQueue, true); break; case Type::STOP_DISTRIBUTED_SENDS: - startStopAction(context, log, query, ActionLocks::DistributedSend, false); + startStopAction(ActionLocks::DistributedSend, false); break; case Type::START_DISTRIBUTED_SENDS: - startStopAction(context, log, query, ActionLocks::DistributedSend, true); + startStopAction(ActionLocks::DistributedSend, true); break; case Type::SYNC_REPLICA: syncReplica(query); @@ -285,7 +285,7 @@ BlockIO InterpreterSystemQuery::execute() restartReplicas(system_context); break; case Type::RESTART_REPLICA: - if (!tryRestartReplica(query.database, query.table, system_context)) + if (!tryRestartReplica(table_id, system_context)) throw Exception("There is no " + query.database + "." + query.table + " replicated table", ErrorCodes::BAD_ARGUMENTS); break; @@ -311,67 +311,61 @@ BlockIO InterpreterSystemQuery::execute() } -StoragePtr InterpreterSystemQuery::tryRestartReplica(const String & database_name, const String & table_name, Context & system_context) +StoragePtr InterpreterSystemQuery::tryRestartReplica(const StorageID & replica, Context & system_context) { - context.checkAccess(AccessType::RESTART_REPLICA, database_name, table_name); - auto database = DatabaseCatalog::instance().getDatabase(database_name); + context.checkAccess(AccessType::RESTART_REPLICA, replica); - auto table_ddl_guard = DatabaseCatalog::instance().getDDLGuard(database_name, table_name); + auto table_ddl_guard = DatabaseCatalog::instance().getDDLGuard(replica.getDatabaseName(), replica.getTableName()); + auto [database, table] = DatabaseCatalog::instance().tryGetDatabaseAndTable(replica); ASTPtr create_ast; /// Detach actions + if (!table || !dynamic_cast(table.get())) + return nullptr; + + table->shutdown(); { - auto table = DatabaseCatalog::instance().tryGetTable({database_name, table_name}); - - if (!table || !dynamic_cast(table.get())) - return nullptr; - - table->shutdown(); - /// If table was already dropped by anyone, an exception will be thrown auto table_lock = table->lockExclusively(context.getCurrentQueryId()); - create_ast = database->getCreateTableQuery(system_context, table_name); + create_ast = database->getCreateTableQuery(system_context, replica.table_name); - database->detachTable(table_name); + database->detachTable(replica.table_name); } + table.reset(); /// Attach actions - { - /// getCreateTableQuery must return canonical CREATE query representation, there are no need for AST postprocessing - auto & create = create_ast->as(); - create.attach = true; + /// getCreateTableQuery must return canonical CREATE query representation, there are no need for AST postprocessing + auto & create = create_ast->as(); + create.attach = true; - auto columns = InterpreterCreateQuery::getColumnsDescription(*create.columns_list->columns, system_context); - auto constraints = InterpreterCreateQuery::getConstraintsDescription(create.columns_list->constraints); + auto columns = InterpreterCreateQuery::getColumnsDescription(*create.columns_list->columns, system_context); + auto constraints = InterpreterCreateQuery::getConstraintsDescription(create.columns_list->constraints); - StoragePtr table = StorageFactory::instance().get(create, - database->getTableDataPath(create), - system_context, - system_context.getGlobalContext(), - columns, - constraints, - false); + table = StorageFactory::instance().get(create, + database->getTableDataPath(create), + system_context, + system_context.getGlobalContext(), + columns, + constraints, + false); - database->createTable(system_context, table_name, table, create_ast); + database->createTable(system_context, replica.table_name, table, create_ast); - table->startup(); - return table; - } + table->startup(); + return table; } void InterpreterSystemQuery::restartReplicas(Context & system_context) { - std::vector> replica_names; + std::vector replica_names; for (auto & elem : DatabaseCatalog::instance().getDatabases()) { DatabasePtr & database = elem.second; - const String & database_name = elem.first; - for (auto iterator = database->getTablesIterator(system_context); iterator->isValid(); iterator->next()) { if (dynamic_cast(iterator->table().get())) - replica_names.emplace_back(database_name, iterator->name()); + replica_names.emplace_back(iterator->table()->getStorageID()); } } @@ -380,44 +374,39 @@ void InterpreterSystemQuery::restartReplicas(Context & system_context) ThreadPool pool(std::min(size_t(getNumberOfPhysicalCPUCores()), replica_names.size())); for (auto & table : replica_names) - pool.scheduleOrThrowOnError([&]() { tryRestartReplica(table.first, table.second, system_context); }); + pool.scheduleOrThrowOnError([&]() { tryRestartReplica(table, system_context); }); pool.wait(); } -void InterpreterSystemQuery::syncReplica(ASTSystemQuery & query) +void InterpreterSystemQuery::syncReplica(ASTSystemQuery &) { - String database_name = context.resolveDatabase(query.database); - const String & table_name = query.table; - - context.checkAccess(AccessType::SYNC_REPLICA, database_name, table_name); - StoragePtr table = DatabaseCatalog::instance().getTable({database_name, table_name}); + context.checkAccess(AccessType::SYNC_REPLICA, table_id); + StoragePtr table = DatabaseCatalog::instance().getTable(table_id); if (auto storage_replicated = dynamic_cast(table.get())) { LOG_TRACE(log, "Synchronizing entries in replica's queue with table's log and waiting for it to become empty"); if (!storage_replicated->waitForShrinkingQueueSize(0, context.getSettingsRef().receive_timeout.totalMilliseconds())) { - LOG_ERROR(log, "SYNC REPLICA " + database_name + "." + table_name + ": Timed out!"); + LOG_ERROR(log, "SYNC REPLICA " + table_id.getNameForLogs() + ": Timed out!"); throw Exception( - "SYNC REPLICA " + database_name + "." + table_name + ": command timed out! " + "SYNC REPLICA " + table_id.getNameForLogs() + ": command timed out! " "See the 'receive_timeout' setting", ErrorCodes::TIMEOUT_EXCEEDED); } - LOG_TRACE(log, "SYNC REPLICA " + database_name + "." + table_name + ": OK"); + LOG_TRACE(log, "SYNC REPLICA " + table_id.getNameForLogs() + ": OK"); } else - throw Exception("Table " + database_name + "." + table_name + " is not replicated", ErrorCodes::BAD_ARGUMENTS); + throw Exception("Table " + table_id.getNameForLogs() + " is not replicated", ErrorCodes::BAD_ARGUMENTS); } -void InterpreterSystemQuery::flushDistributed(ASTSystemQuery & query) +void InterpreterSystemQuery::flushDistributed(ASTSystemQuery &) { - String database_name = context.resolveDatabase(query.database); - String & table_name = query.table; - context.checkAccess(AccessType::FLUSH_DISTRIBUTED, database_name, table_name); + context.checkAccess(AccessType::FLUSH_DISTRIBUTED, table_id); - if (auto storage_distributed = dynamic_cast(DatabaseCatalog::instance().getTable({database_name, table_name}).get())) + if (auto storage_distributed = dynamic_cast(DatabaseCatalog::instance().getTable(table_id).get())) storage_distributed->flushClusterNodesAllData(); else - throw Exception("Table " + database_name + "." + table_name + " is not distributed", ErrorCodes::BAD_ARGUMENTS); + throw Exception("Table " + table_id.getNameForLogs() + " is not distributed", ErrorCodes::BAD_ARGUMENTS); } diff --git a/dbms/src/Interpreters/InterpreterSystemQuery.h b/dbms/src/Interpreters/InterpreterSystemQuery.h index e10a6692a82..bc7abfed424 100644 --- a/dbms/src/Interpreters/InterpreterSystemQuery.h +++ b/dbms/src/Interpreters/InterpreterSystemQuery.h @@ -3,6 +3,8 @@ #include #include #include +#include +#include namespace Poco { class Logger; } @@ -28,16 +30,18 @@ private: ASTPtr query_ptr; Context & context; Poco::Logger * log = nullptr; + StorageID table_id = StorageID::createEmpty(); /// Will be set up if query contains table name /// Tries to get a replicated table and restart it /// Returns pointer to a newly created table if the restart was successful - StoragePtr tryRestartReplica(const String & database_name, const String & table_name, Context & context); + StoragePtr tryRestartReplica(const StorageID & replica, Context & context); void restartReplicas(Context & system_context); void syncReplica(ASTSystemQuery & query); void flushDistributed(ASTSystemQuery & query); AccessRightsElements getRequiredAccessForDDLOnCluster() const; + void startStopAction(StorageActionBlockType action_type, bool start); }; diff --git a/dbms/src/Storages/LiveView/StorageLiveView.h b/dbms/src/Storages/LiveView/StorageLiveView.h index fbd69a22cde..0d5b2874869 100644 --- a/dbms/src/Storages/LiveView/StorageLiveView.h +++ b/dbms/src/Storages/LiveView/StorageLiveView.h @@ -49,7 +49,6 @@ public: ~StorageLiveView() override; String getName() const override { return "LiveView"; } bool isView() const override { return true; } - StorageID getSelectTableID() const { return select_table_id; } StorageID getBlocksStorageID() const { return StorageID("", getStorageID().table_name + "_blocks"); diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp index 41c00f6c7b3..ad28e749e1b 100644 --- a/dbms/src/Storages/StorageDistributed.cpp +++ b/dbms/src/Storages/StorageDistributed.cpp @@ -367,7 +367,7 @@ Pipes StorageDistributed::read( ? ClusterProxy::SelectStreamFactory( header, processed_stage, remote_table_function_ptr, scalars, has_virtual_shard_num_column, context.getExternalTables()) : ClusterProxy::SelectStreamFactory( - header, processed_stage, QualifiedTableName{remote_database, remote_table}, scalars, has_virtual_shard_num_column, context.getExternalTables()); + header, processed_stage, StorageID{remote_database, remote_table}, scalars, has_virtual_shard_num_column, context.getExternalTables()); UInt64 force = settings.force_optimize_skip_unused_shards; if (settings.optimize_skip_unused_shards) diff --git a/dbms/src/Storages/StorageID.cpp b/dbms/src/Storages/StorageID.cpp index 2605b4e2c31..b4c09ddd440 100644 --- a/dbms/src/Storages/StorageID.cpp +++ b/dbms/src/Storages/StorageID.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB { @@ -47,4 +48,10 @@ void StorageID::assertNotEmpty() const throw Exception("Table name is empty, but database name is not", ErrorCodes::LOGICAL_ERROR); } +StorageID StorageID::resolveFromAST(const ASTPtr & table_identifier_node, const Context & context) +{ + DatabaseAndTableWithAlias database_table(table_identifier_node); + return context.tryResolveStorageID({database_table.database, database_table.table}); +} + } diff --git a/dbms/src/Storages/StorageID.h b/dbms/src/Storages/StorageID.h index b940d3b4180..dd0672dfb0a 100644 --- a/dbms/src/Storages/StorageID.h +++ b/dbms/src/Storages/StorageID.h @@ -2,6 +2,9 @@ #include #include #include +#include + +#include namespace DB { @@ -31,6 +34,8 @@ struct StorageID StorageID(const ASTQueryWithTableAndOutput & query, const Context & local_context); + static StorageID resolveFromAST(const ASTPtr & table_identifier_node, const Context & context); + String getDatabaseName() const { assertNotEmpty(); @@ -73,6 +78,8 @@ struct StorageID /// Avoid implicit construction of empty StorageID. However, it's needed for deferred initialization. static StorageID createEmpty() { return {}; } + QualifiedTableName getQualifiedName() const { return {getDatabaseName(), getTableName()}; } + private: StorageID() = default; }; diff --git a/dbms/src/Storages/getStructureOfRemoteTable.cpp b/dbms/src/Storages/getStructureOfRemoteTable.cpp index 731cbb5fee7..21055c43c57 100644 --- a/dbms/src/Storages/getStructureOfRemoteTable.cpp +++ b/dbms/src/Storages/getStructureOfRemoteTable.cpp @@ -73,7 +73,7 @@ ColumnsDescription getStructureOfRemoteTable( auto input = std::make_shared(shard_info.pool, query, sample_block, new_context); input->setPoolMode(PoolMode::GET_ONE); if (!table_func_ptr) - input->setMainTable(QualifiedTableName{database, table}); + input->setMainTable(StorageID{database, table}); input->readPrefix(); const DataTypeFactory & data_type_factory = DataTypeFactory::instance(); From 156e6246c12b889c43b4be0193d34330cc4b383c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 5 Mar 2020 06:12:07 +0300 Subject: [PATCH 061/712] Better code around sessions --- dbms/src/Interpreters/Context.cpp | 328 ++++++++++++++---------------- dbms/src/Interpreters/Context.h | 13 +- 2 files changed, 160 insertions(+), 181 deletions(-) diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index 1c2d4456d96..0a0ec748c56 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -96,6 +96,160 @@ namespace ErrorCodes } +/// Named sessions. The user could specify session identifier to reuse settings and temporary tables in subsequent requests. +class Sessions +{ +public: + using Key = Context::SessionKey; + + ~Sessions() + { + try + { + { + std::lock_guard lock{mutex}; + quit = true; + } + + cond.notify_one(); + thread.join(); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } + } + + /// Find existing session or create a new. + std::shared_ptr acquireSession(const Key & key, Context & context, std::chrono::steady_clock::duration timeout, bool throw_if_not_found) + { + std::unique_lock lock(mutex); + + auto it = sessions.find(key); + if (it == sessions.end()) + { + if (throw_if_not_found) + throw Exception("Session not found.", ErrorCodes::SESSION_NOT_FOUND); + + /// Create a new session from current context. + auto new_session = std::make_shared(context); + scheduleCloseSession(key, *new_session, timeout, lock); + it = sessions.insert(std::make_pair(key, std::move(new_session))).first; + } + else if (it->second->client_info.current_user != context.client_info.current_user) + { + throw Exception("Session belongs to a different user", ErrorCodes::LOGICAL_ERROR); + } + + const auto & session = it->second; + + if (session->session_is_used) + throw Exception("Session is locked by a concurrent client.", ErrorCodes::SESSION_IS_LOCKED); + + session->session_is_used = true; + return session; + } + + void releaseSession(const Key & key, Context & session, std::chrono::steady_clock::duration timeout) + { + std::unique_lock lock(mutex); + session.session_is_used = false; + scheduleCloseSession(key, session, timeout, lock); + } + +private: + class SessionKeyHash + { + public: + size_t operator()(const Key & key) const + { + SipHash hash; + hash.update(key.first); + hash.update(key.second); + return hash.get64(); + } + }; + + using Container = std::unordered_map, SessionKeyHash>; + using CloseTimes = std::deque>; + Container sessions; + CloseTimes close_times; + std::chrono::steady_clock::duration close_interval = std::chrono::seconds(1); + std::chrono::steady_clock::time_point close_cycle_time = std::chrono::steady_clock::now(); + UInt64 close_cycle = 0; + + void scheduleCloseSession(const Key & key, Context & session, std::chrono::steady_clock::duration timeout, std::unique_lock &) + { + const UInt64 close_index = timeout / close_interval + 1; + const auto new_close_cycle = close_cycle + close_index; + + if (session.session_close_cycle != new_close_cycle) + { + session.session_close_cycle = new_close_cycle; + if (close_times.size() < close_index + 1) + close_times.resize(close_index + 1); + close_times[close_index].emplace_back(key); + } + } + + void cleanThread() + { + setThreadName("SessionCleaner"); + + std::unique_lock lock{mutex}; + + while (true) + { + auto interval = closeSessions(lock); + + if (cond.wait_for(lock, interval, [this]() -> bool { return quit; })) + break; + } + } + + /// Close sessions, that has been expired. Returns how long to wait for next session to be expired, if no new sessions will be added. + std::chrono::steady_clock::duration closeSessions(std::unique_lock & lock) + { + const auto now = std::chrono::steady_clock::now(); + + /// The time to close the next session did not come + if (now < close_cycle_time) + return close_cycle_time - now; /// Will sleep until it comes. + + const auto current_cycle = close_cycle; + + ++close_cycle; + close_cycle_time = now + close_interval; + + if (close_times.empty()) + return close_interval; + + auto & sessions_to_close = close_times.front(); + + for (const auto & key : sessions_to_close) + { + const auto session = sessions.find(key); + + if (session != sessions.end() && session->second->session_close_cycle <= current_cycle) + { + if (session->second->session_is_used) + scheduleCloseSession(key, *session->second, std::chrono::seconds(0), lock); + else + sessions.erase(session); + } + } + + close_times.pop_front(); + return close_interval; + } + + std::mutex mutex; + std::condition_variable cond; + std::atomic quit{false}; + ThreadFromGlobalPool thread{&Sessions::cleanThread, this}; +}; + + /** Set of known objects (environment), that could be used in query. * Shared (global) part. Order of members (especially, order of destruction) is very important. */ @@ -166,27 +320,7 @@ struct ContextShared std::optional trace_collector; /// Thread collecting traces from threads executing queries - /// Named sessions. The user could specify session identifier to reuse settings and temporary tables in subsequent requests. - - class SessionKeyHash - { - public: - size_t operator()(const Context::SessionKey & key) const - { - SipHash hash; - hash.update(key.first); - hash.update(key.second); - return hash.get64(); - } - }; - - using Sessions = std::unordered_map, SessionKeyHash>; - using CloseTimes = std::deque>; - mutable Sessions sessions; - mutable CloseTimes close_times; - std::chrono::steady_clock::duration close_interval = std::chrono::seconds(1); - std::chrono::steady_clock::time_point close_cycle_time = std::chrono::steady_clock::now(); - UInt64 close_cycle = 0; + Sessions sessions; /// Controls named HTTP sessions. /// Clusters for distributed tables /// Initialized on demand (on distributed storages initialization) since Settings should be initialized @@ -371,100 +505,15 @@ Context::SessionKey Context::getSessionKey(const String & session_id) const } -void Context::scheduleCloseSession(const Context::SessionKey & key, std::chrono::steady_clock::duration timeout) +std::shared_ptr Context::acquireSession(const String & session_id, std::chrono::steady_clock::duration timeout, bool session_check) { - const UInt64 close_index = timeout / shared->close_interval + 1; - const auto new_close_cycle = shared->close_cycle + close_index; - - if (session_close_cycle != new_close_cycle) - { - session_close_cycle = new_close_cycle; - if (shared->close_times.size() < close_index + 1) - shared->close_times.resize(close_index + 1); - shared->close_times[close_index].emplace_back(key); - } -} - - -std::shared_ptr Context::acquireSession(const String & session_id, std::chrono::steady_clock::duration timeout, bool session_check) const -{ - auto lock = getLock(); - - const auto & key = getSessionKey(session_id); - auto it = shared->sessions.find(key); - - if (it == shared->sessions.end()) - { - if (session_check) - throw Exception("Session not found.", ErrorCodes::SESSION_NOT_FOUND); - - auto new_session = std::make_shared(*this); - - new_session->scheduleCloseSession(key, timeout); - - it = shared->sessions.insert(std::make_pair(key, std::move(new_session))).first; - } - else if (it->second->client_info.current_user != client_info.current_user) - { - throw Exception("Session belongs to a different user", ErrorCodes::LOGICAL_ERROR); - } - - const auto & session = it->second; - - if (session->session_is_used) - throw Exception("Session is locked by a concurrent client.", ErrorCodes::SESSION_IS_LOCKED); - session->session_is_used = true; - - session->client_info = client_info; - - return session; + return shared->sessions.acquireSession(getSessionKey(session_id), *this, timeout, session_check); } void Context::releaseSession(const String & session_id, std::chrono::steady_clock::duration timeout) { - auto lock = getLock(); - - session_is_used = false; - scheduleCloseSession(getSessionKey(session_id), timeout); -} - - -std::chrono::steady_clock::duration Context::closeSessions() const -{ - auto lock = getLock(); - - const auto now = std::chrono::steady_clock::now(); - - if (now < shared->close_cycle_time) - return shared->close_cycle_time - now; - - const auto current_cycle = shared->close_cycle; - - ++shared->close_cycle; - shared->close_cycle_time = now + shared->close_interval; - - if (shared->close_times.empty()) - return shared->close_interval; - - auto & sessions_to_close = shared->close_times.front(); - - for (const auto & key : sessions_to_close) - { - const auto session = shared->sessions.find(key); - - if (session != shared->sessions.end() && session->second->session_close_cycle <= current_cycle) - { - if (session->second->session_is_used) - session->second->scheduleCloseSession(key, std::chrono::seconds(0)); - else - shared->sessions.erase(session); - } - } - - shared->close_times.pop_front(); - - return shared->close_interval; + shared->sessions.releaseSession(getSessionKey(session_id), *this, timeout); } @@ -2259,65 +2308,4 @@ void Context::resetInputCallbacks() input_blocks_reader = {}; } - -class SessionCleaner -{ -public: - SessionCleaner(Context & context_) - : context{context_} - { - } - ~SessionCleaner(); - -private: - void run(); - - Context & context; - - std::mutex mutex; - std::condition_variable cond; - std::atomic quit{false}; - ThreadFromGlobalPool thread{&SessionCleaner::run, this}; -}; - -SessionCleaner::~SessionCleaner() -{ - try - { - { - std::lock_guard lock{mutex}; - quit = true; - } - - cond.notify_one(); - - thread.join(); - } - catch (...) - { - DB::tryLogCurrentException(__PRETTY_FUNCTION__); - } -} - -void SessionCleaner::run() -{ - setThreadName("SessionCleaner"); - - std::unique_lock lock{mutex}; - - while (true) - { - auto interval = context.closeSessions(); - - if (cond.wait_for(lock, interval, [this]() -> bool { return quit; })) - break; - } -} - -void Context::createSessionCleaner() -{ - session_cleaner = std::make_unique(*this); -} - - } diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index da7cb98310e..4395c1da4f5 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -96,11 +96,8 @@ class DiskSelector; class StoragePolicy; using StoragePolicyPtr = std::shared_ptr; class StoragePolicySelector; -class SessionCleaner; - class IOutputFormat; using OutputFormatPtr = std::shared_ptr; - class Volume; using VolumePtr = std::shared_ptr; @@ -179,7 +176,7 @@ private: Context * session_context = nullptr; /// Session context or nullptr. Could be equal to this. Context * global_context = nullptr; /// Global context. Could be equal to this. - std::shared_ptr session_cleaner; /// It will launch a thread to clean old named HTTP sessions. See 'createSessionCleaner'. + friend class Sessions; UInt64 session_close_cycle = 0; bool session_is_used = false; @@ -423,12 +420,9 @@ public: const Databases getDatabases() const; Databases getDatabases(); - std::shared_ptr acquireSession(const String & session_id, std::chrono::steady_clock::duration timeout, bool session_check) const; + std::shared_ptr acquireSession(const String & session_id, std::chrono::steady_clock::duration timeout, bool session_check); void releaseSession(const String & session_id, std::chrono::steady_clock::duration timeout); - /// Close sessions, that has been expired. Returns how long to wait for next session to be expired, if no new sessions will be added. - std::chrono::steady_clock::duration closeSessions() const; - /// For methods below you may need to acquire a lock by yourself. std::unique_lock getLock() const; @@ -637,9 +631,6 @@ private: SessionKey getSessionKey(const String & session_id) const; - /// Session will be closed after specified timeout. - void scheduleCloseSession(const SessionKey & key, std::chrono::steady_clock::duration timeout); - void checkCanBeDropped(const String & database, const String & table, const size_t & size, const size_t & max_size_to_drop) const; }; From aac2f98870e7dee800cd6d84e7a980e57c920f0d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 5 Mar 2020 06:57:31 +0300 Subject: [PATCH 062/712] Better code around sessions, step 2 --- dbms/programs/server/HTTPHandler.cpp | 11 +-- dbms/src/Interpreters/Context.cpp | 103 +++++++++++++++++---------- dbms/src/Interpreters/Context.h | 13 +--- 3 files changed, 70 insertions(+), 57 deletions(-) diff --git a/dbms/programs/server/HTTPHandler.cpp b/dbms/programs/server/HTTPHandler.cpp index eb83fc6f5d7..69689a55e3d 100644 --- a/dbms/programs/server/HTTPHandler.cpp +++ b/dbms/programs/server/HTTPHandler.cpp @@ -273,7 +273,7 @@ void HTTPHandler::processQuery( /// The user could specify session identifier and session timeout. /// It allows to modify settings, create temporary tables and reuse them in subsequent requests. - std::shared_ptr session; + std::shared_ptr session; String session_id; std::chrono::steady_clock::duration session_timeout; bool session_is_set = params.has("session_id"); @@ -287,15 +287,10 @@ void HTTPHandler::processQuery( session = context.acquireSession(session_id, session_timeout, session_check == "1"); - context = *session; - context.setSessionContext(*session); + context = session->context; + context.setSessionContext(session->context); } - SCOPE_EXIT({ - if (session_is_set) - session->releaseSession(session_id, session_timeout); - }); - /// The client can pass a HTTP header indicating supported compression method (gzip or deflate). String http_response_compression_methods = request.get("Accept-Encoding", ""); CompressionMethod http_response_compression_method = CompressionMethod::None; diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index 0a0ec748c56..c5e0dd19b8c 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -96,11 +96,34 @@ namespace ErrorCodes } +class Sessions; + +/// User name and session identifier. Named sessions are local to users. +using SessionKey = std::pair; + /// Named sessions. The user could specify session identifier to reuse settings and temporary tables in subsequent requests. +struct Session +{ + SessionKey key; + UInt64 close_cycle = 0; + bool is_used = false; + Context context; + std::chrono::steady_clock::duration timeout; + Sessions & parent; + + Session(SessionKey key_, Context & context_, std::chrono::steady_clock::duration timeout_, Sessions & parent_) + : key(key_), context(context_), timeout(timeout_), parent(parent_) + { + } + + ~Session(); +}; + + class Sessions { public: - using Key = Context::SessionKey; + using Key = SessionKey; ~Sessions() { @@ -121,10 +144,17 @@ public: } /// Find existing session or create a new. - std::shared_ptr acquireSession(const Key & key, Context & context, std::chrono::steady_clock::duration timeout, bool throw_if_not_found) + std::shared_ptr acquireSession(const String & session_id, Context & context, std::chrono::steady_clock::duration timeout, bool throw_if_not_found) { std::unique_lock lock(mutex); + auto & user_name = context.client_info.current_user; + + if (user_name.empty()) + throw Exception("Empty user name.", ErrorCodes::LOGICAL_ERROR); + + Key key(user_name, session_id); + auto it = sessions.find(key); if (it == sessions.end()) { @@ -132,29 +162,31 @@ public: throw Exception("Session not found.", ErrorCodes::SESSION_NOT_FOUND); /// Create a new session from current context. - auto new_session = std::make_shared(context); - scheduleCloseSession(key, *new_session, timeout, lock); + auto new_session = std::make_shared(key, context, timeout, *this); + + scheduleCloseSession(*new_session, lock); it = sessions.insert(std::make_pair(key, std::move(new_session))).first; } - else if (it->second->client_info.current_user != context.client_info.current_user) + else if (it->second->key.first != context.client_info.current_user) { throw Exception("Session belongs to a different user", ErrorCodes::LOGICAL_ERROR); } + /// Use existing session. const auto & session = it->second; - if (session->session_is_used) + if (session->is_used) throw Exception("Session is locked by a concurrent client.", ErrorCodes::SESSION_IS_LOCKED); - session->session_is_used = true; + session->is_used = true; return session; } - void releaseSession(const Key & key, Context & session, std::chrono::steady_clock::duration timeout) + void releaseSession(Session & session) { std::unique_lock lock(mutex); - session.session_is_used = false; - scheduleCloseSession(key, session, timeout, lock); + session.is_used = false; + scheduleCloseSession(session, lock); } private: @@ -170,7 +202,7 @@ private: } }; - using Container = std::unordered_map, SessionKeyHash>; + using Container = std::unordered_map, SessionKeyHash>; using CloseTimes = std::deque>; Container sessions; CloseTimes close_times; @@ -178,17 +210,20 @@ private: std::chrono::steady_clock::time_point close_cycle_time = std::chrono::steady_clock::now(); UInt64 close_cycle = 0; - void scheduleCloseSession(const Key & key, Context & session, std::chrono::steady_clock::duration timeout, std::unique_lock &) + void scheduleCloseSession(Session & session, std::unique_lock &) { - const UInt64 close_index = timeout / close_interval + 1; + /// Push it on a queue of sessions to close, on a position corresponding to the timeout. + /// (timeout is measured from current moment of time) + + const UInt64 close_index = session.timeout / close_interval + 1; const auto new_close_cycle = close_cycle + close_index; - if (session.session_close_cycle != new_close_cycle) + if (session.close_cycle != new_close_cycle) { - session.session_close_cycle = new_close_cycle; + session.close_cycle = new_close_cycle; if (close_times.size() < close_index + 1) close_times.resize(close_index + 1); - close_times[close_index].emplace_back(key); + close_times[close_index].emplace_back(session.key); } } @@ -230,10 +265,13 @@ private: { const auto session = sessions.find(key); - if (session != sessions.end() && session->second->session_close_cycle <= current_cycle) + if (session != sessions.end() && session->second->close_cycle <= current_cycle) { - if (session->second->session_is_used) - scheduleCloseSession(key, *session->second, std::chrono::seconds(0), lock); + if (session->second->is_used) + { + session->second->timeout = std::chrono::steady_clock::duration{0}; + scheduleCloseSession(*session->second, lock); + } else sessions.erase(session); } @@ -250,6 +288,12 @@ private: }; +Session::~Session() +{ + parent.releaseSession(*this); +} + + /** Set of known objects (environment), that could be used in query. * Shared (global) part. Order of members (especially, order of destruction) is very important. */ @@ -494,26 +538,9 @@ Databases Context::getDatabases() } -Context::SessionKey Context::getSessionKey(const String & session_id) const +std::shared_ptr Context::acquireSession(const String & session_id, std::chrono::steady_clock::duration timeout, bool session_check) { - auto & user_name = client_info.current_user; - - if (user_name.empty()) - throw Exception("Empty user name.", ErrorCodes::LOGICAL_ERROR); - - return SessionKey(user_name, session_id); -} - - -std::shared_ptr Context::acquireSession(const String & session_id, std::chrono::steady_clock::duration timeout, bool session_check) -{ - return shared->sessions.acquireSession(getSessionKey(session_id), *this, timeout, session_check); -} - - -void Context::releaseSession(const String & session_id, std::chrono::steady_clock::duration timeout) -{ - shared->sessions.releaseSession(getSessionKey(session_id), *this, timeout); + return shared->sessions.acquireSession(session_id, *this, timeout, session_check); } diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index 4395c1da4f5..07d2414660b 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -100,11 +100,10 @@ class IOutputFormat; using OutputFormatPtr = std::shared_ptr; class Volume; using VolumePtr = std::shared_ptr; +struct Session; #if USE_EMBEDDED_COMPILER - class CompiledExpressionCache; - #endif /// Table -> set of table-views that make SELECT from it. @@ -177,8 +176,6 @@ private: Context * global_context = nullptr; /// Global context. Could be equal to this. friend class Sessions; - UInt64 session_close_cycle = 0; - bool session_is_used = false; using SampleBlockCache = std::unordered_map; mutable SampleBlockCache sample_block_cache; @@ -420,8 +417,7 @@ public: const Databases getDatabases() const; Databases getDatabases(); - std::shared_ptr acquireSession(const String & session_id, std::chrono::steady_clock::duration timeout, bool session_check); - void releaseSession(const String & session_id, std::chrono::steady_clock::duration timeout); + std::shared_ptr acquireSession(const String & session_id, std::chrono::steady_clock::duration timeout, bool session_check); /// For methods below you may need to acquire a lock by yourself. std::unique_lock getLock() const; @@ -582,9 +578,6 @@ public: String getFormatSchemaPath() const; void setFormatSchemaPath(const String & path); - /// User name and session identifier. Named sessions are local to users. - using SessionKey = std::pair; - SampleBlockCache & getSampleBlockCache() const; /// Query parameters for prepared statements. @@ -629,8 +622,6 @@ private: StoragePtr getTableImpl(const StorageID & table_id, std::optional * exception) const; - SessionKey getSessionKey(const String & session_id) const; - void checkCanBeDropped(const String & database, const String & table, const size_t & size, const size_t & max_size_to_drop) const; }; From 4e15d744699f70d4ffbc554d78d79718e9dba182 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 5 Mar 2020 07:10:48 +0300 Subject: [PATCH 063/712] Better code around sessions, step 3 --- dbms/programs/server/HTTPHandler.cpp | 5 +++ dbms/src/Interpreters/Context.cpp | 47 +++------------------------- dbms/src/Interpreters/Context.h | 25 +++++++++++++++ 3 files changed, 35 insertions(+), 42 deletions(-) diff --git a/dbms/programs/server/HTTPHandler.cpp b/dbms/programs/server/HTTPHandler.cpp index 69689a55e3d..7eafe00fdd6 100644 --- a/dbms/programs/server/HTTPHandler.cpp +++ b/dbms/programs/server/HTTPHandler.cpp @@ -291,6 +291,11 @@ void HTTPHandler::processQuery( context.setSessionContext(session->context); } + SCOPE_EXIT({ + if (session) + session->release(); + }); + /// The client can pass a HTTP header indicating supported compression method (gzip or deflate). String http_response_compression_methods = request.get("Accept-Encoding", ""); CompressionMethod http_response_compression_method = CompressionMethod::None; diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index c5e0dd19b8c..a7011a1d4ae 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -96,30 +96,6 @@ namespace ErrorCodes } -class Sessions; - -/// User name and session identifier. Named sessions are local to users. -using SessionKey = std::pair; - -/// Named sessions. The user could specify session identifier to reuse settings and temporary tables in subsequent requests. -struct Session -{ - SessionKey key; - UInt64 close_cycle = 0; - bool is_used = false; - Context context; - std::chrono::steady_clock::duration timeout; - Sessions & parent; - - Session(SessionKey key_, Context & context_, std::chrono::steady_clock::duration timeout_, Sessions & parent_) - : key(key_), context(context_), timeout(timeout_), parent(parent_) - { - } - - ~Session(); -}; - - class Sessions { public: @@ -162,10 +138,7 @@ public: throw Exception("Session not found.", ErrorCodes::SESSION_NOT_FOUND); /// Create a new session from current context. - auto new_session = std::make_shared(key, context, timeout, *this); - - scheduleCloseSession(*new_session, lock); - it = sessions.insert(std::make_pair(key, std::move(new_session))).first; + it = sessions.insert(std::make_pair(key, std::make_shared(key, context, timeout, *this))).first; } else if (it->second->key.first != context.client_info.current_user) { @@ -175,17 +148,15 @@ public: /// Use existing session. const auto & session = it->second; - if (session->is_used) + if (!session.unique()) throw Exception("Session is locked by a concurrent client.", ErrorCodes::SESSION_IS_LOCKED); - session->is_used = true; return session; } void releaseSession(Session & session) { std::unique_lock lock(mutex); - session.is_used = false; scheduleCloseSession(session, lock); } @@ -265,16 +236,8 @@ private: { const auto session = sessions.find(key); - if (session != sessions.end() && session->second->close_cycle <= current_cycle) - { - if (session->second->is_used) - { - session->second->timeout = std::chrono::steady_clock::duration{0}; - scheduleCloseSession(*session->second, lock); - } - else - sessions.erase(session); - } + if (session != sessions.end() && session->second.unique() && session->second->close_cycle <= current_cycle) + sessions.erase(session); } close_times.pop_front(); @@ -288,7 +251,7 @@ private: }; -Session::~Session() +void Session::release() { parent.releaseSession(*this); } diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index 07d2414660b..de1d12e1a47 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -102,6 +102,7 @@ class Volume; using VolumePtr = std::shared_ptr; struct Session; + #if USE_EMBEDDED_COMPILER class CompiledExpressionCache; #endif @@ -133,6 +134,7 @@ struct IHostContext using IHostContextPtr = std::shared_ptr; + /** A set of known objects that can be used in the query. * Consists of a shared part (always common to all sessions and queries) * and copied part (which can be its own for each session or query). @@ -653,4 +655,27 @@ private: std::unique_lock table_lock; }; + +class Sessions; + +/// User name and session identifier. Named sessions are local to users. +using SessionKey = std::pair; + +/// Named sessions. The user could specify session identifier to reuse settings and temporary tables in subsequent requests. +struct Session +{ + SessionKey key; + UInt64 close_cycle = 0; + Context context; + std::chrono::steady_clock::duration timeout; + Sessions & parent; + + Session(SessionKey key_, Context & context_, std::chrono::steady_clock::duration timeout_, Sessions & parent_) + : key(key_), context(context_), timeout(timeout_), parent(parent_) + { + } + + void release(); +}; + } From b1d63c51fe65fdd3cf9c88368d3cf7652c6931ed Mon Sep 17 00:00:00 2001 From: Sergei Shtykov Date: Thu, 5 Mar 2020 11:38:54 +0300 Subject: [PATCH 064/712] CLICKHOUSEDOCS-548: More adopter. --- docs/en/introduction/adopters.md | 95 +++++++++++++++++++++++--------- 1 file changed, 68 insertions(+), 27 deletions(-) diff --git a/docs/en/introduction/adopters.md b/docs/en/introduction/adopters.md index edfc774f7b6..eafb9d6d0fb 100644 --- a/docs/en/introduction/adopters.md +++ b/docs/en/introduction/adopters.md @@ -10,6 +10,8 @@ | [Appsflyer](https://www.appsflyer.com) | Mobile analytics | Main product | — | — | [Talk in Russian, July 2019](https://www.youtube.com/watch?v=M3wbRlcpBbY) | | [Badoo](https://badoo.com) | Dating | Timeseries | — | — | [Slides in Russian, December 2019](https://presentations.clickhouse.tech/meetup38/forecast.pdf) | | [Bloomberg](https://www.bloomberg.com/) | Finance, Media | Monitoring | 102 servers | — | [Slides, May 2018](https://www.slideshare.net/Altinity/http-analytics-for-6m-requests-per-second-using-clickhouse-by-alexander-bocharov) | +| [Bloxy](https://bloxy.info) | Blockchain | Analytics | — | — | [Slides in Russian, August 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/4_bloxy.pptx) | +| `Dataliance/UltraPower` | Telecom | Analytics | — | — | [Slides in Chinese, January 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup12/telecom.pdf) | | [CARTO](https://carto.com/) | Business Intelligence | Geo analytics | — | — | [Geospatial processing with Clickhouse](https://carto.com/blog/geospatial-processing-with-clickhouse/) | | [CERN](http://public.web.cern.ch/public/) | Research | Experiment | — | — | [Press release, April 2012](https://www.yandex.com/company/press_center/press_releases/2012/2012-04-10/) | | [Cisco](http://cisco.com/) | Networking | Traffic analysis | — | — | [Lightning talk, October 2019](https://youtu.be/-hI1vDR2oPY?t=5057) | @@ -18,20 +20,37 @@ | [ContentSquare](https://contentsquare.com) | Web analytics | Main product | — | — | [Blog post in French, November 2018](http://souslecapot.net/2018/11/21/patrick-chatain-vp-engineering-chez-contentsquare-penser-davantage-amelioration-continue-que-revolution-constante/) | | [Cloudflare](https://cloudflare.com) | CDN | Traffic analysis | 36 servers | — | [Blog post, May 2017](https://blog.cloudflare.com/how-cloudflare-analyzes-1m-dns-queries-per-second/), [Blog post, March 2018](https://blog.cloudflare.com/http-analytics-for-6m-requests-per-second-using-clickhouse/) | | [Corunet](https://coru.net/) | Analytics | Main product | — | — | [Slides in English, April 2019 ](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup21/predictive_models.pdf) | +| [Criteo/Storetail] | Retail | Main product | — | — | [Slides in English, October 2018 ](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup18/3_storetail.pptx) | | [Deutsche Bank](db.com) | Finance | BI Analytics | — | — | [Slides in English, October 2019](https://bigdatadays.ru/wp-content/uploads/2019/10/D2-H3-3_Yakunin-Goihburg.pdf) | | [Exness](https://www.exness.com) | Trading | Metrics, Logging | — | — | [Talk in Russian, May 2019](https://youtu.be/_rpU-TvSfZ8?t=3215) | | [Geniee](https://geniee.co.jp) | Ad network | Main product | — | — | [Blog post in Japanese, July 2017](https://tech.geniee.co.jp/entry/2017/07/20/160100) | +| [HUYA](https://www.huya.com/) | Video Streaming | Analytics | — | — | [Slides in Chinese, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/7.%20ClickHouse万亿数据分析实践%20李本旺(sundy-li)%20虎牙.pdf) | | [Idealista](www.idealista.com) | Real Estate | Analytics | — | — | [Blog Post in English, April 2019](https://clickhouse.yandex/blog/en/clickhouse-meetup-in-madrid-on-april-2-2019) | +| [Kodiak Data](https://www.kodiakdata.com/) | Clouds | Main product | — | — | [Slides in Engish, April 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup13/kodiak_data.pdf) | | [Kontur](https://kontur.ru) | Software Development | Metrics | — | — | [Talk in Russian, November 2018](https://www.youtube.com/watch?v=U4u4Bd0FtrY) | -| [LifeStreet](https://cloudflare.com) | Ad network | Main product | — | — | [Blog post in Russian, February 2017](https://habr.com/en/post/322620/) | +| [LifeStreet](https://cloudflare.com) | Ad network | Main product | 60 servers in 3 replicas | 2-2.5 PiB | [Blog post in Russian, February 2017](https://habr.com/en/post/322620/) | | [Mail.ru Cloud Solutions](https://mcs.mail.ru/) | Cloud services | Main product | — | — | [Running ClickHouse Instance, in Russian](https://mcs.mail.ru/help/db-create/clickhouse#) | -| [MGID](https://www.mgid.com/) | Ad network | Web-analytics | --- | --- | [Our experience in implementing analytical DBMS ClickHouse, in Russian](http://gs-studio.com/news-about-it/32777----clickhouse---c) | +| [MGID](https://www.mgid.com/) | Ad network | Web-analytics | — | — | [Our experience in implementing analytical DBMS ClickHouse, in Russian](http://gs-studio.com/news-about-it/32777----clickhouse---c) | +| [MessageBird](https://www.messagebird.com) | Telecommunications | Statistics | — | — | [Slides in English, November 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup20/messagebird.pdf) | +| [OneAPM](https://www.oneapm.com/) | Monitorings and Data Analysis | Main product | — | — | [Slides in Chinese, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/8.%20clickhouse在OneAPM的应用%20杜龙.pdf) | +| [Pragma Innovation](http://www.pragma-innovation.fr/) | Telemetry and Big Data Analysis | Main product | — | — | [Slides in English, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup18/4_pragma_innovation.pdf) | +| [QINGCLOUD](https://www.qingcloud.com/) | Cloud services | Main product | — | — | [Slides in Chinese, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/4.%20Cloud%20%2B%20TSDB%20for%20ClickHouse%20张健%20QingCloud.pdf) | | [Qrator](https://qrator.net) | DDoS protection | Main product | — | — | [Blog Post, March 2019](https://blog.qrator.net/en/clickhouse-ddos-mitigation_37/) | +| [Rambler](rambler.ru) | Internet services | Analytics | — | — | [Talk in Russian, April 2018](https://medium.com/@ramblertop/разработка-api-clickhouse-для-рамблер-топ-100-f4c7e56f3141) | | [Tencent](https://www.tencent.com) | Messaging | Logging | — | — | [Talk in Chinese, November 2019](https://youtu.be/T-iVQRuw-QY?t=5050) | +| [Traffic Stars](https://trafficstars.com/) | AD network | — | — | — | [Slides in Russian, May 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup15/lightning/ninja.pdf) | | [S7 Airlines](https://www.s7.ru) | Airlines | Metrics, Logging | — | — | [Talk in Russian, March 2019](https://www.youtube.com/watch?v=nwG68klRpPg&t=15s) | -| [scireum GmbH](https://www.scireum.de/) | e-Commerce | ??? | — | — | [Talk in German, February 2020](https://www.youtube.com/watch?v=7QWAn5RbyR4) | +| [SEMrush](https://www.semrush.com/) | Marketing | Main product | — | — | [Slides in Russian, August 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/5_semrush.pdf) | +| [scireum GmbH](https://www.scireum.de/) | e-Commerce | Main product | — | — | [Talk in German, February 2020](https://www.youtube.com/watch?v=7QWAn5RbyR4) | +| [Sentry](https://sentry.io/) | Software developer | Backend for product | — | — | [Blog Post in English, May 2019](https://blog.sentry.io/2019/05/16/introducing-snuba-sentrys-new-search-infrastructure) | +| [Sina](http://english.sina.com/index.html) | News | — | — | — | [Slides in Chinese, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/6.%20ClickHouse最佳实践%20高鹏_新浪.pdf) | +| [SMI2](https://smi2.ru/) | News | Analytics | — | — | [Blog Post in Russian, November 2017](https://habr.com/ru/company/smi2/blog/314558/) | +| [Splunk](https://www.splunk.com/) | Business Analytics | Main product | — | — | [Slides in English, January 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup12/splunk.pdf) | | [Spotify](https://www.spotify.com) | Music | Experimentation | — | — | [Slides, July 2018](https://www.slideshare.net/glebus/using-clickhouse-for-experimentation-104247173) | +| [Tencent](https://www.tencent.com) | Big Data | Data processing | — | — | [Slides in Chinese, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/5.%20ClickHouse大数据集群应用_李俊飞腾讯网媒事业部.pdf) | + Нераспознанный китайский источник https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/5.%20ClickHouse大数据集群应用_李俊飞腾讯网媒事业部.pdf | [Uber](https://www.uber.com) | Taxi | Logging | — | — | [Slides, February 2020](https://presentations.clickhouse.tech/meetup40/ml.pdf) | +| [VKontakte](vk.com) | Social Network | Statistics, Logging | — | — | [Slides in Russian, August 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/3_vk.pdf) | | [Yandex Cloud](https://cloud.yandex.ru/services/managed-clickhouse) | Public Cloud | Main product | — | — | [Talk in Russian, December 2019](https://www.youtube.com/watch?v=pgnak9e_E0o) | | [Yandex DataLens](https://cloud.yandex.ru/services/datalens) | Business Intelligence | Main product | — | — | [Slides in Russian, December 2019](https://presentations.clickhouse.tech/meetup38/datalens.pdf) | | [Yandex Market](https://market.yandex.ru/) | e-Commerce | Metrics, Logging | — | — | [Talk in Russian, January 2019](https://youtu.be/_l1qP0DyBcA?t=478) | @@ -57,6 +76,19 @@ https://blockchair.com/ - https://www.octonica.ru/ презентация https://github.com/ClickHouse/clickhouse-presentations/blob/master/database_saturday_2018_2/octonica/meetup.pptx +Yandex.Mail is using ClickHouse to record user activity logs in a web interface for investigations and analytics. + +Yandex.Browser is using ClickHouse for performance histograms. Browsers send many mini-histograms from clients. +They are all stored into ClickHouse for the purpose of Browser version comparisons, analytics and investigations. + +- Infinidat https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup11/clickhouse_in_the_world.pptx +- Vertamedia +- NVidia +- Percona https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup11/clickhouse_in_the_world.pptx Но это не компания, это разработчик тулзов для OpenSource. https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup14/analyzing_mysql_logs.pptx +- BI Tableau https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup11/tableau.pptx December 2017? это дата загрузки презентации. +- https://my.com/ @mail.ru https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup11/target.pdf +- [Эрливидео](https://flussonic.ru/) https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup15/lightning/erlyvideo.key +- mipt https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup15/lightning/mipt.pdf ### Stash @@ -81,6 +113,15 @@ https://blockchair.com/ - ByteDance, единственное прямое доказательство было ссылкой на уже не существующую вакансию. +- Booking.com. Экспериментальная поддержка ClickHouse в https://github.com/bookingcom/carbonapi + +- [Roistat](https://roistat.com) разработала [Go ClickHouse Connector](https://github.com/roistat/go-clickhouse) и поддерживает свой [Docker](https://github.com/roistat/docker-clickhouse). + +- JD.com ClickHouse Meetup in Beijing 2019 +- KuaiShou ClickHouse Meetup in Beijing 2018 + +- JetBrains DataGrip ? https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/6_datagrip.pdf + [Original article](https://clickhouse.tech/docs/en/introduction/adopters/) @@ -96,30 +137,30 @@ https://blockchair.com/ + database_saturday_2018_2/octonica - database_saturday_2019 + dataops_2019 (CARTO, Mercadona, Zara, Idealista, Corunet, ... Cloudflare, Spotify, Amadeus, Bloomberg, Cisco, Deutsche Bank, Tencent, ByteDance) -drafts yandex/ClickHouse -> ClickHouse/ClickHouse, part 2 5 months ago -evolution Correction on KuaiShou company name 3 months ago -group_by Changed tabs to spaces in code [#CLICKHOUSE-3]. 3 years ago -hash_tables yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago -highload2016 Added historical presentation from HighLoad 2016 11 days ago -highload2017 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago -highload2018 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago -highload2019 Added presentation from HighLoad++ 2019 4 months ago -highload_siberia_2018 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago -highload_siberia_2019 Added another presentation from HighLoad Siberia 2019 8 months ago -highload_spb_2019 Added presentation from Saint Highload 2019 11 months ago -hse_2019 Added one more presentation from HSE 8 months ago -internals add sources for the internals presentation 2 years ago -it_rzn_2019 Updated presentation from IT Rzn 3 months ago -meetup10 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago -meetup11 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago -meetup12 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago -meetup13 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago -meetup14 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago -meetup15 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago -meetup16 Added all presentations from Summer Berlin Meetup 2 years ago -meetup17 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago -meetup18 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago -meetup19 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago ++ drafts (CloudFlare, Booking.com, Crobox, Rambler, QRator, СКБ Контур, Roistat, SMI2) ++ evolution (ByteDance, Sina, Tencent, JD.com, KuaiShou) +- group_by +- hash_tables +- highload2016 +- highload2017 +- highload2018 +- highload2019 +- highload_siberia_2018 +- highload_siberia_2019 +- highload_spb_2019 +- hse_2019 +- internals +- it_rzn_2019 ++ meetup10 ++ meetup11 ++ meetup12 ++ meetup13 +- meetup14 ++ meetup15 +- meetup16 ++ meetup17 ++ meetup18 ++ meetup19 meetup20 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago meetup21 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago meetup22 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago From 2a18ada8900b50cb662036debbd1ac90e7912bd4 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 5 Mar 2020 13:49:25 +0300 Subject: [PATCH 065/712] fix test_part_log_table part_log must be created in system database, because otherwise it is not guaranteed that part_log loaded before it can be used by other table --- dbms/tests/integration/test_part_log_table/test.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/dbms/tests/integration/test_part_log_table/test.py b/dbms/tests/integration/test_part_log_table/test.py index 5d4c005714f..a97c8cf7b3a 100644 --- a/dbms/tests/integration/test_part_log_table/test.py +++ b/dbms/tests/integration/test_part_log_table/test.py @@ -33,11 +33,10 @@ def test_config_with_standard_part_log(start_cluster): assert node2.query("SELECT * FROM system.part_log") != "" def test_config_with_non_standard_part_log(start_cluster): - node3.query("CREATE DATABASE database_name") - assert "table_name" not in node3.query("SHOW TABLES FROM database_name") - node3.query("CREATE TABLE test_table(word String, value UInt64) ENGINE=MergeTree() ORDER BY value") - assert "table_name" not in node3.query("SHOW TABLES FROM database_name") + assert "Table system.table_name doesn't exist" in node3.query_and_get_error("SELECT * FROM system.table_name") + node3.query("CREATE TABLE test_table(word String, value UInt64) ENGINE=MergeTree() Order by value") + assert "Table system.table_name doesn't exist" in node3.query_and_get_error("SELECT * FROM system.table_name") node3.query("INSERT INTO test_table VALUES ('name', 1)") time.sleep(10) - assert "table_name" in node3.query("SHOW TABLES FROM database_name") + assert node3.query("SELECT * FROM system.table_name") != "" From 96bb607e6693398cee0bc85c57979919c1d80037 Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Thu, 5 Mar 2020 17:01:19 +0300 Subject: [PATCH 066/712] Open file for reading lazily in CachedCompressedReadBuffer. --- .../CachedCompressedReadBuffer.cpp | 22 +++++-------------- .../Compression/CachedCompressedReadBuffer.h | 14 ++---------- .../MergeTree/MergeTreeReaderCompact.cpp | 15 +++++++++---- .../MergeTree/MergeTreeReaderStream.cpp | 11 ++++++++-- 4 files changed, 27 insertions(+), 35 deletions(-) diff --git a/dbms/src/Compression/CachedCompressedReadBuffer.cpp b/dbms/src/Compression/CachedCompressedReadBuffer.cpp index 00cb5fe3fe9..1b083c004c0 100644 --- a/dbms/src/Compression/CachedCompressedReadBuffer.cpp +++ b/dbms/src/Compression/CachedCompressedReadBuffer.cpp @@ -1,10 +1,10 @@ #include "CachedCompressedReadBuffer.h" -#include #include -#include #include +#include + namespace DB { @@ -19,7 +19,7 @@ void CachedCompressedReadBuffer::initInput() { if (!file_in) { - file_in = createReadBufferFromFileBase(path, estimated_size, aio_threshold, mmap_threshold, buf_size); + file_in = file_in_creator(); compressed_in = file_in.get(); if (profile_callback) @@ -71,24 +71,12 @@ bool CachedCompressedReadBuffer::nextImpl() return true; } - -CachedCompressedReadBuffer::CachedCompressedReadBuffer(std::unique_ptr file_in_, UncompressedCache * cache_) - : ReadBuffer(nullptr, 0), cache(cache_), file_in(std::move(file_in_)), path(file_in->getFileName()), file_pos(0) -{ - compressed_in = file_in.get(); -} - - CachedCompressedReadBuffer::CachedCompressedReadBuffer( - const std::string & path_, UncompressedCache * cache_, - size_t estimated_size_, size_t aio_threshold_, size_t mmap_threshold_, - size_t buf_size_) - : ReadBuffer(nullptr, 0), cache(cache_), path(path_), buf_size(buf_size_), estimated_size(estimated_size_), - aio_threshold(aio_threshold_), mmap_threshold(mmap_threshold_), file_pos(0) + const std::string & path_, std::function()> file_in_creator_, UncompressedCache * cache_) + : ReadBuffer(nullptr, 0), file_in_creator(std::move(file_in_creator_)), cache(cache_), path(path_), file_pos(0) { } - void CachedCompressedReadBuffer::seek(size_t offset_in_compressed_file, size_t offset_in_decompressed_block) { if (owned_cell && diff --git a/dbms/src/Compression/CachedCompressedReadBuffer.h b/dbms/src/Compression/CachedCompressedReadBuffer.h index 4f9f1fce480..88bcec8197d 100644 --- a/dbms/src/Compression/CachedCompressedReadBuffer.h +++ b/dbms/src/Compression/CachedCompressedReadBuffer.h @@ -20,15 +20,11 @@ namespace DB class CachedCompressedReadBuffer : public CompressedReadBufferBase, public ReadBuffer { private: + std::function()> file_in_creator; UncompressedCache * cache; std::unique_ptr file_in; const std::string path; - size_t buf_size {}; - size_t estimated_size {}; - size_t aio_threshold {}; - size_t mmap_threshold {}; - size_t file_pos; /// A piece of data from the cache, or a piece of read data that we put into the cache. @@ -42,13 +38,7 @@ private: clockid_t clock_type {}; public: - CachedCompressedReadBuffer(std::unique_ptr file_in, UncompressedCache * cache_); - - CachedCompressedReadBuffer( - const std::string & path_, UncompressedCache * cache_, - size_t estimated_size_, size_t aio_threshold_, size_t mmap_threshold_, - size_t buf_size_ = DBMS_DEFAULT_BUFFER_SIZE); - + CachedCompressedReadBuffer(const std::string & path, std::function()> file_in_creator, UncompressedCache * cache_); void seek(size_t offset_in_compressed_file, size_t offset_in_decompressed_block); diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index 8698ba78676..2b05cbf28d5 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -39,10 +39,17 @@ MergeTreeReaderCompact::MergeTreeReaderCompact( if (uncompressed_cache) { - auto buffer = - std::make_unique( - data_part->disk->readFile(full_data_path, buffer_size, 0, settings.min_bytes_to_use_direct_io, 0), - uncompressed_cache); + auto buffer = std::make_unique( + full_data_path, + [&]() { + return data_part->disk->readFile( + full_data_path, + buffer_size, + 0, + settings.min_bytes_to_use_direct_io, + 0); + }, + uncompressed_cache); if (profile_callback_) buffer->setProfileCallback(profile_callback_, clock_type_); diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp index d0e73cd16e0..393e753a4ba 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp @@ -79,8 +79,15 @@ MergeTreeReaderStream::MergeTreeReaderStream( if (uncompressed_cache) { auto buffer = std::make_unique( - disk->readFile(path_prefix + data_file_extension, buffer_size, - sum_mark_range_bytes, settings.min_bytes_to_use_direct_io, settings.min_bytes_to_use_mmap_io), + path_prefix + data_file_extension, + [&]() { + return disk->readFile( + path_prefix + data_file_extension, + buffer_size, + sum_mark_range_bytes, + settings.min_bytes_to_use_direct_io, + settings.min_bytes_to_use_mmap_io); + }, uncompressed_cache); if (profile_callback) From caf26d8affde441cd8fff209c9e4709b83af0253 Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Thu, 5 Mar 2020 17:02:15 +0300 Subject: [PATCH 067/712] IDisk get/set last modified time implementation. --- dbms/src/Disks/DiskLocal.cpp | 10 ++++++++++ dbms/src/Disks/DiskLocal.h | 4 ++++ dbms/src/Disks/DiskMemory.h | 4 ++++ dbms/src/Disks/DiskS3.cpp | 10 ++++++++++ dbms/src/Disks/DiskS3.h | 4 ++++ dbms/src/Disks/IDisk.h | 7 +++++++ .../Storages/MergeTree/IMergeTreeDataPart.cpp | 2 +- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 19 +++++++------------ 8 files changed, 47 insertions(+), 13 deletions(-) diff --git a/dbms/src/Disks/DiskLocal.cpp b/dbms/src/Disks/DiskLocal.cpp index 8c8bffc0f5a..418ce966955 100644 --- a/dbms/src/Disks/DiskLocal.cpp +++ b/dbms/src/Disks/DiskLocal.cpp @@ -244,6 +244,16 @@ void DiskLocal::listFiles(const String & path, std::vector & file_names) Poco::File(disk_path + path).list(file_names); } +void DiskLocal::setLastModified(const String & path, const Poco::Timestamp & timestamp) +{ + Poco::File(disk_path + path).setLastModified(timestamp); +} + +Poco::Timestamp DiskLocal::getLastModified(const String & path) +{ + return Poco::File(disk_path + path).getLastModified(); +} + void DiskLocalReservation::update(UInt64 new_size) { diff --git a/dbms/src/Disks/DiskLocal.h b/dbms/src/Disks/DiskLocal.h index 03000f9e9a8..0bca5dc72d4 100644 --- a/dbms/src/Disks/DiskLocal.h +++ b/dbms/src/Disks/DiskLocal.h @@ -87,6 +87,10 @@ public: void removeRecursive(const String & path) override; + void setLastModified(const String & path, const Poco::Timestamp & timestamp) override; + + Poco::Timestamp getLastModified(const String & path) override; + private: bool tryReserve(UInt64 bytes); diff --git a/dbms/src/Disks/DiskMemory.h b/dbms/src/Disks/DiskMemory.h index 2b9ca65de70..f67a361a948 100644 --- a/dbms/src/Disks/DiskMemory.h +++ b/dbms/src/Disks/DiskMemory.h @@ -80,6 +80,10 @@ public: void removeRecursive(const String & path) override; + void setLastModified(const String &, const Poco::Timestamp &) override { } + + Poco::Timestamp getLastModified(const String &) override { return Poco::Timestamp(); } + private: void createDirectoriesImpl(const String & path); void replaceFileImpl(const String & from_path, const String & to_path); diff --git a/dbms/src/Disks/DiskS3.cpp b/dbms/src/Disks/DiskS3.cpp index 68e7beab9c6..abf40bccf83 100644 --- a/dbms/src/Disks/DiskS3.cpp +++ b/dbms/src/Disks/DiskS3.cpp @@ -614,6 +614,16 @@ void DiskS3::listFiles(const String & path, std::vector & file_names) file_names.push_back(it->name()); } +void DiskS3::setLastModified(const String & path, const Poco::Timestamp & timestamp) +{ + Poco::File(metadata_path + path).setLastModified(timestamp); +} + +Poco::Timestamp DiskS3::getLastModified(const String & path) +{ + return Poco::File(metadata_path + path).getLastModified(); +} + DiskS3Reservation::~DiskS3Reservation() { diff --git a/dbms/src/Disks/DiskS3.h b/dbms/src/Disks/DiskS3.h index 984544b018e..5b59089ffd1 100644 --- a/dbms/src/Disks/DiskS3.h +++ b/dbms/src/Disks/DiskS3.h @@ -87,6 +87,10 @@ public: void removeRecursive(const String & path) override; + void setLastModified(const String & path, const Poco::Timestamp & timestamp) override; + + Poco::Timestamp getLastModified(const String & path) override; + private: String getRandomName() const; diff --git a/dbms/src/Disks/IDisk.h b/dbms/src/Disks/IDisk.h index 0f0447a7169..877c6f84706 100644 --- a/dbms/src/Disks/IDisk.h +++ b/dbms/src/Disks/IDisk.h @@ -10,6 +10,7 @@ #include #include #include +#include namespace CurrentMetrics @@ -145,6 +146,12 @@ public: /// Remove file or directory with all children. Use with extra caution. Throws exception if file doesn't exists. virtual void removeRecursive(const String & path) = 0; + + /// Set last modified time to file or directory at `path`. + virtual void setLastModified(const String & path, const Poco::Timestamp & timestamp) = 0; + + /// Get last modified time of file or directory at `path`. + virtual Poco::Timestamp getLastModified(const String & path) = 0; }; using DiskPtr = std::shared_ptr; diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 3cf4fede415..c7ac56821d7 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -673,7 +673,7 @@ void IMergeTreeDataPart::renameTo(const String & new_relative_path, bool remove_ } } - //from_file.setLastModified(Poco::Timestamp::fromEpochTime(time(nullptr))); + disk->setLastModified(from, Poco::Timestamp::fromEpochTime(time(nullptr))); disk->moveFile(from, to); relative_path = new_relative_path; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 474f8512980..135f31d3c32 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -1074,8 +1074,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) else has_adaptive_parts.store(true, std::memory_order_relaxed); - /// TODO: Add modification time functionality to IDisk. - part->modification_time = Poco::File(getFullPathOnDisk(part_disk_ptr) + part_name).getLastModified().epochTime(); + part->modification_time = part_disk_ptr->getLastModified(relative_data_path + part_name).epochTime(); /// Assume that all parts are Committed, covered parts will be detected and marked as Outdated later part->state = DataPartState::Committed; @@ -1158,14 +1157,13 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) /// Is the part directory old. /// True if its modification time and the modification time of all files inside it is less then threshold. /// (Only files on the first level of nesting are considered). -static bool isOldPartDirectory(Poco::File & directory, time_t threshold) +static bool isOldPartDirectory(const DiskPtr & disk, const String & directory_path, time_t threshold) { - if (directory.getLastModified().epochTime() >= threshold) + if (disk->getLastModified(directory_path).epochTime() >= threshold) return false; - Poco::DirectoryIterator end; - for (Poco::DirectoryIterator it(directory); it != end; ++it) - if (it->getLastModified().epochTime() >= threshold) + for (auto it = disk->iterateDirectory(directory_path); it->isValid(); it->next()) + if (disk->getLastModified(it->path()).epochTime() >= threshold) return false; return true; @@ -1192,15 +1190,12 @@ void MergeTreeData::clearOldTemporaryDirectories(ssize_t custom_directories_life { if (startsWith(it->name(), "tmp_")) { - /// TODO: Add modification time functionality to IDisk. - Poco::File tmp_dir(fullPath(disk, it->path())); - try { - if (tmp_dir.isDirectory() && isOldPartDirectory(tmp_dir, deadline)) + if (disk->isDirectory(it->path()) && isOldPartDirectory(disk, it->path(), deadline)) { LOG_WARNING(log, "Removing temporary directory " << fullPath(disk, it->path())); - tmp_dir.remove(true); + disk->removeRecursive(it->path()); } } catch (const Poco::FileNotFoundException &) From 2acd6afe0b51319d98779bdab3ba03118b368b8a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 5 Mar 2020 18:08:25 +0300 Subject: [PATCH 068/712] Fixed build --- dbms/src/Interpreters/Context.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index af7cd742299..a7ec11ee2cb 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -214,7 +214,7 @@ private: } /// Close sessions, that has been expired. Returns how long to wait for next session to be expired, if no new sessions will be added. - std::chrono::steady_clock::duration closeSessions(std::unique_lock & lock) + std::chrono::steady_clock::duration closeSessions(std::unique_lock &) { const auto now = std::chrono::steady_clock::now(); From 8587d47c5cc4785fdaf24ed64a803cd897a1dd7c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 5 Mar 2020 18:33:34 +0300 Subject: [PATCH 069/712] Fixed build --- dbms/src/Interpreters/Context.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index 94b8933e3b9..d67fa3fb9c0 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -599,8 +599,6 @@ public: void dropCompiledExpressionCache() const; #endif - void createSessionCleaner(); - /// Add started bridge command. It will be killed after context destruction void addXDBCBridgeCommand(std::unique_ptr cmd) const; From dd13e0353a615e5ccb6325d1b741c105fb3f1518 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 5 Mar 2020 18:40:59 +0300 Subject: [PATCH 070/712] Fixed build --- dbms/programs/server/Server.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/dbms/programs/server/Server.cpp b/dbms/programs/server/Server.cpp index 5ab73a93e7a..4410d6ea696 100644 --- a/dbms/programs/server/Server.cpp +++ b/dbms/programs/server/Server.cpp @@ -1020,8 +1020,6 @@ int Server::main(const std::vector & /*args*/) global_context->getConfigRef(), graphite_key, async_metrics)); } - global_context->createSessionCleaner(); - waitForTerminationRequest(); } From 3b85f2ffef0f6915da7e3fdc532a6ffc339e57c7 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 5 Mar 2020 22:23:39 +0300 Subject: [PATCH 071/712] Do not run session cleaner for client, local --- dbms/programs/server/HTTPHandler.cpp | 4 +-- dbms/programs/server/Server.cpp | 2 ++ dbms/src/Interpreters/Context.cpp | 39 ++++++++++++++++++---------- dbms/src/Interpreters/Context.h | 22 +++++++++------- 4 files changed, 42 insertions(+), 25 deletions(-) diff --git a/dbms/programs/server/HTTPHandler.cpp b/dbms/programs/server/HTTPHandler.cpp index 7eafe00fdd6..a4c59ff9e25 100644 --- a/dbms/programs/server/HTTPHandler.cpp +++ b/dbms/programs/server/HTTPHandler.cpp @@ -273,7 +273,7 @@ void HTTPHandler::processQuery( /// The user could specify session identifier and session timeout. /// It allows to modify settings, create temporary tables and reuse them in subsequent requests. - std::shared_ptr session; + std::shared_ptr session; String session_id; std::chrono::steady_clock::duration session_timeout; bool session_is_set = params.has("session_id"); @@ -285,7 +285,7 @@ void HTTPHandler::processQuery( session_timeout = parseSessionTimeout(config, params); std::string session_check = params.get("session_check", ""); - session = context.acquireSession(session_id, session_timeout, session_check == "1"); + session = context.acquireNamedSession(session_id, session_timeout, session_check == "1"); context = session->context; context.setSessionContext(session->context); diff --git a/dbms/programs/server/Server.cpp b/dbms/programs/server/Server.cpp index 4410d6ea696..636911c71ca 100644 --- a/dbms/programs/server/Server.cpp +++ b/dbms/programs/server/Server.cpp @@ -908,6 +908,8 @@ int Server::main(const std::vector & /*args*/) if (servers.empty()) throw Exception("No servers started (add valid listen_host and 'tcp_port' or 'http_port' to configuration file.)", ErrorCodes::NO_ELEMENTS_IN_CONFIG); + global_context->enableNamedSessions(); + for (auto & server : servers) server->start(); diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index a7ec11ee2cb..62f2ceb32cb 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -96,12 +96,12 @@ namespace ErrorCodes } -class Sessions +class NamedSessions { public: - using Key = SessionKey; + using Key = NamedSessionKey; - ~Sessions() + ~NamedSessions() { try { @@ -120,7 +120,11 @@ public: } /// Find existing session or create a new. - std::shared_ptr acquireSession(const String & session_id, Context & context, std::chrono::steady_clock::duration timeout, bool throw_if_not_found) + std::shared_ptr acquireSession( + const String & session_id, + Context & context, + std::chrono::steady_clock::duration timeout, + bool throw_if_not_found) { std::unique_lock lock(mutex); @@ -138,7 +142,7 @@ public: throw Exception("Session not found.", ErrorCodes::SESSION_NOT_FOUND); /// Create a new session from current context. - it = sessions.insert(std::make_pair(key, std::make_shared(key, context, timeout, *this))).first; + it = sessions.insert(std::make_pair(key, std::make_shared(key, context, timeout, *this))).first; } else if (it->second->key.first != context.client_info.current_user) { @@ -154,7 +158,7 @@ public: return session; } - void releaseSession(Session & session) + void releaseSession(NamedSession & session) { std::unique_lock lock(mutex); scheduleCloseSession(session, lock); @@ -173,7 +177,7 @@ private: } }; - using Container = std::unordered_map, SessionKeyHash>; + using Container = std::unordered_map, SessionKeyHash>; using CloseTimes = std::deque>; Container sessions; CloseTimes close_times; @@ -181,7 +185,7 @@ private: std::chrono::steady_clock::time_point close_cycle_time = std::chrono::steady_clock::now(); UInt64 close_cycle = 0; - void scheduleCloseSession(Session & session, std::unique_lock &) + void scheduleCloseSession(NamedSession & session, std::unique_lock &) { /// Push it on a queue of sessions to close, on a position corresponding to the timeout. /// (timeout is measured from current moment of time) @@ -247,11 +251,11 @@ private: std::mutex mutex; std::condition_variable cond; std::atomic quit{false}; - ThreadFromGlobalPool thread{&Sessions::cleanThread, this}; + ThreadFromGlobalPool thread{&NamedSessions::cleanThread, this}; }; -void Session::release() +void NamedSession::release() { parent.releaseSession(*this); } @@ -326,8 +330,7 @@ struct ContextShared RemoteHostFilter remote_host_filter; /// Allowed URL from config.xml std::optional trace_collector; /// Thread collecting traces from threads executing queries - - Sessions sessions; /// Controls named HTTP sessions. + std::optional named_sessions; /// Controls named HTTP sessions. /// Clusters for distributed tables /// Initialized on demand (on distributed storages initialization) since Settings should be initialized @@ -501,9 +504,17 @@ Databases Context::getDatabases() } -std::shared_ptr Context::acquireSession(const String & session_id, std::chrono::steady_clock::duration timeout, bool session_check) +void Context::enableNamedSessions() { - return shared->sessions.acquireSession(session_id, *this, timeout, session_check); + shared->named_sessions.emplace(); +} + +std::shared_ptr Context::acquireNamedSession(const String & session_id, std::chrono::steady_clock::duration timeout, bool session_check) +{ + if (!shared->named_sessions) + throw Exception("Support for named sessions is not enabled", ErrorCodes::NOT_IMPLEMENTED); + + return shared->named_sessions->acquireSession(session_id, *this, timeout, session_check); } diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index d67fa3fb9c0..5b5b8bdabd5 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -103,7 +103,7 @@ class IOutputFormat; using OutputFormatPtr = std::shared_ptr; class Volume; using VolumePtr = std::shared_ptr; -struct Session; +struct NamedSession; #if USE_EMBEDDED_COMPILER @@ -180,7 +180,7 @@ private: Context * session_context = nullptr; /// Session context or nullptr. Could be equal to this. Context * global_context = nullptr; /// Global context. Could be equal to this. - friend class Sessions; + friend class NamedSessions; using SampleBlockCache = std::unordered_map; mutable SampleBlockCache sample_block_cache; @@ -422,7 +422,11 @@ public: const Databases getDatabases() const; Databases getDatabases(); - std::shared_ptr acquireSession(const String & session_id, std::chrono::steady_clock::duration timeout, bool session_check); + /// Allow to use named sessions. The thread will be run to cleanup sessions after timeout has expired. + /// The method must be called at the server startup. + void enableNamedSessions(); + + std::shared_ptr acquireNamedSession(const String & session_id, std::chrono::steady_clock::duration timeout, bool session_check); /// For methods below you may need to acquire a lock by yourself. std::unique_lock getLock() const; @@ -659,21 +663,21 @@ private: }; -class Sessions; +class NamedSessions; /// User name and session identifier. Named sessions are local to users. -using SessionKey = std::pair; +using NamedSessionKey = std::pair; /// Named sessions. The user could specify session identifier to reuse settings and temporary tables in subsequent requests. -struct Session +struct NamedSession { - SessionKey key; + NamedSessionKey key; UInt64 close_cycle = 0; Context context; std::chrono::steady_clock::duration timeout; - Sessions & parent; + NamedSessions & parent; - Session(SessionKey key_, Context & context_, std::chrono::steady_clock::duration timeout_, Sessions & parent_) + NamedSession(NamedSessionKey key_, Context & context_, std::chrono::steady_clock::duration timeout_, NamedSessions & parent_) : key(key_), context(context_), timeout(timeout_), parent(parent_) { } From d4d48a016594ee7e2ccc8c62b3ad72f24bddcabc Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 5 Mar 2020 23:02:55 +0300 Subject: [PATCH 072/712] Fix style --- dbms/src/Interpreters/Context.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index 62f2ceb32cb..b057cda53ea 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -93,6 +93,7 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; extern const int UNKNOWN_SCALAR; extern const int AUTHENTICATION_FAILED; + extern const int NOT_IMPLEMENTED; } From 5d077e0213dc70975a7579611cfc2113aadb7312 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 6 Mar 2020 00:41:46 +0300 Subject: [PATCH 073/712] Fixed error --- dbms/src/Interpreters/Context.cpp | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index a7011a1d4ae..25f2c2a679e 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -138,7 +138,7 @@ public: throw Exception("Session not found.", ErrorCodes::SESSION_NOT_FOUND); /// Create a new session from current context. - it = sessions.insert(std::make_pair(key, std::make_shared(key, context, timeout, *this))).first; + it = sessions.emplace(key, std::make_shared(key, context, timeout, *this)).first; } else if (it->second->key.first != context.client_info.current_user) { @@ -174,6 +174,8 @@ private: }; using Container = std::unordered_map, SessionKeyHash>; + + /// TODO it's very complicated. Make simple std::map with time_t or boost::multi_index. using CloseTimes = std::deque>; Container sessions; CloseTimes close_times; @@ -236,8 +238,17 @@ private: { const auto session = sessions.find(key); - if (session != sessions.end() && session->second.unique() && session->second->close_cycle <= current_cycle) - sessions.erase(session); + if (session != sessions.end() && session->second->close_cycle <= current_cycle) + { + if (!session->second.unique()) + { + /// Skip but move it to close on the next cycle. + session->second->timeout = std::chrono::steady_clock::duration{0}; + scheduleCloseSession(*session->second, lock); + } + else + sessions.erase(session); + } } close_times.pop_front(); From a0ace3e94ad87dde64d1fdc309531cb4ec718839 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 6 Mar 2020 00:50:58 +0300 Subject: [PATCH 074/712] Fixed error --- dbms/src/Interpreters/Context.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index 145c9c84ae0..dbc963e0a27 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -220,7 +220,7 @@ private: } /// Close sessions, that has been expired. Returns how long to wait for next session to be expired, if no new sessions will be added. - std::chrono::steady_clock::duration closeSessions(std::unique_lock &) + std::chrono::steady_clock::duration closeSessions(std::unique_lock & lock) { const auto now = std::chrono::steady_clock::now(); From ab808d1fc4a9e38dc2f5c7118d30be81a37a999c Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Fri, 6 Mar 2020 01:08:28 +0300 Subject: [PATCH 075/712] Code style issues. --- dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp | 3 ++- dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index 2b05cbf28d5..313c4a08980 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -41,7 +41,8 @@ MergeTreeReaderCompact::MergeTreeReaderCompact( { auto buffer = std::make_unique( full_data_path, - [&]() { + [&]() + { return data_part->disk->readFile( full_data_path, buffer_size, diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp index 393e753a4ba..bbfbebb71ae 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp @@ -80,7 +80,8 @@ MergeTreeReaderStream::MergeTreeReaderStream( { auto buffer = std::make_unique( path_prefix + data_file_extension, - [&]() { + [&]() + { return disk->readFile( path_prefix + data_file_extension, buffer_size, From f5518c0c439f0156812ae3f3c5bb1423c5c46d84 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 6 Mar 2020 01:45:59 +0300 Subject: [PATCH 076/712] Simplification --- dbms/programs/server/HTTPHandler.cpp | 10 ++-- dbms/src/Interpreters/Context.cpp | 83 ++++++++-------------------- dbms/src/Interpreters/Context.h | 7 +-- 3 files changed, 30 insertions(+), 70 deletions(-) diff --git a/dbms/programs/server/HTTPHandler.cpp b/dbms/programs/server/HTTPHandler.cpp index a4c59ff9e25..b360b0a89f4 100644 --- a/dbms/programs/server/HTTPHandler.cpp +++ b/dbms/programs/server/HTTPHandler.cpp @@ -141,15 +141,15 @@ static Poco::Net::HTTPResponse::HTTPStatus exceptionCodeToHTTPStatus(int excepti } -static std::chrono::steady_clock::duration parseSessionTimeout( +static uint32_t parseSessionTimeout( const Poco::Util::AbstractConfiguration & config, const HTMLForm & params) { - unsigned session_timeout = config.getInt("default_session_timeout", 60); + uint32_t session_timeout = config.getInt("default_session_timeout", 60); if (params.has("session_timeout")) { - unsigned max_session_timeout = config.getUInt("max_session_timeout", 3600); + uint32_t max_session_timeout = config.getUInt("max_session_timeout", 3600); std::string session_timeout_str = params.get("session_timeout"); ReadBufferFromString buf(session_timeout_str); @@ -162,7 +162,7 @@ static std::chrono::steady_clock::duration parseSessionTimeout( ErrorCodes::INVALID_SESSION_TIMEOUT); } - return std::chrono::seconds(session_timeout); + return session_timeout; } @@ -275,7 +275,7 @@ void HTTPHandler::processQuery( std::shared_ptr session; String session_id; - std::chrono::steady_clock::duration session_timeout; + uint32_t session_timeout; bool session_is_set = params.has("session_id"); const auto & config = server.config(); diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index dbc963e0a27..aa688ff9dd5 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -124,7 +124,7 @@ public: std::shared_ptr acquireSession( const String & session_id, Context & context, - std::chrono::steady_clock::duration timeout, + uint32_t timeout, bool throw_if_not_found) { std::unique_lock lock(mutex); @@ -162,7 +162,7 @@ public: void releaseSession(NamedSession & session) { std::unique_lock lock(mutex); - scheduleCloseSession(session, lock); + close_times.emplace(time(nullptr) + session.timeout, session.key); } private: @@ -178,31 +178,11 @@ private: } }; - /// TODO it's very complicated. Make simple std::map with time_t or boost::multi_index. using Container = std::unordered_map, SessionKeyHash>; - using CloseTimes = std::deque>; + using CloseTimes = std::multimap; + Container sessions; CloseTimes close_times; - std::chrono::steady_clock::duration close_interval = std::chrono::seconds(1); - std::chrono::steady_clock::time_point close_cycle_time = std::chrono::steady_clock::now(); - UInt64 close_cycle = 0; - - void scheduleCloseSession(NamedSession & session, std::unique_lock &) - { - /// Push it on a queue of sessions to close, on a position corresponding to the timeout. - /// (timeout is measured from current moment of time) - - const UInt64 close_index = session.timeout / close_interval + 1; - const auto new_close_cycle = close_cycle + close_index; - - if (session.close_cycle != new_close_cycle) - { - session.close_cycle = new_close_cycle; - if (close_times.size() < close_index + 1) - close_times.resize(close_index + 1); - close_times[close_index].emplace_back(session.key); - } - } void cleanThread() { @@ -212,51 +192,32 @@ private: while (true) { - auto interval = closeSessions(lock); - - if (cond.wait_for(lock, interval, [this]() -> bool { return quit; })) + closeSessions(lock); + if (cond.wait_for(lock, std::chrono::seconds(1), [this]() -> bool { return quit; })) break; } } /// Close sessions, that has been expired. Returns how long to wait for next session to be expired, if no new sessions will be added. - std::chrono::steady_clock::duration closeSessions(std::unique_lock & lock) + void closeSessions(std::unique_lock &) { - const auto now = std::chrono::steady_clock::now(); - - /// The time to close the next session did not come - if (now < close_cycle_time) - return close_cycle_time - now; /// Will sleep until it comes. - - const auto current_cycle = close_cycle; - - ++close_cycle; - close_cycle_time = now + close_interval; - - if (close_times.empty()) - return close_interval; - - auto & sessions_to_close = close_times.front(); - - for (const auto & key : sessions_to_close) + time_t now = time(nullptr); + for (auto it = close_times.begin(); it != close_times.end();) { - const auto session = sessions.find(key); + if (it->first >= now) + break; - if (session != sessions.end() && session->second->close_cycle <= current_cycle) - { - if (!session->second.unique()) - { - /// Skip but move it to close on the next cycle. - session->second->timeout = std::chrono::steady_clock::duration{0}; - scheduleCloseSession(*session->second, lock); - } - else - sessions.erase(session); - } + const auto session_it = sessions.find(it->second); + it = close_times.erase(it); + + if (session_it == sessions.end()) + continue; + + if (session_it->second.unique()) + sessions.erase(session_it); + else + close_times.emplace(now + session_it->second->timeout, session_it->second->key); /// Does not invalidate iterators. } - - close_times.pop_front(); - return close_interval; } std::mutex mutex; @@ -520,7 +481,7 @@ void Context::enableNamedSessions() shared->named_sessions.emplace(); } -std::shared_ptr Context::acquireNamedSession(const String & session_id, std::chrono::steady_clock::duration timeout, bool session_check) +std::shared_ptr Context::acquireNamedSession(const String & session_id, uint32_t timeout, bool session_check) { if (!shared->named_sessions) throw Exception("Support for named sessions is not enabled", ErrorCodes::NOT_IMPLEMENTED); diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index 5b5b8bdabd5..13053e4c8f2 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -426,7 +426,7 @@ public: /// The method must be called at the server startup. void enableNamedSessions(); - std::shared_ptr acquireNamedSession(const String & session_id, std::chrono::steady_clock::duration timeout, bool session_check); + std::shared_ptr acquireNamedSession(const String & session_id, uint32_t timeout, bool session_check); /// For methods below you may need to acquire a lock by yourself. std::unique_lock getLock() const; @@ -672,12 +672,11 @@ using NamedSessionKey = std::pair; struct NamedSession { NamedSessionKey key; - UInt64 close_cycle = 0; Context context; - std::chrono::steady_clock::duration timeout; + uint32_t timeout; NamedSessions & parent; - NamedSession(NamedSessionKey key_, Context & context_, std::chrono::steady_clock::duration timeout_, NamedSessions & parent_) + NamedSession(NamedSessionKey key_, Context & context_, uint32_t timeout_, NamedSessions & parent_) : key(key_), context(context_), timeout(timeout_), parent(parent_) { } From 7dd9b057874fd8c434e00e61ba4e1d3ec0ffd482 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 6 Mar 2020 03:43:17 +0300 Subject: [PATCH 077/712] Make sure that all columns are generated in performance test --- .../performance/generate_table_function.xml | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/dbms/tests/performance/generate_table_function.xml b/dbms/tests/performance/generate_table_function.xml index 4674b81af99..02dcbc493df 100644 --- a/dbms/tests/performance/generate_table_function.xml +++ b/dbms/tests/performance/generate_table_function.xml @@ -8,23 +8,23 @@ - SELECT COUNT(*) FROM (SELECT * FROM generate('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8') LIMIT 100000); - SELECT COUNT(*) FROM (SELECT * FROM generate('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8', 10, 10, 1) LIMIT 100000); - SELECT COUNT(*) FROM (SELECT * FROM generate('i Enum8(\'hello\' = 1, \'world\' = 5)', 10, 10, 1) LIMIT 100000); - SELECT COUNT(*) FROM (SELECT * FROM generate('i Array(Nullable(Enum8(\'hello\' = 1, \'world\' = 5)))', 10, 10, 1) LIMIT 100000); - SELECT COUNT(*) FROM (SELECT * FROM generate('i Nullable(Enum16(\'h\' = 1, \'w\' = 5 , \'o\' = -200)))', 10, 10, 1) LIMIT 100000); - SELECT COUNT(*) FROM (SELECT * FROM generate('d Date, dt DateTime, dtm DateTime(\'Europe/Moscow\')', 10, 10, 1) LIMIT 100000); - SELECT COUNT(*) FROM (SELECT * FROM generate('dt64 DateTime64, dts64 DateTime64(6), dtms64 DateTime64(6 ,\'Europe/Moscow\')', 10, 10, 1) LIMIT 100000); - SELECT COUNT(*) FROM (SELECT * FROM generate('f32 Float32, f64 Float64', 10, 10, 1) LIMIT 100000); - SELECT COUNT(*) FROM (SELECT * FROM generate('d32 Decimal32(4), d64 Decimal64(8), d128 Decimal128(16)', 10, 10, 1) LIMIT 100000); - SELECT COUNT(*) FROM (SELECT * FROM generate('i Tuple(Int32, Int64)', 10, 10, 1) LIMIT 100000); - SELECT COUNT(*) FROM (SELECT * FROM generate('i Array(Int8)', 10, 10, 1) LIMIT 100000); - SELECT COUNT(*) FROM (SELECT * FROM generate('i Array(Nullable(Int32))', 10, 10, 1) LIMIT 100000); - SELECT COUNT(*) FROM (SELECT * FROM generate('i Tuple(Int32, Array(Int64))', 10, 10, 1) LIMIT 100000); - SELECT COUNT(*) FROM (SELECT * FROM generate('i Nullable(String)', 10, 10, 1) LIMIT 100000); - SELECT COUNT(*) FROM (SELECT * FROM generate('i Array(String)', 10, 10, 1) LIMIT 100000); - SELECT COUNT(*) FROM (SELECT * FROM generate('i UUID', 10, 10, 1) LIMIT 100000); - SELECT COUNT(*) FROM (SELECT * FROM generate('i Array(Nullable(UUID))', 10, 10, 1) LIMIT 100000); - SELECT COUNT(*) FROM (SELECT * FROM generate('i FixedString(4)', 10, 10, 1) LIMIT 100000); - SELECT COUNT(*) FROM (SELECT * FROM generate('i String', 10, 10, 1) LIMIT 100000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generate('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8') LIMIT 100000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generate('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8', 10, 10, 1) LIMIT 100000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generate('i Enum8(\'hello\' = 1, \'world\' = 5)', 10, 10, 1) LIMIT 100000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generate('i Array(Nullable(Enum8(\'hello\' = 1, \'world\' = 5)))', 10, 10, 1) LIMIT 100000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generate('i Nullable(Enum16(\'h\' = 1, \'w\' = 5 , \'o\' = -200)))', 10, 10, 1) LIMIT 100000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generate('d Date, dt DateTime, dtm DateTime(\'Europe/Moscow\')', 10, 10, 1) LIMIT 100000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generate('dt64 DateTime64, dts64 DateTime64(6), dtms64 DateTime64(6 ,\'Europe/Moscow\')', 10, 10, 1) LIMIT 100000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generate('f32 Float32, f64 Float64', 10, 10, 1) LIMIT 100000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generate('d32 Decimal32(4), d64 Decimal64(8), d128 Decimal128(16)', 10, 10, 1) LIMIT 100000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generate('i Tuple(Int32, Int64)', 10, 10, 1) LIMIT 100000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generate('i Array(Int8)', 10, 10, 1) LIMIT 100000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generate('i Array(Nullable(Int32))', 10, 10, 1) LIMIT 100000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generate('i Tuple(Int32, Array(Int64))', 10, 10, 1) LIMIT 100000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generate('i Nullable(String)', 10, 10, 1) LIMIT 100000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generate('i Array(String)', 10, 10, 1) LIMIT 100000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generate('i UUID', 10, 10, 1) LIMIT 100000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generate('i Array(Nullable(UUID))', 10, 10, 1) LIMIT 100000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generate('i FixedString(4)', 10, 10, 1) LIMIT 100000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generate('i String', 10, 10, 1) LIMIT 100000); From 2fd918a162765c41431c0aadae9b807719d7ebc8 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 6 Mar 2020 05:12:18 +0300 Subject: [PATCH 078/712] Better code --- dbms/src/Storages/StorageGenerate.cpp | 491 ------------------ dbms/src/Storages/StorageGenerateRandom.cpp | 370 +++++++++++++ ...rageGenerate.h => StorageGenerateRandom.h} | 8 +- ...te.cpp => TableFunctionGenerateRandom.cpp} | 10 +- ...nerate.h => TableFunctionGenerateRandom.h} | 8 +- .../performance/generate_table_function.xml | 38 +- 6 files changed, 403 insertions(+), 522 deletions(-) delete mode 100644 dbms/src/Storages/StorageGenerate.cpp create mode 100644 dbms/src/Storages/StorageGenerateRandom.cpp rename dbms/src/Storages/{StorageGenerate.h => StorageGenerateRandom.h} (65%) rename dbms/src/TableFunctions/{TableFunctionGenerate.cpp => TableFunctionGenerateRandom.cpp} (82%) rename dbms/src/TableFunctions/{TableFunctionGenerate.h => TableFunctionGenerateRandom.h} (52%) diff --git a/dbms/src/Storages/StorageGenerate.cpp b/dbms/src/Storages/StorageGenerate.cpp deleted file mode 100644 index bb306d2542f..00000000000 --- a/dbms/src/Storages/StorageGenerate.cpp +++ /dev/null @@ -1,491 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -namespace DB -{ - -namespace ErrorCodes -{ -extern const int NOT_IMPLEMENTED; -extern const int LOGICAL_ERROR; -extern const int BAD_TYPE_OF_FIELD; -extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; -} - - -void fillColumnWithRandomData(IColumn & column, const DataTypePtr type, UInt64 limit, - UInt64 max_array_length, UInt64 max_string_length, pcg32 & generator, pcg64_fast & generator64) -{ - TypeIndex idx = type->getTypeId(); - - switch (idx) - { - case TypeIndex::Nothing: - throw Exception("Random Generator not implemented for type 'Nothing'.", ErrorCodes::NOT_IMPLEMENTED); - case TypeIndex::UInt8: - { - auto & data = typeid_cast &>(column).getData(); - data.resize(limit); - for (UInt64 i = 0; i < limit; ++i) - { - data[i] = static_cast(generator()); - } - break; - } - case TypeIndex::UInt16: - { - auto & data = typeid_cast &>(column).getData(); - data.resize(limit); - for (UInt64 i = 0; i < limit; ++i) - { - data[i] = static_cast(generator()); - } - break; - } - case TypeIndex::UInt32: - { - auto & data = typeid_cast &>(column).getData(); - data.resize(limit); - for (UInt64 i = 0; i < limit; ++i) - { - data[i] = static_cast(generator()); - } - break; - } - case TypeIndex::UInt64: - { - auto & data = typeid_cast &>(column).getData(); - data.resize(limit); - for (UInt64 i = 0; i < limit; ++i) - { - UInt64 a = static_cast(generator64()); - data[i] = static_cast(a); - } - break; - } - case TypeIndex::UInt128: - throw Exception("There is no DataType 'UInt128' support.", ErrorCodes::NOT_IMPLEMENTED); - case TypeIndex::Int8: - { - auto & data = typeid_cast &>(column).getData(); - data.resize(limit); - for (UInt64 i = 0; i < limit; ++i) - { - data[i] = static_cast(generator()); - } - break; - } - case TypeIndex::Int16: - { - auto & data = typeid_cast &>(column).getData(); - data.resize(limit); - for (UInt64 i = 0; i < limit; ++i) - { - data[i] = static_cast(generator()); - } - break; - } - case TypeIndex::Int32: - { - auto & data = typeid_cast &>(column).getData(); - data.resize(limit); - for (UInt64 i = 0; i < limit; ++i) - { - data[i] = static_cast(generator()); - } - break; - } - case TypeIndex::Int64: - { - auto & data = typeid_cast &>(column).getData(); - data.resize(limit); - for (UInt64 i = 0; i < limit; ++i) - { - data[i] = static_cast(generator64()); - } - break; - } - case TypeIndex::Int128: - throw Exception("There is no DataType 'Int128' support.", ErrorCodes::NOT_IMPLEMENTED); - case TypeIndex::Float32: - { - auto & data = typeid_cast &>(column).getData(); - data.resize(limit); - double d = 1.0; - for (UInt64 i = 0; i < limit; ++i) - { - d = std::numeric_limits::max(); - data[i] = (d / pcg32::max()) * generator(); - } - break; - } - case TypeIndex::Float64: - { - auto & data = typeid_cast &>(column).getData(); - data.resize(limit); - double d = 1.0; - for (UInt64 i = 0; i < limit; ++i) - { - d = std::numeric_limits::max(); - data[i] = (d / pcg64::max()) * generator64(); - } - break; - } - case TypeIndex::Date: - { - auto & data = typeid_cast &>(column).getData(); - data.resize(limit); - for (UInt64 i = 0; i < limit; ++i) - { - data[i] = static_cast(generator()); - } - break; - } - case TypeIndex::DateTime: - { - auto & data = typeid_cast &>(column).getData(); - data.resize(limit); - for (UInt64 i = 0; i < limit; ++i) - { - data[i] = static_cast(generator()); - } - break; - } - case TypeIndex::DateTime64: - { - UInt32 scale; - if (auto * ptype = typeid_cast(type.get())) - scale = ptype->getScale(); - else - throw Exception("Static cast to DataTypeDateTime64 failed ", ErrorCodes::BAD_TYPE_OF_FIELD); - auto & data = typeid_cast &>(column).getData(); - data.resize(limit); - for (UInt64 i = 0; i < limit; ++i) - { - UInt32 fractional = static_cast(generator()) % intExp10(scale); - UInt32 whole = static_cast(generator()); - DateTime64 dt = DecimalUtils::decimalFromComponents(whole, fractional, scale); - data[i] = dt; - } - break; - } - case TypeIndex::String: - { - auto & column_string = typeid_cast(column); - auto & offsets = column_string.getOffsets(); - auto & chars = column_string.getChars(); - - UInt64 offset = 0; - { - offsets.resize(limit); - for (UInt64 i = 0; i < limit; ++i) - { - offset += 1 + static_cast(generator()) % max_string_length; - offsets[i] = offset; - } - chars.resize(offset); - for (UInt64 i = 0; i < offset; ++i) - { - if (offset - i > 5) - { - UInt32 r = generator(); - chars[i] = 32 + (r & 0x7F) % 95; - chars[i + 1] = 32 + ((r >> 7) & 0x7F) % 95; - chars[i + 2] = 32 + ((r >> 14) & 0x7F) % 95; - chars[i + 3] = 32 + ((r >> 21) & 0x7F) % 95; - chars[i + 4] = 32 + (r >> 28); - i += 4; - } - else - { - UInt32 r = generator(); - chars[i] = 32 + (r % 95); - } - } - // add terminating zero char - for (auto & i : offsets) - { - chars[i - 1] = 0; - } - } - break; - } - case TypeIndex::FixedString: - { - auto & column_string = typeid_cast(column); - const size_t len = column_string.sizeOfValueIfFixed(); - auto & chars = column_string.getChars(); - - UInt64 num_chars = static_cast(len) * limit; - { - chars.resize(num_chars); - for (UInt64 i = 0; i < num_chars; ++i) - { - chars[i] = static_cast(generator()); - } - } - break; - } - case TypeIndex::Enum8: - { - auto values = typeid_cast *>(type.get())->getValues(); - auto & data = typeid_cast &>(column).getData(); - data.resize(limit); - - UInt8 size = values.size(); - UInt8 off; - for (UInt64 i = 0; i < limit; ++i) - { - off = static_cast(generator()) % size; - data[i] = values[off].second; - } - break; - } - case TypeIndex::Enum16: - { - auto values = typeid_cast *>(type.get())->getValues(); - auto & data = typeid_cast &>(column).getData(); - data.resize(limit); - - UInt16 size = values.size(); - UInt8 off; - for (UInt64 i = 0; i < limit; ++i) - { - off = static_cast(generator()) % size; - data[i] = values[off].second; - } - break; - } - case TypeIndex::Decimal32: - { - auto & data = typeid_cast &>(column).getData(); - data.resize(limit); - for (UInt64 i = 0; i < limit; ++i) - { - data[i] = static_cast(generator()); - } - break; - } - case TypeIndex::Decimal64: - { - auto & data = typeid_cast &>(column).getData(); - data.resize(limit); - for (UInt64 i = 0; i < limit; ++i) - { - UInt64 a = static_cast(generator()) << 32 | static_cast(generator()); - data[i] = a; - } - break; - } - case TypeIndex::Decimal128: - { - auto & data = typeid_cast &>(column).getData(); - data.resize(limit); - for (UInt64 i = 0; i < limit; ++i) - { - Int128 x = static_cast(generator64()) << 64 | static_cast(generator64()); - data[i] = x; - } - break; - } - case TypeIndex::UUID: - { - auto & data = typeid_cast &>(column).getData(); - data.resize(limit); - for (UInt64 i = 0; i < limit; ++i) - { - UInt64 a = static_cast(generator64()); - UInt64 b = static_cast(generator64()); - auto x = UInt128(a, b); - data[i] = x; - } - break; - } - case TypeIndex::Array: - { - auto & column_array = typeid_cast(column); - auto nested_type = typeid_cast(type.get())->getNestedType(); - - auto & offsets = column_array.getOffsets(); - IColumn & data = column_array.getData(); - - UInt64 offset = 0; - { - offsets.resize(limit); - for (UInt64 i = 0; i < limit; ++i) - { - offset += static_cast(generator()) % max_array_length; - offsets[i] = offset; - } - } - fillColumnWithRandomData(data, nested_type, offset, max_array_length, max_string_length, generator, generator64); - break; - } - case TypeIndex::Tuple: - { - auto &column_tuple = typeid_cast(column); - auto elements = typeid_cast(type.get())->getElements(); - - for (size_t i = 0; i < column_tuple.tupleSize(); ++i) - { - fillColumnWithRandomData(column_tuple.getColumn(i), elements[i], limit, max_array_length, max_string_length, generator, generator64); - } - break; - } - case TypeIndex::Set: - throw Exception("Type 'Set' can not be stored in a table.", ErrorCodes::LOGICAL_ERROR); - case TypeIndex::Interval: - throw Exception("Type 'Interval' can not be stored in a table.", ErrorCodes::LOGICAL_ERROR); - case TypeIndex::Nullable: - { - auto & column_nullable = typeid_cast(column); - auto nested_type = typeid_cast(type.get())->getNestedType(); - - auto & null_map = column_nullable.getNullMapData(); - IColumn & nested_column = column_nullable.getNestedColumn(); - - fillColumnWithRandomData(nested_column, nested_type, limit, max_array_length, max_string_length, generator, generator64); - - null_map.resize(limit); - for (UInt64 i = 0; i < limit; ++i) - { - null_map[i] = generator() < 1024; /// No real motivation for this. - } - break; - } - case TypeIndex::Function: - throw Exception("Type 'Function' can not be stored in a table.", ErrorCodes::LOGICAL_ERROR); - case TypeIndex::AggregateFunction: - throw Exception("Random Generator not implemented for type 'AggregateFunction'.", ErrorCodes::NOT_IMPLEMENTED); - case TypeIndex::LowCardinality: - throw Exception("Random Generator not implemented for type 'LowCardinality'.", ErrorCodes::NOT_IMPLEMENTED); - } -} - -StorageGenerate::StorageGenerate(const StorageID & table_id_, const ColumnsDescription & columns_, - UInt64 max_array_length_, UInt64 max_string_length_, UInt64 random_seed_) - : IStorage(table_id_), max_array_length(max_array_length_), max_string_length(max_string_length_) -{ - random_seed = random_seed_ ? random_seed_ : randomSeed(); - setColumns(columns_); -} - - -class GenerateSource : public SourceWithProgress -{ -public: - GenerateSource(UInt64 block_size_, UInt64 max_array_length_, UInt64 max_string_length_, UInt64 random_seed_, Block block_header_) - : SourceWithProgress(block_header_), block_size(block_size_), max_array_length(max_array_length_), max_string_length(max_string_length_) - , block_header(block_header_), r32(random_seed_), r64(random_seed_) {} - - String getName() const override { return "Generate"; } - -protected: - Chunk generate() override - { - auto columns = block_header.cloneEmptyColumns(); - DataTypes types = block_header.getDataTypes(); - auto cur_type = types.cbegin(); - for (auto & col : columns) - { - fillColumnWithRandomData(col->assumeMutableRef(), *cur_type, block_size, max_array_length, max_string_length, r32, r64); - ++cur_type; - } - return {std::move(columns), block_size}; - } - -private: - UInt64 block_size; - UInt64 max_array_length; - UInt64 max_string_length; - Block block_header; - - pcg32 r32; - pcg64_fast r64; - -}; - - -void registerStorageGenerate(StorageFactory & factory) -{ - factory.registerStorage("Generate", [](const StorageFactory::Arguments & args) - { - ASTs & engine_args = args.engine_args; - - if (engine_args.size() > 3) - throw Exception("Storage Generate requires at most three arguments: "\ - "max_array_length, max_string_length, random_seed.", - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - - UInt64 max_array_length_ = 10; - UInt64 max_string_length_ = 10; - UInt64 random_seed_ = 0; // zero for random - - /// Parsing second argument if present - if (engine_args.size() >= 1) - max_array_length_ = engine_args[0]->as().value.safeGet(); - - if (engine_args.size() >= 2) - max_string_length_ = engine_args[1]->as().value.safeGet(); - - if (engine_args.size() == 3) - random_seed_ = engine_args[2]->as().value.safeGet(); - - return StorageGenerate::create(args.table_id, args.columns, max_array_length_, max_string_length_, random_seed_); - }); -} - -Pipes StorageGenerate::read( - const Names & column_names, - const SelectQueryInfo & /*query_info*/, - const Context & /*context*/, - QueryProcessingStage::Enum /*processed_stage*/, - size_t max_block_size, - unsigned num_streams) -{ - check(column_names, true); - - Pipes pipes; - pipes.reserve(num_streams); - - const ColumnsDescription & columns_ = getColumns(); - Block block_header; - for (const auto & name : column_names) - { - const auto & name_type = columns_.get(name); - MutableColumnPtr column = name_type.type->createColumn(); - block_header.insert({std::move(column), name_type.type, name_type.name}); - } - - pcg32 generate(random_seed); - for (UInt64 i = 0; i < num_streams; ++i) - { - pipes.emplace_back(std::make_shared(max_block_size, max_array_length, max_string_length, generate(), block_header)); - } - return pipes; -} - -} diff --git a/dbms/src/Storages/StorageGenerateRandom.cpp b/dbms/src/Storages/StorageGenerateRandom.cpp new file mode 100644 index 00000000000..d77c9f336e9 --- /dev/null +++ b/dbms/src/Storages/StorageGenerateRandom.cpp @@ -0,0 +1,370 @@ +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include + +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int NOT_IMPLEMENTED; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + + +namespace +{ + +void fillBufferWithRandomData(char * __restrict data, size_t size, pcg64_fast & rng) +{ + char * __restrict end = data + size; + while (data < end) + { + /// The loop can be further optimized. + UInt64 number = rng(); + unalignedStore(data, number); + data += sizeof(UInt64); /// We assume that data has 15-byte padding (see PaddedPODArray) + } +} + + +ColumnPtr fillColumnWithRandomData( + const DataTypePtr type, UInt64 limit, UInt64 max_array_length, UInt64 max_string_length, pcg64_fast & rng, const Context & context) +{ + TypeIndex idx = type->getTypeId(); + + switch (idx) + { + case TypeIndex::String: + { + Block block{ColumnWithTypeAndName{nullptr, type, "result"}}; + FunctionFactory::instance().get("randomPrintableASCII", context)->build({})->execute(block, {}, 0, limit); + return block.getByPosition(0).column; + } + + case TypeIndex::Enum8: + { + auto column = ColumnVector::create(); + auto values = typeid_cast *>(type.get())->getValues(); + auto & data = column->getData(); + data.resize(limit); + + UInt8 size = values.size(); + UInt8 off; + for (UInt64 i = 0; i < limit; ++i) + { + off = static_cast(rng()) % size; + data[i] = values[off].second; + } + + return column; + } + + case TypeIndex::Enum16: + { + auto column = ColumnVector::create(); + auto values = typeid_cast *>(type.get())->getValues(); + auto & data = column->getData(); + data.resize(limit); + + UInt16 size = values.size(); + UInt8 off; + for (UInt64 i = 0; i < limit; ++i) + { + off = static_cast(rng()) % size; + data[i] = values[off].second; + } + + return column; + } + + case TypeIndex::Array: + { + auto nested_type = typeid_cast(type.get())->getNestedType(); + + auto offsets_column = ColumnVector::create(); + auto & offsets = offsets_column->getData(); + + UInt64 offset = 0; + offsets.resize(limit); + for (UInt64 i = 0; i < limit; ++i) + { + offset += static_cast(rng()) % max_array_length; + offsets[i] = offset; + } + + auto data_column = fillColumnWithRandomData(nested_type, offset, max_array_length, max_string_length, rng, context); + + return ColumnArray::create(std::move(data_column), std::move(offsets_column)); + } + + case TypeIndex::Tuple: + { + auto elements = typeid_cast(type.get())->getElements(); + const size_t tuple_size = elements.size(); + Columns tuple_columns(tuple_size); + + for (size_t i = 0; i < tuple_size; ++i) + tuple_columns[i] = fillColumnWithRandomData(elements[i], limit, max_array_length, max_string_length, rng, context); + + return ColumnTuple::create(std::move(tuple_columns)); + } + + case TypeIndex::Nullable: + { + auto nested_type = typeid_cast(type.get())->getNestedType(); + auto nested_column = fillColumnWithRandomData(nested_type, limit, max_array_length, max_string_length, rng, context); + + auto null_map_column = ColumnUInt8::create(); + auto & null_map = null_map_column->getData(); + null_map.resize(limit); + for (UInt64 i = 0; i < limit; ++i) + null_map[i] = rng() % 16 == 0; /// No real motivation for this. + + return ColumnNullable::create(std::move(nested_column), std::move(null_map_column)); + } + + case TypeIndex::UInt8: + { + auto column = ColumnUInt8::create(); + column->getData().resize(limit); + fillBufferWithRandomData(reinterpret_cast(column->getData().data()), limit * sizeof(UInt8), rng); + return column; + } + case TypeIndex::UInt16: [[fallthrough]]; + case TypeIndex::Date: + { + auto column = ColumnUInt16::create(); + column->getData().resize(limit); + fillBufferWithRandomData(reinterpret_cast(column->getData().data()), limit * sizeof(UInt16), rng); + return column; + } + case TypeIndex::UInt32: [[fallthrough]]; + case TypeIndex::DateTime: + { + auto column = ColumnUInt32::create(); + column->getData().resize(limit); + fillBufferWithRandomData(reinterpret_cast(column->getData().data()), limit * sizeof(UInt32), rng); + return column; + } + case TypeIndex::UInt64: + { + auto column = ColumnUInt64::create(); + column->getData().resize(limit); + fillBufferWithRandomData(reinterpret_cast(column->getData().data()), limit * sizeof(UInt64), rng); + return column; + } + case TypeIndex::UInt128: [[fallthrough]]; + case TypeIndex::UUID: + { + auto column = ColumnUInt128::create(); + column->getData().resize(limit); + fillBufferWithRandomData(reinterpret_cast(column->getData().data()), limit * sizeof(UInt128), rng); + return column; + } + case TypeIndex::Int8: + { + auto column = ColumnInt8::create(); + column->getData().resize(limit); + fillBufferWithRandomData(reinterpret_cast(column->getData().data()), limit * sizeof(Int8), rng); + return column; + } + case TypeIndex::Int16: + { + auto column = ColumnInt16::create(); + column->getData().resize(limit); + fillBufferWithRandomData(reinterpret_cast(column->getData().data()), limit * sizeof(Int16), rng); + return column; + } + case TypeIndex::Int32: + { + auto column = ColumnInt32::create(); + column->getData().resize(limit); + fillBufferWithRandomData(reinterpret_cast(column->getData().data()), limit * sizeof(Int32), rng); + return column; + } + case TypeIndex::Int64: + { + auto column = ColumnInt64::create(); + column->getData().resize(limit); + fillBufferWithRandomData(reinterpret_cast(column->getData().data()), limit * sizeof(Int64), rng); + return column; + } + case TypeIndex::Float32: + { + auto column = ColumnFloat32::create(); + column->getData().resize(limit); + fillBufferWithRandomData(reinterpret_cast(column->getData().data()), limit * sizeof(Float32), rng); + return column; + } + case TypeIndex::Float64: + { + auto column = ColumnFloat64::create(); + column->getData().resize(limit); + fillBufferWithRandomData(reinterpret_cast(column->getData().data()), limit * sizeof(Float64), rng); + return column; + } + case TypeIndex::Decimal32: + { + auto column = type->createColumn(); + auto & column_concrete = typeid_cast &>(*column); + column_concrete.getData().resize(limit); + fillBufferWithRandomData(reinterpret_cast(column_concrete.getData().data()), limit * sizeof(Decimal32), rng); + return column; + } + case TypeIndex::Decimal64: [[fallthrough]]; + case TypeIndex::DateTime64: + { + auto column = type->createColumn(); + auto & column_concrete = typeid_cast &>(*column); + column_concrete.getData().resize(limit); + fillBufferWithRandomData(reinterpret_cast(column_concrete.getData().data()), limit * sizeof(Decimal64), rng); + return column; + } + case TypeIndex::Decimal128: + { + auto column = type->createColumn(); + auto & column_concrete = typeid_cast &>(*column); + column_concrete.getData().resize(limit); + fillBufferWithRandomData(reinterpret_cast(column_concrete.getData().data()), limit * sizeof(Decimal128), rng); + return column; + } + + default: + throw Exception("The 'GenerateRandom' is not implemented for type " + type->getName(), ErrorCodes::NOT_IMPLEMENTED); + } +} + + +class GenerateSource : public SourceWithProgress +{ +public: + GenerateSource(UInt64 block_size_, UInt64 max_array_length_, UInt64 max_string_length_, UInt64 random_seed_, Block block_header_, const Context & context_) + : SourceWithProgress(block_header_), block_size(block_size_), max_array_length(max_array_length_), max_string_length(max_string_length_) + , block_header(block_header_), rng(random_seed_), context(context_) {} + + String getName() const override { return "GenerateRandom"; } + +protected: + Chunk generate() override + { + Columns columns; + columns.reserve(block_header.columns()); + DataTypes types = block_header.getDataTypes(); + + for (const auto & type : types) + columns.emplace_back(fillColumnWithRandomData(type, block_size, max_array_length, max_string_length, rng, context)); + + return {std::move(columns), block_size}; + } + +private: + UInt64 block_size; + UInt64 max_array_length; + UInt64 max_string_length; + Block block_header; + + pcg64_fast rng; + + const Context & context; +}; + +} + + +StorageGenerateRandom::StorageGenerateRandom(const StorageID & table_id_, const ColumnsDescription & columns_, + UInt64 max_array_length_, UInt64 max_string_length_, UInt64 random_seed_) + : IStorage(table_id_), max_array_length(max_array_length_), max_string_length(max_string_length_) +{ + random_seed = random_seed_ ? random_seed_ : randomSeed(); + setColumns(columns_); +} + + +void registerStorageGenerateRandom(StorageFactory & factory) +{ + factory.registerStorage("GenerateRandom", [](const StorageFactory::Arguments & args) + { + ASTs & engine_args = args.engine_args; + + if (engine_args.size() > 3) + throw Exception("Storage GenerateRandom requires at most three arguments: "\ + "max_array_length, max_string_length, random_seed.", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + UInt64 max_array_length_ = 10; + UInt64 max_string_length_ = 10; + UInt64 random_seed_ = 0; // zero for random + + /// Parsing second argument if present + if (engine_args.size() >= 1) + max_array_length_ = engine_args[0]->as().value.safeGet(); + + if (engine_args.size() >= 2) + max_string_length_ = engine_args[1]->as().value.safeGet(); + + if (engine_args.size() == 3) + random_seed_ = engine_args[2]->as().value.safeGet(); + + return StorageGenerateRandom::create(args.table_id, args.columns, max_array_length_, max_string_length_, random_seed_); + }); +} + +Pipes StorageGenerateRandom::read( + const Names & column_names, + const SelectQueryInfo & /*query_info*/, + const Context & context, + QueryProcessingStage::Enum /*processed_stage*/, + size_t max_block_size, + unsigned num_streams) +{ + check(column_names, true); + + Pipes pipes; + pipes.reserve(num_streams); + + const ColumnsDescription & columns_ = getColumns(); + Block block_header; + for (const auto & name : column_names) + { + const auto & name_type = columns_.get(name); + MutableColumnPtr column = name_type.type->createColumn(); + block_header.insert({std::move(column), name_type.type, name_type.name}); + } + + /// Will create more seed values for each source from initial seed. + pcg64_fast generate(random_seed); + + for (UInt64 i = 0; i < num_streams; ++i) + pipes.emplace_back(std::make_shared(max_block_size, max_array_length, max_string_length, generate(), block_header, context)); + + return pipes; +} + +} diff --git a/dbms/src/Storages/StorageGenerate.h b/dbms/src/Storages/StorageGenerateRandom.h similarity index 65% rename from dbms/src/Storages/StorageGenerate.h rename to dbms/src/Storages/StorageGenerateRandom.h index 4bb955bbabe..7622099dcbb 100644 --- a/dbms/src/Storages/StorageGenerate.h +++ b/dbms/src/Storages/StorageGenerateRandom.h @@ -8,11 +8,11 @@ namespace DB { /* Generates random data for given schema. */ -class StorageGenerate : public ext::shared_ptr_helper, public IStorage +class StorageGenerateRandom : public ext::shared_ptr_helper, public IStorage { - friend struct ext::shared_ptr_helper; + friend struct ext::shared_ptr_helper; public: - std::string getName() const override { return "Generate"; } + std::string getName() const override { return "GenerateRandom"; } Pipes read( const Names & column_names, @@ -28,7 +28,7 @@ private: UInt64 random_seed = 0; protected: - StorageGenerate(const StorageID & table_id_, const ColumnsDescription & columns_, + StorageGenerateRandom(const StorageID & table_id_, const ColumnsDescription & columns_, UInt64 max_array_length, UInt64 max_string_length, UInt64 random_seed); }; diff --git a/dbms/src/TableFunctions/TableFunctionGenerate.cpp b/dbms/src/TableFunctions/TableFunctionGenerateRandom.cpp similarity index 82% rename from dbms/src/TableFunctions/TableFunctionGenerate.cpp rename to dbms/src/TableFunctions/TableFunctionGenerateRandom.cpp index c68206c3d51..327e941508a 100644 --- a/dbms/src/TableFunctions/TableFunctionGenerate.cpp +++ b/dbms/src/TableFunctions/TableFunctionGenerateRandom.cpp @@ -2,7 +2,7 @@ #include #include -#include +#include #include #include @@ -10,7 +10,7 @@ #include #include -#include +#include #include #include "registerTableFunctions.h" @@ -25,7 +25,7 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -StoragePtr TableFunctionGenerate::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const +StoragePtr TableFunctionGenerateRandom::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const { ASTs & args_func = ast_function->children; @@ -63,14 +63,14 @@ StoragePtr TableFunctionGenerate::executeImpl(const ASTPtr & ast_function, const ColumnsDescription columns = parseColumnsListFromString(structure, context); - auto res = StorageGenerate::create(StorageID(getDatabaseName(), table_name), columns, max_array_length, max_string_length, random_seed); + auto res = StorageGenerateRandom::create(StorageID(getDatabaseName(), table_name), columns, max_array_length, max_string_length, random_seed); res->startup(); return res; } void registerTableFunctionGenerate(TableFunctionFactory & factory) { - factory.registerFunction(TableFunctionFactory::CaseInsensitive); + factory.registerFunction(); } } diff --git a/dbms/src/TableFunctions/TableFunctionGenerate.h b/dbms/src/TableFunctions/TableFunctionGenerateRandom.h similarity index 52% rename from dbms/src/TableFunctions/TableFunctionGenerate.h rename to dbms/src/TableFunctions/TableFunctionGenerateRandom.h index e4c7f9beeba..042a5c59dbe 100644 --- a/dbms/src/TableFunctions/TableFunctionGenerate.h +++ b/dbms/src/TableFunctions/TableFunctionGenerateRandom.h @@ -4,12 +4,14 @@ namespace DB { -/* generate(structure, [max_array_length, max_string_length, random_seed]) - creates a temporary storage that generates columns with random data + +/* generateRandom(structure, [max_array_length, max_string_length, random_seed]) + * - creates a temporary storage that generates columns with random data */ -class TableFunctionGenerate : public ITableFunction +class TableFunctionGenerateRandom : public ITableFunction { public: - static constexpr auto name = "generate"; + static constexpr auto name = "generateRandom"; std::string getName() const override { return name; } private: StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const override; diff --git a/dbms/tests/performance/generate_table_function.xml b/dbms/tests/performance/generate_table_function.xml index 02dcbc493df..b703424ba49 100644 --- a/dbms/tests/performance/generate_table_function.xml +++ b/dbms/tests/performance/generate_table_function.xml @@ -8,23 +8,23 @@ - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generate('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8') LIMIT 100000); - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generate('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8', 10, 10, 1) LIMIT 100000); - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generate('i Enum8(\'hello\' = 1, \'world\' = 5)', 10, 10, 1) LIMIT 100000); - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generate('i Array(Nullable(Enum8(\'hello\' = 1, \'world\' = 5)))', 10, 10, 1) LIMIT 100000); - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generate('i Nullable(Enum16(\'h\' = 1, \'w\' = 5 , \'o\' = -200)))', 10, 10, 1) LIMIT 100000); - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generate('d Date, dt DateTime, dtm DateTime(\'Europe/Moscow\')', 10, 10, 1) LIMIT 100000); - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generate('dt64 DateTime64, dts64 DateTime64(6), dtms64 DateTime64(6 ,\'Europe/Moscow\')', 10, 10, 1) LIMIT 100000); - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generate('f32 Float32, f64 Float64', 10, 10, 1) LIMIT 100000); - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generate('d32 Decimal32(4), d64 Decimal64(8), d128 Decimal128(16)', 10, 10, 1) LIMIT 100000); - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generate('i Tuple(Int32, Int64)', 10, 10, 1) LIMIT 100000); - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generate('i Array(Int8)', 10, 10, 1) LIMIT 100000); - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generate('i Array(Nullable(Int32))', 10, 10, 1) LIMIT 100000); - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generate('i Tuple(Int32, Array(Int64))', 10, 10, 1) LIMIT 100000); - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generate('i Nullable(String)', 10, 10, 1) LIMIT 100000); - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generate('i Array(String)', 10, 10, 1) LIMIT 100000); - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generate('i UUID', 10, 10, 1) LIMIT 100000); - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generate('i Array(Nullable(UUID))', 10, 10, 1) LIMIT 100000); - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generate('i FixedString(4)', 10, 10, 1) LIMIT 100000); - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generate('i String', 10, 10, 1) LIMIT 100000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8') LIMIT 100000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8', 10, 10, 1) LIMIT 100000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Enum8(\'hello\' = 1, \'world\' = 5)', 10, 10, 1) LIMIT 100000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Array(Nullable(Enum8(\'hello\' = 1, \'world\' = 5)))', 10, 10, 1) LIMIT 100000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Nullable(Enum16(\'h\' = 1, \'w\' = 5 , \'o\' = -200)))', 10, 10, 1) LIMIT 100000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('d Date, dt DateTime, dtm DateTime(\'Europe/Moscow\')', 10, 10, 1) LIMIT 100000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('dt64 DateTime64, dts64 DateTime64(6), dtms64 DateTime64(6 ,\'Europe/Moscow\')', 10, 10, 1) LIMIT 100000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('f32 Float32, f64 Float64', 10, 10, 1) LIMIT 100000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('d32 Decimal32(4), d64 Decimal64(8), d128 Decimal128(16)', 10, 10, 1) LIMIT 100000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Tuple(Int32, Int64)', 10, 10, 1) LIMIT 100000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Array(Int8)', 10, 10, 1) LIMIT 100000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Array(Nullable(Int32))', 10, 10, 1) LIMIT 100000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Tuple(Int32, Array(Int64))', 10, 10, 1) LIMIT 100000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Nullable(String)', 10, 10, 1) LIMIT 100000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Array(String)', 10, 10, 1) LIMIT 100000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i UUID', 10, 10, 1) LIMIT 100000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Array(Nullable(UUID))', 10, 10, 1) LIMIT 100000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i FixedString(4)', 10, 10, 1) LIMIT 100000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i String', 10, 10, 1) LIMIT 100000); From e30173f54be89c709a4cf10c31180397f9d81de9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 6 Mar 2020 05:19:28 +0300 Subject: [PATCH 079/712] Fixed build --- dbms/src/Storages/registerStorages.cpp | 2 +- dbms/src/Storages/registerStorages.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/registerStorages.cpp b/dbms/src/Storages/registerStorages.cpp index e9601577f1d..f5fab52285d 100644 --- a/dbms/src/Storages/registerStorages.cpp +++ b/dbms/src/Storages/registerStorages.cpp @@ -29,7 +29,7 @@ void registerStorages() registerStorageView(factory); registerStorageMaterializedView(factory); registerStorageLiveView(factory); - registerStorageGenerate(factory); + registerStorageGenerateRandom(factory); #if USE_AWS_S3 registerStorageS3(factory); diff --git a/dbms/src/Storages/registerStorages.h b/dbms/src/Storages/registerStorages.h index ed00225104f..63a758f5b38 100644 --- a/dbms/src/Storages/registerStorages.h +++ b/dbms/src/Storages/registerStorages.h @@ -23,7 +23,7 @@ void registerStorageJoin(StorageFactory & factory); void registerStorageView(StorageFactory & factory); void registerStorageMaterializedView(StorageFactory & factory); void registerStorageLiveView(StorageFactory & factory); -void registerStorageGenerate(StorageFactory & factory); +void registerStorageGenerateRandom(StorageFactory & factory); #if USE_AWS_S3 void registerStorageS3(StorageFactory & factory); From d893eace1723a28597d81700f707927ca9765173 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 6 Mar 2020 05:19:40 +0300 Subject: [PATCH 080/712] Update tests --- .../0_stateless/01087_storage_generate.sql | 4 +- .../01087_table_function_generate.sql | 76 +++++++++---------- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/dbms/tests/queries/0_stateless/01087_storage_generate.sql b/dbms/tests/queries/0_stateless/01087_storage_generate.sql index 46d49dc165f..54ecd3007a9 100644 --- a/dbms/tests/queries/0_stateless/01087_storage_generate.sql +++ b/dbms/tests/queries/0_stateless/01087_storage_generate.sql @@ -1,5 +1,5 @@ DROP TABLE IF EXISTS test_table; -CREATE TABLE test_table(a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3), UUID)) ENGINE=Generate(); +CREATE TABLE test_table(a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3), UUID)) ENGINE=GenerateRandom(); SELECT COUNT(*) FROM (SELECT * FROM test_table LIMIT 100); DROP TABLE IF EXISTS test_table; @@ -7,7 +7,7 @@ DROP TABLE IF EXISTS test_table; SELECT '-'; DROP TABLE IF EXISTS test_table_2; -CREATE TABLE test_table_2(a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3), UUID)) ENGINE=Generate(3, 5, 10); +CREATE TABLE test_table_2(a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3), UUID)) ENGINE=GenerateRandom(3, 5, 10); SELECT * FROM test_table_2 LIMIT 100; diff --git a/dbms/tests/queries/0_stateless/01087_table_function_generate.sql b/dbms/tests/queries/0_stateless/01087_table_function_generate.sql index 0329fa81bf1..6891dd94520 100644 --- a/dbms/tests/queries/0_stateless/01087_table_function_generate.sql +++ b/dbms/tests/queries/0_stateless/01087_table_function_generate.sql @@ -3,173 +3,173 @@ SELECT toTypeName(ui32), toTypeName(i32), toTypeName(ui16), toTypeName(i16), toTypeName(ui8), toTypeName(i8) -FROM generate('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8') +FROM generateRandom('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8') LIMIT 1; SELECT ui64, i64, ui32, i32, ui16, i16, ui8, i8 -FROM generate('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8', 10, 10, 1) +FROM generateRandom('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8', 10, 10, 1) LIMIT 10; SELECT '-'; SELECT toTypeName(i) -FROM generate('i Enum8(\'hello\' = 1, \'world\' = 5)') +FROM generateRandom('i Enum8(\'hello\' = 1, \'world\' = 5)') LIMIT 1; SELECT i -FROM generate('i Enum8(\'hello\' = 1, \'world\' = 5)', 10, 10, 1) +FROM generateRandom('i Enum8(\'hello\' = 1, \'world\' = 5)', 10, 10, 1) LIMIT 10; SELECT '-'; SELECT toTypeName(i) -FROM generate('i Array(Nullable(Enum8(\'hello\' = 1, \'world\' = 5)))') +FROM generateRandom('i Array(Nullable(Enum8(\'hello\' = 1, \'world\' = 5)))') LIMIT 1; SELECT i -FROM generate('i Array(Nullable(Enum8(\'hello\' = 1, \'world\' = 5)))', 10, 10, 1) +FROM generateRandom('i Array(Nullable(Enum8(\'hello\' = 1, \'world\' = 5)))', 10, 10, 1) LIMIT 10; SELECT '-'; SELECT toTypeName(i)s -FROM generate('i Nullable(Enum16(\'h\' = 1, \'w\' = 5 , \'o\' = -200)))') +FROM generateRandom('i Nullable(Enum16(\'h\' = 1, \'w\' = 5 , \'o\' = -200)))') LIMIT 1; SELECT i -FROM generate('i Nullable(Enum16(\'h\' = 1, \'w\' = 5 , \'o\' = -200)))', 10, 10, 1) +FROM generateRandom('i Nullable(Enum16(\'h\' = 1, \'w\' = 5 , \'o\' = -200)))', 10, 10, 1) LIMIT 10; SELECT '-'; SELECT toTypeName(d), toTypeName(dt), toTypeName(dtm) -FROM generate('d Date, dt DateTime, dtm DateTime(\'Europe/Moscow\')') +FROM generateRandom('d Date, dt DateTime, dtm DateTime(\'Europe/Moscow\')') LIMIT 1; SELECT d, dt, dtm -FROM generate('d Date, dt DateTime, dtm DateTime(\'Europe/Moscow\')', 10, 10, 1) +FROM generateRandom('d Date, dt DateTime, dtm DateTime(\'Europe/Moscow\')', 10, 10, 1) LIMIT 10; SELECT '-'; SELECT toTypeName(dt64), toTypeName(dts64), toTypeName(dtms64) -FROM generate('dt64 DateTime64, dts64 DateTime64(6), dtms64 DateTime64(6 ,\'Europe/Moscow\')') +FROM generateRandom('dt64 DateTime64, dts64 DateTime64(6), dtms64 DateTime64(6 ,\'Europe/Moscow\')') LIMIT 1; SELECT dt64, dts64, dtms64 -FROM generate('dt64 DateTime64, dts64 DateTime64(6), dtms64 DateTime64(6 ,\'Europe/Moscow\')', 10, 10, 1) +FROM generateRandom('dt64 DateTime64, dts64 DateTime64(6), dtms64 DateTime64(6 ,\'Europe/Moscow\')', 10, 10, 1) LIMIT 10; SELECT '-'; SELECT toTypeName(f32), toTypeName(f64) -FROM generate('f32 Float32, f64 Float64') +FROM generateRandom('f32 Float32, f64 Float64') LIMIT 1; SELECT f32, f64 -FROM generate('f32 Float32, f64 Float64', 10, 10, 1) +FROM generateRandom('f32 Float32, f64 Float64', 10, 10, 1) LIMIT 10; SELECT '-'; SELECT toTypeName(d32), toTypeName(d64), toTypeName(d64) -FROM generate('d32 Decimal32(4), d64 Decimal64(8), d128 Decimal128(16)') +FROM generateRandom('d32 Decimal32(4), d64 Decimal64(8), d128 Decimal128(16)') LIMIT 1; SELECT d32, d64, d128 -FROM generate('d32 Decimal32(4), d64 Decimal64(8), d128 Decimal128(16)', 10, 10, 1) +FROM generateRandom('d32 Decimal32(4), d64 Decimal64(8), d128 Decimal128(16)', 10, 10, 1) LIMIT 10; SELECT '-'; SELECT toTypeName(i) -FROM generate('i Tuple(Int32, Int64)') +FROM generateRandom('i Tuple(Int32, Int64)') LIMIT 1; SELECT i -FROM generate('i Tuple(Int32, Int64)', 10, 10, 1) +FROM generateRandom('i Tuple(Int32, Int64)', 10, 10, 1) LIMIT 10; SELECT '-'; SELECT toTypeName(i) -FROM generate('i Array(Int8)') +FROM generateRandom('i Array(Int8)') LIMIT 1; SELECT i -FROM generate('i Array(Int8)', 10, 10, 1) +FROM generateRandom('i Array(Int8)', 10, 10, 1) LIMIT 10; SELECT '-'; SELECT toTypeName(i) -FROM generate('i Array(Nullable(Int32))') +FROM generateRandom('i Array(Nullable(Int32))') LIMIT 1; SELECT i -FROM generate('i Array(Nullable(Int32))', 10, 10, 1) +FROM generateRandom('i Array(Nullable(Int32))', 10, 10, 1) LIMIT 10; SELECT '-'; SELECT toTypeName(i) -FROM generate('i Tuple(Int32, Array(Int64))') +FROM generateRandom('i Tuple(Int32, Array(Int64))') LIMIT 1; SELECT i -FROM generate('i Tuple(Int32, Array(Int64))', 10, 10, 1) +FROM generateRandom('i Tuple(Int32, Array(Int64))', 10, 10, 1) LIMIT 10; SELECT '-'; SELECT toTypeName(i) -FROM generate('i Nullable(String)', 1) +FROM generateRandom('i Nullable(String)', 1) LIMIT 1; SELECT i -FROM generate('i Nullable(String)', 10, 10, 1) +FROM generateRandom('i Nullable(String)', 10, 10, 1) LIMIT 10; SELECT '-'; SELECT toTypeName(i) -FROM generate('i Array(String)') +FROM generateRandom('i Array(String)') LIMIT 1; SELECT i -FROM generate('i Array(String)', 10, 10, 1) +FROM generateRandom('i Array(String)', 10, 10, 1) LIMIT 10; SELECT '-'; SELECT toTypeName(i) -FROM generate('i UUID') +FROM generateRandom('i UUID') LIMIT 1; SELECT i -FROM generate('i UUID', 10, 10, 1) +FROM generateRandom('i UUID', 10, 10, 1) LIMIT 10; SELECT '-'; SELECT toTypeName(i) -FROM generate('i Array(Nullable(UUID))') +FROM generateRandom('i Array(Nullable(UUID))') LIMIT 1; SELECT i -FROM generate('i Array(Nullable(UUID))', 10, 10, 1) +FROM generateRandom('i Array(Nullable(UUID))', 10, 10, 1) LIMIT 10; SELECT '-'; SELECT toTypeName(i) -FROM generate('i FixedString(4)') +FROM generateRandom('i FixedString(4)') LIMIT 1; SELECT i -FROM generate('i FixedString(4)', 10, 10, 1) +FROM generateRandom('i FixedString(4)', 10, 10, 1) LIMIT 10; SELECT '-'; SELECT toTypeName(i) -FROM generate('i String') +FROM generateRandom('i String') LIMIT 1; SELECT i -FROM generate('i String', 10, 10, 1) +FROM generateRandom('i String', 10, 10, 1) LIMIT 10; SELECT '-'; DROP TABLE IF EXISTS test_table; CREATE TABLE test_table(a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3), UUID)) ENGINE=Memory; -INSERT INTO test_table SELECT * FROM generate('a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3), UUID)', 2, 10, 1) +INSERT INTO test_table SELECT * FROM generateRandom('a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3), UUID)', 2, 10, 1) LIMIT 10; SELECT * FROM test_table; @@ -180,7 +180,7 @@ SELECT '-'; DROP TABLE IF EXISTS test_table_2; CREATE TABLE test_table_2(a Array(Int8), b UInt32, c Nullable(String), d Decimal32(4), e Nullable(Enum16('h' = 1, 'w' = 5 , 'o' = -200)), f Float64, g Tuple(Date, DateTime, DateTime64, UUID), h FixedString(2)) ENGINE=Memory; -INSERT INTO test_table_2 SELECT * FROM generate('a Array(Int8), b UInt32, c Nullable(String), d Decimal32(4), e Nullable(Enum16(\'h\' = 1, \'w\' = 5 , \'o\' = -200)), f Float64, g Tuple(Date, DateTime, DateTime64, UUID), h FixedString(2)', 3, 5, 10) +INSERT INTO test_table_2 SELECT * FROM generateRandom('a Array(Int8), b UInt32, c Nullable(String), d Decimal32(4), e Nullable(Enum16(\'h\' = 1, \'w\' = 5 , \'o\' = -200)), f Float64, g Tuple(Date, DateTime, DateTime64, UUID), h FixedString(2)', 3, 5, 10) LIMIT 10; SELECT * FROM test_table_2; From ca24156be9d46a86e37c904a07cee40c1aa319e3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 6 Mar 2020 05:19:55 +0300 Subject: [PATCH 081/712] Updated docs --- docs/en/operations/table_engines/generate.md | 8 ++++---- docs/en/query_language/table_functions/generate.md | 6 +++--- docs/toc_en.yml | 3 ++- docs/toc_fa.yml | 3 ++- docs/toc_ja.yml | 3 ++- docs/toc_ru.yml | 3 ++- docs/toc_zh.yml | 3 ++- 7 files changed, 17 insertions(+), 12 deletions(-) diff --git a/docs/en/operations/table_engines/generate.md b/docs/en/operations/table_engines/generate.md index 126acb05626..bdf52f84ac1 100644 --- a/docs/en/operations/table_engines/generate.md +++ b/docs/en/operations/table_engines/generate.md @@ -1,6 +1,6 @@ -# Generate {#table_engines-generate} +# GenerateRandom {#table_engines-generate} -The Generate table engine produces random data for given table schema. +The GenerateRandom table engine produces random data for given table schema. Usage examples: @@ -10,7 +10,7 @@ Usage examples: ## Usage in ClickHouse Server ```sql -Generate(max_array_length, max_string_length, random_seed) +ENGINE = GenerateRandom(max_array_length, max_string_length, random_seed) ``` The `max_array_length` and `max_string_length` parameters specify maximum length of all @@ -25,7 +25,7 @@ It supports all [DataTypes](../../data_types/index.md) that can be stored in a t **1.** Set up the `generate_engine_table` table: ```sql -CREATE TABLE generate_engine_table (name String, value UInt32) ENGINE=Generate(3, 5, 1) +CREATE TABLE generate_engine_table (name String, value UInt32) ENGINE=GenerateRandom(3, 5, 1) ``` **2.** Query the data: diff --git a/docs/en/query_language/table_functions/generate.md b/docs/en/query_language/table_functions/generate.md index ed9e2150b03..2f43bf453db 100644 --- a/docs/en/query_language/table_functions/generate.md +++ b/docs/en/query_language/table_functions/generate.md @@ -1,11 +1,11 @@ -# generate +# generateRandom Generates random data with given schema. Allows to populate test tables with data. Supports all data types that can be stored in table except `LowCardinality` and `AggregateFunction`. ```sql -generate('name TypeName[, name TypeName]...', 'limit'[, 'max_array_length'[, 'max_string_length'[, 'random_seed']]]); +generateRandom('name TypeName[, name TypeName]...', 'limit'[, 'max_array_length'[, 'max_string_length'[, 'random_seed']]]); ``` **Parameters** @@ -25,7 +25,7 @@ A table object with requested schema. ```sql -SELECT * FROM generate('a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3), UUID)', 3, 2, 10, 1); +SELECT * FROM generateRandom('a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3), UUID)', 3, 2, 10, 1); ``` ```text ┌─a────────┬────────────d─┬─c──────────────────────────────────────────────────────────────────┐ diff --git a/docs/toc_en.yml b/docs/toc_en.yml index fc8b9b99ee8..60ef9655e17 100644 --- a/docs/toc_en.yml +++ b/docs/toc_en.yml @@ -75,6 +75,7 @@ nav: - 'MaterializedView': 'operations/table_engines/materializedview.md' - 'Memory': 'operations/table_engines/memory.md' - 'Buffer': 'operations/table_engines/buffer.md' + - 'GenerateRandom': 'operations/table_engines/generate.md' - 'Database Engines': - 'Introduction': 'database_engines/index.md' @@ -143,7 +144,7 @@ nav: - 'odbc': 'query_language/table_functions/odbc.md' - 'hdfs': 'query_language/table_functions/hdfs.md' - 'input': 'query_language/table_functions/input.md' - - 'generate': 'query_language/table_functions/generate.md' + - 'generateRandom': 'query_language/table_functions/generate.md' - 'Dictionaries': - 'Introduction': 'query_language/dicts/index.md' - 'External Dictionaries': diff --git a/docs/toc_fa.yml b/docs/toc_fa.yml index 0e5604d1521..d1fde93fcdb 100644 --- a/docs/toc_fa.yml +++ b/docs/toc_fa.yml @@ -109,6 +109,7 @@ nav: - 'MaterializedView': 'operations/table_engines/materializedview.md' - 'Memory': 'operations/table_engines/memory.md' - 'Buffer': 'operations/table_engines/buffer.md' + - 'GenerateRandom': 'operations/table_engines/generate.md' - 'SQL Reference': - 'hidden': 'query_language/index.md' @@ -170,7 +171,7 @@ nav: - 'odbc': 'query_language/table_functions/odbc.md' - 'hdfs': 'query_language/table_functions/hdfs.md' - 'input': 'query_language/table_functions/input.md' - - 'generate': 'query_language/table_functions/generate.md' + - 'generateRandom': 'query_language/table_functions/generate.md' - 'Dictionaries': - 'Introduction': 'query_language/dicts/index.md' - 'External Dictionaries': diff --git a/docs/toc_ja.yml b/docs/toc_ja.yml index fea1f8780ce..319a1a140a6 100644 --- a/docs/toc_ja.yml +++ b/docs/toc_ja.yml @@ -79,6 +79,7 @@ nav: - 'MaterializedView': 'operations/table_engines/materializedview.md' - 'Memory': 'operations/table_engines/memory.md' - 'Buffer': 'operations/table_engines/buffer.md' + - 'GenerateRandom': 'operations/table_engines/generate.md' - 'SQL Reference': - 'hidden': 'query_language/index.md' @@ -142,7 +143,7 @@ nav: - 'odbc': 'query_language/table_functions/odbc.md' - 'hdfs': 'query_language/table_functions/hdfs.md' - 'input': 'query_language/table_functions/input.md' - - 'generate': 'query_language/table_functions/generate.md' + - 'generateRandom': 'query_language/table_functions/generate.md' - 'Dictionaries': - 'Introduction': 'query_language/dicts/index.md' - 'External Dictionaries': diff --git a/docs/toc_ru.yml b/docs/toc_ru.yml index 99af6d02545..b2327515250 100644 --- a/docs/toc_ru.yml +++ b/docs/toc_ru.yml @@ -80,6 +80,7 @@ nav: - 'MaterializedView': 'operations/table_engines/materializedview.md' - 'Memory': 'operations/table_engines/memory.md' - 'Buffer': 'operations/table_engines/buffer.md' + - 'GenerateRandom': 'operations/table_engines/generate.md' - 'Справка по SQL': - 'hidden': 'query_language/index.md' @@ -143,7 +144,7 @@ nav: - 'odbc': 'query_language/table_functions/odbc.md' - 'hdfs': 'query_language/table_functions/hdfs.md' - 'input': 'query_language/table_functions/input.md' - - 'generate': 'query_language/table_functions/generate.md' + - 'generateRandom': 'query_language/table_functions/generate.md' - 'Словари': - 'Введение': 'query_language/dicts/index.md' - 'Внешние словари': diff --git a/docs/toc_zh.yml b/docs/toc_zh.yml index bd5c4308bce..3d8fd5004f7 100644 --- a/docs/toc_zh.yml +++ b/docs/toc_zh.yml @@ -109,6 +109,7 @@ nav: - 'MaterializedView': 'operations/table_engines/materializedview.md' - 'Memory': 'operations/table_engines/memory.md' - 'Buffer': 'operations/table_engines/buffer.md' + - 'GenerateRandom': 'operations/table_engines/generate.md' - 'SQL语法': - 'hidden': 'query_language/index.md' @@ -170,7 +171,7 @@ nav: - 'odbc': 'query_language/table_functions/odbc.md' - 'hdfs': 'query_language/table_functions/hdfs.md' - 'input': 'query_language/table_functions/input.md' - - 'generate': 'query_language/table_functions/generate.md' + - 'generateRandom': 'query_language/table_functions/generate.md' - '字典': - '介绍': 'query_language/dicts/index.md' - '外部字典': From 12123390236677ffaf0e63de4f3306b046a64b59 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 6 Mar 2020 05:34:35 +0300 Subject: [PATCH 082/712] Fixed errors --- dbms/src/Storages/StorageGenerateRandom.cpp | 36 ++++++++++++++++++--- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/dbms/src/Storages/StorageGenerateRandom.cpp b/dbms/src/Storages/StorageGenerateRandom.cpp index d77c9f336e9..15634a9ae76 100644 --- a/dbms/src/Storages/StorageGenerateRandom.cpp +++ b/dbms/src/Storages/StorageGenerateRandom.cpp @@ -63,9 +63,23 @@ ColumnPtr fillColumnWithRandomData( { case TypeIndex::String: { - Block block{ColumnWithTypeAndName{nullptr, type, "result"}}; - FunctionFactory::instance().get("randomPrintableASCII", context)->build({})->execute(block, {}, 0, limit); - return block.getByPosition(0).column; + auto size_column = ColumnUInt32::create(); + auto & sizes = size_column->getData(); + + sizes.resize(limit); + for (UInt64 i = 0; i < limit; ++i) + sizes[i] = static_cast(rng()) % max_string_length; /// Slow + + ColumnWithTypeAndName argument{std::move(size_column), std::make_shared(), "size"}; + + Block block + { + argument, + {nullptr, type, "result"} + }; + + FunctionFactory::instance().get("randomPrintableASCII", context)->build({argument})->execute(block, {0}, 1, limit); + return block.getByPosition(1).column; } case TypeIndex::Enum8: @@ -238,8 +252,7 @@ ColumnPtr fillColumnWithRandomData( fillBufferWithRandomData(reinterpret_cast(column_concrete.getData().data()), limit * sizeof(Decimal32), rng); return column; } - case TypeIndex::Decimal64: [[fallthrough]]; - case TypeIndex::DateTime64: + case TypeIndex::Decimal64: { auto column = type->createColumn(); auto & column_concrete = typeid_cast &>(*column); @@ -255,6 +268,19 @@ ColumnPtr fillColumnWithRandomData( fillBufferWithRandomData(reinterpret_cast(column_concrete.getData().data()), limit * sizeof(Decimal128), rng); return column; } + case TypeIndex::DateTime64: + { + auto column = type->createColumn(); + auto & column_concrete = typeid_cast &>(*column); + column_concrete.getData().resize(limit); + + UInt64 range = (1ULL << 32) * intExp10(typeid_cast(*type).getScale()); + + for (size_t i = 0; i < limit; ++i) + column_concrete.getData()[i] = rng() % range; /// Slow + + return column; + } default: throw Exception("The 'GenerateRandom' is not implemented for type " + type->getName(), ErrorCodes::NOT_IMPLEMENTED); From 3a2f36dd54087eb842e55ed078bfbdfe5f0f7d29 Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Fri, 6 Mar 2020 10:31:00 +0300 Subject: [PATCH 083/712] Fixed compilation. --- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index c3998f4fbff..80604ec5428 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -221,7 +221,7 @@ MergeTreeData::MergeTreeData( /// If not choose any if (version_file.first.empty()) - version_file = {relative_data_path + "format_version.txt", storage_policy->getAnyDisk()}; + version_file = {relative_data_path + "format_version.txt", getStoragePolicy()->getAnyDisk()}; bool version_file_exists = version_file.second->exists(version_file.first); From ff119a044b86a07e5ebf1edb4442fe52264966b9 Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Fri, 6 Mar 2020 10:32:46 +0300 Subject: [PATCH 084/712] Fixed compilation. --- dbms/src/Storages/MergeTree/IMergeTreeReader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeReader.cpp b/dbms/src/Storages/MergeTree/IMergeTreeReader.cpp index 7ac74da0ec5..91d52cfa1fc 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeReader.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeReader.cpp @@ -226,7 +226,7 @@ void IMergeTreeReader::performRequiredConversions(Columns & res_columns) catch (Exception & e) { /// Better diagnostics. - e.addMessage("(while reading from part " + path + ")"); + e.addMessage("(while reading from part " + data_part->getFullPath() + ")"); throw; } } From 898eb1f1d72a44a76f7f4a0eefa409ca0829c099 Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Fri, 6 Mar 2020 11:29:38 +0300 Subject: [PATCH 085/712] Fixed compilation. --- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 80604ec5428..545c04ee002 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -3654,7 +3654,7 @@ MergeTreeData::PathsWithDisks MergeTreeData::getDataPathsWithDisks() const MergeTreeData::PathsWithDisks MergeTreeData::getRelativeDataPathsWithDisks() const { PathsWithDisks res; - auto disks = storage_policy->getDisks(); + auto disks = getStoragePolicy()->getDisks(); for (const auto & disk : disks) res.emplace_back(relative_data_path, disk); return res; From 36de225bb2803e5e9a89446286f91b703d53561b Mon Sep 17 00:00:00 2001 From: Sergei Shtykov Date: Fri, 6 Mar 2020 13:50:43 +0300 Subject: [PATCH 086/712] CLICKHOUSEDOCS-548: Adopters --- docs/en/introduction/adopters.md | 177 ++++--------------------------- 1 file changed, 22 insertions(+), 155 deletions(-) diff --git a/docs/en/introduction/adopters.md b/docs/en/introduction/adopters.md index eafb9d6d0fb..e0d409d9cfc 100644 --- a/docs/en/introduction/adopters.md +++ b/docs/en/introduction/adopters.md @@ -6,9 +6,12 @@ | Company | Industry | Usecase | Cluster Size | (Un)Compressed Data Size* | Reference | | --- | --- | --- | --- | --- | --- | | [2gis](https://2gis.ru) | Maps | Monitoring | — | — | [Talk in Russian, July 2019](https://youtu.be/58sPkXfq6nw) | +| [Aloha Browser](https://alohabrowser.com/) | Mobile App | Browser backend | — | — | [Slides in Russian, May 2019](https://github.com/yandex/clickhouse-presentations/blob/master/meetup22/aloha.pdf) | | [Amadeus](https://amadeus.com/) | Travel | Analytics | — | — | [Press Release, April 2018](https://www.altinity.com/blog/2018/4/5/amadeus-technologies-launches-investment-and-insights-tool-based-on-machine-learning-and-strategy-algorithms) | | [Appsflyer](https://www.appsflyer.com) | Mobile analytics | Main product | — | — | [Talk in Russian, July 2019](https://www.youtube.com/watch?v=M3wbRlcpBbY) | +| [ArenaData](https://arenadata.tech/) | Data Platform | Main product | — | — | [Slides in Russian, December 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup38/indexes.pdf) | | [Badoo](https://badoo.com) | Dating | Timeseries | — | — | [Slides in Russian, December 2019](https://presentations.clickhouse.tech/meetup38/forecast.pdf) | +| [Benocs](https://www.benocs.com/) | Network Telemetry and Analytics | Main Product | — | — | [Slides in English, October 2017](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup9/lpm.pdf) | | [Bloomberg](https://www.bloomberg.com/) | Finance, Media | Monitoring | 102 servers | — | [Slides, May 2018](https://www.slideshare.net/Altinity/http-analytics-for-6m-requests-per-second-using-clickhouse-by-alexander-bocharov) | | [Bloxy](https://bloxy.info) | Blockchain | Analytics | — | — | [Slides in Russian, August 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/4_bloxy.pptx) | | `Dataliance/UltraPower` | Telecom | Analytics | — | — | [Slides in Chinese, January 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup12/telecom.pdf) | @@ -20,190 +23,54 @@ | [ContentSquare](https://contentsquare.com) | Web analytics | Main product | — | — | [Blog post in French, November 2018](http://souslecapot.net/2018/11/21/patrick-chatain-vp-engineering-chez-contentsquare-penser-davantage-amelioration-continue-que-revolution-constante/) | | [Cloudflare](https://cloudflare.com) | CDN | Traffic analysis | 36 servers | — | [Blog post, May 2017](https://blog.cloudflare.com/how-cloudflare-analyzes-1m-dns-queries-per-second/), [Blog post, March 2018](https://blog.cloudflare.com/http-analytics-for-6m-requests-per-second-using-clickhouse/) | | [Corunet](https://coru.net/) | Analytics | Main product | — | — | [Slides in English, April 2019 ](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup21/predictive_models.pdf) | -| [Criteo/Storetail] | Retail | Main product | — | — | [Slides in English, October 2018 ](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup18/3_storetail.pptx) | -| [Deutsche Bank](db.com) | Finance | BI Analytics | — | — | [Slides in English, October 2019](https://bigdatadays.ru/wp-content/uploads/2019/10/D2-H3-3_Yakunin-Goihburg.pdf) | +| [CraiditX 氪信](https://creditx.com) | Finance AI | Analysis | — | — | [Slides in English, November 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/udf.pptx) | +| [Criteo/Storetail](https://www.criteo.com/) | Retail | Main product | — | — | [Slides in English, October 2018 ](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup18/3_storetail.pptx) | +| [Deutsche Bank](https://db.com) | Finance | BI Analytics | — | — | [Slides in English, October 2019](https://bigdatadays.ru/wp-content/uploads/2019/10/D2-H3-3_Yakunin-Goihburg.pdf) | +| [Diva-e](https://www.diva-e.com) | Digital consulting | Main Product | — | — | [Slides in English, September 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup29/ClickHouse-MeetUp-Unusual-Applications-sd-2019-09-17.pdf) | | [Exness](https://www.exness.com) | Trading | Metrics, Logging | — | — | [Talk in Russian, May 2019](https://youtu.be/_rpU-TvSfZ8?t=3215) | | [Geniee](https://geniee.co.jp) | Ad network | Main product | — | — | [Blog post in Japanese, July 2017](https://tech.geniee.co.jp/entry/2017/07/20/160100) | | [HUYA](https://www.huya.com/) | Video Streaming | Analytics | — | — | [Slides in Chinese, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/7.%20ClickHouse万亿数据分析实践%20李本旺(sundy-li)%20虎牙.pdf) | -| [Idealista](www.idealista.com) | Real Estate | Analytics | — | — | [Blog Post in English, April 2019](https://clickhouse.yandex/blog/en/clickhouse-meetup-in-madrid-on-april-2-2019) | +| [Idealista](https://www.idealista.com) | Real Estate | Analytics | — | — | [Blog Post in English, April 2019](https://clickhouse.yandex/blog/en/clickhouse-meetup-in-madrid-on-april-2-2019) | +| [Infovista](https://www.infovista.com/) | Networks | Analytics | — | — | [Slides in English, October 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup30/infovista.pdf) | +| [InnoGames](https://www.innogames.com) | Games | Metrics, Logging | — | — | [Slides in Russian, September 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/graphite_and_clickHouse.pdf) | +| [Integros](https://integros.com) | Platform for video services | Analytics | — | — | [Slides in Russian, May 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup22/strategies.pdf) | | [Kodiak Data](https://www.kodiakdata.com/) | Clouds | Main product | — | — | [Slides in Engish, April 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup13/kodiak_data.pdf) | | [Kontur](https://kontur.ru) | Software Development | Metrics | — | — | [Talk in Russian, November 2018](https://www.youtube.com/watch?v=U4u4Bd0FtrY) | | [LifeStreet](https://cloudflare.com) | Ad network | Main product | 60 servers in 3 replicas | 2-2.5 PiB | [Blog post in Russian, February 2017](https://habr.com/en/post/322620/) | | [Mail.ru Cloud Solutions](https://mcs.mail.ru/) | Cloud services | Main product | — | — | [Running ClickHouse Instance, in Russian](https://mcs.mail.ru/help/db-create/clickhouse#) | -| [MGID](https://www.mgid.com/) | Ad network | Web-analytics | — | — | [Our experience in implementing analytical DBMS ClickHouse, in Russian](http://gs-studio.com/news-about-it/32777----clickhouse---c) | | [MessageBird](https://www.messagebird.com) | Telecommunications | Statistics | — | — | [Slides in English, November 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup20/messagebird.pdf) | +| [MGID](https://www.mgid.com/) | Ad network | Web-analytics | — | — | [Our experience in implementing analytical DBMS ClickHouse, in Russian](http://gs-studio.com/news-about-it/32777----clickhouse---c) | | [OneAPM](https://www.oneapm.com/) | Monitorings and Data Analysis | Main product | — | — | [Slides in Chinese, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/8.%20clickhouse在OneAPM的应用%20杜龙.pdf) | | [Pragma Innovation](http://www.pragma-innovation.fr/) | Telemetry and Big Data Analysis | Main product | — | — | [Slides in English, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup18/4_pragma_innovation.pdf) | | [QINGCLOUD](https://www.qingcloud.com/) | Cloud services | Main product | — | — | [Slides in Chinese, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/4.%20Cloud%20%2B%20TSDB%20for%20ClickHouse%20张健%20QingCloud.pdf) | | [Qrator](https://qrator.net) | DDoS protection | Main product | — | — | [Blog Post, March 2019](https://blog.qrator.net/en/clickhouse-ddos-mitigation_37/) | -| [Rambler](rambler.ru) | Internet services | Analytics | — | — | [Talk in Russian, April 2018](https://medium.com/@ramblertop/разработка-api-clickhouse-для-рамблер-топ-100-f4c7e56f3141) | +| [Percent百分点](https://www.percent.cn/) | Analytics | Main Product | — | — | [Slides in Chinese, June 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/4.%20ClickHouse万亿数据双中心的设计与实践%20.pdf) | +| [Rambler](https://rambler.ru) | Internet services | Analytics | — | — | [Talk in Russian, April 2018](https://medium.com/@ramblertop/разработка-api-clickhouse-для-рамблер-топ-100-f4c7e56f3141) | | [Tencent](https://www.tencent.com) | Messaging | Logging | — | — | [Talk in Chinese, November 2019](https://youtu.be/T-iVQRuw-QY?t=5050) | | [Traffic Stars](https://trafficstars.com/) | AD network | — | — | — | [Slides in Russian, May 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup15/lightning/ninja.pdf) | | [S7 Airlines](https://www.s7.ru) | Airlines | Metrics, Logging | — | — | [Talk in Russian, March 2019](https://www.youtube.com/watch?v=nwG68klRpPg&t=15s) | | [SEMrush](https://www.semrush.com/) | Marketing | Main product | — | — | [Slides in Russian, August 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/5_semrush.pdf) | | [scireum GmbH](https://www.scireum.de/) | e-Commerce | Main product | — | — | [Talk in German, February 2020](https://www.youtube.com/watch?v=7QWAn5RbyR4) | | [Sentry](https://sentry.io/) | Software developer | Backend for product | — | — | [Blog Post in English, May 2019](https://blog.sentry.io/2019/05/16/introducing-snuba-sentrys-new-search-infrastructure) | +| [SGK](http://www.sgk.gov.tr/wps/portal/sgk/tr) | Goverment Social Security | Analytics | — | — | [Slides in English, November 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup35/ClickHouse%20Meetup-Ramazan%20POLAT.pdf) | +| [seo.do](https://seo.do/) | Analytics | Main product | — | — | [Slides in English, November 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup35/CH%20Presentation-%20Metehan%20Çetinkaya.pdf) | | [Sina](http://english.sina.com/index.html) | News | — | — | — | [Slides in Chinese, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/6.%20ClickHouse最佳实践%20高鹏_新浪.pdf) | | [SMI2](https://smi2.ru/) | News | Analytics | — | — | [Blog Post in Russian, November 2017](https://habr.com/ru/company/smi2/blog/314558/) | | [Splunk](https://www.splunk.com/) | Business Analytics | Main product | — | — | [Slides in English, January 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup12/splunk.pdf) | | [Spotify](https://www.spotify.com) | Music | Experimentation | — | — | [Slides, July 2018](https://www.slideshare.net/glebus/using-clickhouse-for-experimentation-104247173) | | [Tencent](https://www.tencent.com) | Big Data | Data processing | — | — | [Slides in Chinese, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/5.%20ClickHouse大数据集群应用_李俊飞腾讯网媒事业部.pdf) | - Нераспознанный китайский источник https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/5.%20ClickHouse大数据集群应用_李俊飞腾讯网媒事业部.pdf | [Uber](https://www.uber.com) | Taxi | Logging | — | — | [Slides, February 2020](https://presentations.clickhouse.tech/meetup40/ml.pdf) | -| [VKontakte](vk.com) | Social Network | Statistics, Logging | — | — | [Slides in Russian, August 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/3_vk.pdf) | +| [VKontakte](https://vk.com) | Social Network | Statistics, Logging | — | — | [Slides in Russian, August 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/3_vk.pdf) | +| [Wisebits](https://wisebits.com/) | IT Solutions | Analytics | — | — | [Slides in Russian, May 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup22/strategies.pdf) | +| [Xiaoxin Tech.](https://www.xiaoheiban.cn/) | Education | Common purpose | — | — | [Slides in English, November 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/sync-clickhouse-with-mysql-mongodb.pptx) | +| [Ximalaya](https://www.ximalaya.com/) | Audio sharing | OLAP | — | — | [Slides in English, November 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/ximalaya.pdf) | | [Yandex Cloud](https://cloud.yandex.ru/services/managed-clickhouse) | Public Cloud | Main product | — | — | [Talk in Russian, December 2019](https://www.youtube.com/watch?v=pgnak9e_E0o) | | [Yandex DataLens](https://cloud.yandex.ru/services/datalens) | Business Intelligence | Main product | — | — | [Slides in Russian, December 2019](https://presentations.clickhouse.tech/meetup38/datalens.pdf) | | [Yandex Market](https://market.yandex.ru/) | e-Commerce | Metrics, Logging | — | — | [Talk in Russian, January 2019](https://youtu.be/_l1qP0DyBcA?t=478) | | [Yandex Metrica](https://metrica.yandex.com) | Web analytics | Main product | 360 servers in one cluster, 1862 servers in one department | 66.41 PiB / 5.68 PiB | [Slides, February 2020](https://presentations.clickhouse.tech/meetup40/introduction/#13) | | [ЦВТ](https://htc-cs.ru/) | Software Development | Metrics, Logging | — | — | [Blog Post, March 2019, in Russian](https://vc.ru/dev/62715-kak-my-stroili-monitoring-na-prometheus-clickhouse-i-elk) | -| --- | --- | --- | --- | --- | --- | -| --- | --- | --- | --- | --- | --- | +| [МКБ](https://mkb.ru/) | Bank | Web-system monitoring | — | — | [Slides in Russian, September 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/mkb.pdf) | +| [金数据](https://jinshuju.net) | BI Analytics | Main product | — | — | [Slides in Chinese, October 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/3.%20金数据数据架构调整方案Public.pdf) | - -### Not checked mentions - -- Bioinformatics - evolutionary genetics: -https://github.com/msestak/FindOrigin - -"We are exploring evolution of novel genes in genomes because if seems that genomes are far from being static as previously believed and what actually happens is that new genes are constantly being added and old genes are lost." - -- Search engine and analytics for Bitcoin transactions: -https://blockchair.com/ - -"We have quite large tables on just single server and everything works really fast &mdash with any filters and sorting everything is processed just instantly." - -- https://www.octonica.ru/ презентация https://github.com/ClickHouse/clickhouse-presentations/blob/master/database_saturday_2018_2/octonica/meetup.pptx - - -Yandex.Mail is using ClickHouse to record user activity logs in a web interface for investigations and analytics. - -Yandex.Browser is using ClickHouse for performance histograms. Browsers send many mini-histograms from clients. -They are all stored into ClickHouse for the purpose of Browser version comparisons, analytics and investigations. - -- Infinidat https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup11/clickhouse_in_the_world.pptx -- Vertamedia -- NVidia -- Percona https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup11/clickhouse_in_the_world.pptx Но это не компания, это разработчик тулзов для OpenSource. https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup14/analyzing_mysql_logs.pptx -- BI Tableau https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup11/tableau.pptx December 2017? это дата загрузки презентации. -- https://my.com/ @mail.ru https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup11/target.pdf -- [Эрливидео](https://flussonic.ru/) https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup15/lightning/erlyvideo.key -- mipt https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup15/lightning/mipt.pdf - -### Stash - -- [СБИС, Тензор](https://sbis.ru/about), Analytics. В своих вакансиях регулярно указывают ClickHouse + есть задание для студентов 2018 https://tensor.ru/stipend2018/projects Active Manager - Анализатор активности на странице, где подразумевается использование ClickHouse. - -- Компания Republer https://republer.com/ О том, что они используют ClickHouse видно из вакансии https://telegram.me/nodejs_jobs_feed/607 - -- Kaspersky заявлен в нескольких презентациях, Однако живых публичных источников найти не получается. Поисковики приводят на сайт вакансий Касперского, но в явном виде ClickHouse там нигде не появляется. Есть https://careers.kaspersky.ru/tech/, поиск, на котором приводит на вакансию девопса, при этом в описании вакансии ClickHouse не упоминается. Вот такая штука есть https://sudonull.com/post/76060-The-second-mitap-of-the-Rust-community-in-Kaspersky-Lab-Kaspersky-Labs-blog - -- Мегафон. На сайтах вакансий от него есть несколько вакансий с ClickHouse типа https://ekaterinburg.hh.ru/vacancy/35891497?utm_source=jooble.org&utm_medium=meta&utm_campaign=RU_paid_cpc_applicant_feed_magic1 и есть вот такая хрень https://newprolab.com/ru/dataengineer-megafon, с которой непонятно, что делать. Вакансии не кажутся хоть сколько-нибудь надёжным источником, поскольку сейчас есть - завтра нет. Как у касперского. - - -- [Quantrum.Team](https://quantrum.me). Непонятное комьюнити трейдеров, один из которых решил попробовать ClickHouse. https://quantrum.me/1709-clickhouse-i-python-dlya-xraneniya-istorii-cen/ - -- https://severalnines.com/database-blog/hybrid-oltpanalytics-database-workloads-replicating-mysql-data-clickhouse какая-то компания типа альтинити видимо. - -- В презентациях есть Wikimedia Foundation, но реальных упоминаний об этом не нашел. Есть какой-то ответ в блоге https://www.mail-archive.com/analytics@lists.wikimedia.org/msg03619.html, но он не указывает прямо на использование ClickHouse. Вот этот вот указывает, но я с ходу не разобрал, что за источник вообще такой https://phabricator.wikimedia.org/T150343 - -- [Mercadona](mercadona.com) не нашел ни единой связи с ClickHouse. - -- [Zara](zara.com) не нашел связи с ClickHouse. - -- ByteDance, единственное прямое доказательство было ссылкой на уже не существующую вакансию. - -- Booking.com. Экспериментальная поддержка ClickHouse в https://github.com/bookingcom/carbonapi - -- [Roistat](https://roistat.com) разработала [Go ClickHouse Connector](https://github.com/roistat/go-clickhouse) и поддерживает свой [Docker](https://github.com/roistat/docker-clickhouse). - -- JD.com ClickHouse Meetup in Beijing 2019 -- KuaiShou ClickHouse Meetup in Beijing 2018 - -- JetBrains DataGrip ? https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/6_datagrip.pdf - [Original article](https://clickhouse.tech/docs/en/introduction/adopters/) - - - - -- bdtc_2019 -+ cern (Mail.ru, MGID,) -- cpp_russia_2019 -- cpp_sprint_2019 -- data_at_scale -- database_saturday_2018 -- database_saturday_2018_2/pipeline -+ database_saturday_2018_2/octonica -- database_saturday_2019 -+ dataops_2019 (CARTO, Mercadona, Zara, Idealista, Corunet, ... Cloudflare, Spotify, Amadeus, Bloomberg, Cisco, Deutsche Bank, Tencent, ByteDance) -+ drafts (CloudFlare, Booking.com, Crobox, Rambler, QRator, СКБ Контур, Roistat, SMI2) -+ evolution (ByteDance, Sina, Tencent, JD.com, KuaiShou) -- group_by -- hash_tables -- highload2016 -- highload2017 -- highload2018 -- highload2019 -- highload_siberia_2018 -- highload_siberia_2019 -- highload_spb_2019 -- hse_2019 -- internals -- it_rzn_2019 -+ meetup10 -+ meetup11 -+ meetup12 -+ meetup13 -- meetup14 -+ meetup15 -- meetup16 -+ meetup17 -+ meetup18 -+ meetup19 -meetup20 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago -meetup21 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago -meetup22 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago -meetup23 Added presentations from ClickHouse Meetup in San Francisco 9 months ago -meetup24 yandex/ClickHouse -> ClickHouse/ClickHouse, part 2 5 months ago -meetup25 Add lost slides from Novosibirsk 8 months ago -meetup26 add one more slide deck from Minsk meetup 8 months ago -meetup27 Added more presentations from 27th meetup. 7 months ago -meetup28 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago -meetup29 Merge branch 'master' of github.com:ClickHouse/clickhouse-presentations 5 months ago -meetup3 Presentation: added analysts part [#CLICKHOUSE-2]. 3 years ago -meetup30 Add more slides from Paris Meetup 4 months ago -meetup31 Correction on KuaiShou company name 3 months ago -meetup32 Correction on KuaiShou company name 3 months ago -meetup33 Correction on KuaiShou company name 3 months ago -meetup34 Add tokyo meetup 3 months ago -meetup35 Added half of presentations from ClickHouse Instanbul Meetup 3 months ago -meetup36/new_features Added a presentation from 36th ClickHouse Meetup 3 months ago -meetup37 Moved presentation from 37th ClickHouse Meetup 3 months ago -meetup38 Remaining Moscow meetup slides 3 months ago -meetup39 Remaining slides from SF meetup 14 days ago -meetup4 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago -meetup40 More slides from NYC meetup 19 days ago -meetup5 Added presentation from 5th meetup [#CLICKHOUSE-3]. 3 years ago -meetup6 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago -meetup7 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago -meetup9 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago -misc Added resized images for "evolution" article. 3 years ago -percona2017 percona2017 3 years ago -percona2018 Added part of presentations from Percona 2018 and Sunnyvale Meetup 2 years ago -percona2019 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago -percona_europe_2017 add presentation from Percona Europe 2017 2 years ago -percona_europe_2018 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago -percona_europe_2019 Merge branch 'master' of github.com:ClickHouse/clickhouse-presentations 5 months ago -pgday2017 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago -rit2017 add sources for RIT++2017 3 years ago -rit2018 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago -rit2019 Add rit2019 presentation 8 months ago -roadmap2018 Added roadmap for 2018..2019 15 months ago -shad2017 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago -tbd yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago -tutorials Create catboost_with_clickhouse_en.md 2 years ago -unknown_developers_reissue Added slightly modified version of "Unknown developers" presentation 15 months ago -uwdc yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago -yatalks_2019 yandex/ClickHouse -> ClickHouse/ClickHouse, part 1 5 months ago -yatalks_2019_moscow Fixed error in presentation from YaTalks 2019 in Moscow 3 months ago From bf3cd54c192424e47c6e8492721986619d35e4b4 Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Fri, 6 Mar 2020 15:28:26 +0300 Subject: [PATCH 087/712] Proper variables capturing in CachedCompressedReadBuffer. --- .../tests/cached_compressed_read_buffer.cpp | 19 +++++++++++++++++-- .../MergeTree/MergeTreeReaderCompact.cpp | 4 ++-- .../MergeTree/MergeTreeReaderStream.cpp | 4 ++-- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/dbms/src/Compression/tests/cached_compressed_read_buffer.cpp b/dbms/src/Compression/tests/cached_compressed_read_buffer.cpp index 01dcd5a9fcd..ed198e36e46 100644 --- a/dbms/src/Compression/tests/cached_compressed_read_buffer.cpp +++ b/dbms/src/Compression/tests/cached_compressed_read_buffer.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -32,7 +33,14 @@ int main(int argc, char ** argv) { Stopwatch watch; - CachedCompressedReadBuffer in(path, &cache, 0, 0, 0); + CachedCompressedReadBuffer in( + path, + [&]() + { + return createReadBufferFromFileBase(path, 0, 0, 0); + }, + &cache + ); WriteBufferFromFile out("/dev/null"); copyData(in, out); @@ -44,7 +52,14 @@ int main(int argc, char ** argv) { Stopwatch watch; - CachedCompressedReadBuffer in(path, &cache, 0, 0, 0); + CachedCompressedReadBuffer in( + path, + [&]() + { + return createReadBufferFromFileBase(path, 0, 0, 0); + }, + &cache + ); WriteBufferFromFile out("/dev/null"); copyData(in, out); diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index 313c4a08980..2c0a35df107 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -40,8 +40,8 @@ MergeTreeReaderCompact::MergeTreeReaderCompact( if (uncompressed_cache) { auto buffer = std::make_unique( - full_data_path, - [&]() + fullPath(data_part->disk, full_data_path), + [this, full_data_path, buffer_size]() { return data_part->disk->readFile( full_data_path, diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp index bbfbebb71ae..90415f6eace 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp @@ -79,8 +79,8 @@ MergeTreeReaderStream::MergeTreeReaderStream( if (uncompressed_cache) { auto buffer = std::make_unique( - path_prefix + data_file_extension, - [&]() + fullPath(disk, path_prefix + data_file_extension), + [this, buffer_size, sum_mark_range_bytes, &settings]() { return disk->readFile( path_prefix + data_file_extension, From 117845b2be10460952776a8c2b1a7307cd857517 Mon Sep 17 00:00:00 2001 From: Sergei Shtykov Date: Fri, 6 Mar 2020 15:44:22 +0300 Subject: [PATCH 088/712] CLICKHOUSEDOCS-548: Repaired master build. --- .../operations/settings/query_complexity.md | 24 +++++++++++++++++-- docs/en/operations/settings/settings.md | 2 +- .../operations/settings/query_complexity.md | 4 ++-- docs/ru/operations/settings/settings.md | 2 +- docs/toc_en.yml | 1 + docs/toc_fa.yml | 1 + docs/toc_ja.yml | 4 ++-- docs/toc_ru.yml | 1 + docs/toc_zh.yml | 1 + 9 files changed, 32 insertions(+), 8 deletions(-) diff --git a/docs/en/operations/settings/query_complexity.md b/docs/en/operations/settings/query_complexity.md index e5f993540a7..e20d2fe926d 100644 --- a/docs/en/operations/settings/query_complexity.md +++ b/docs/en/operations/settings/query_complexity.md @@ -96,7 +96,7 @@ Maximum number of bytes before sorting. What to do if the number of rows received before sorting exceeds one of the limits: 'throw' or 'break'. By default, throw. -## max_result_rows +## max_result_rows {#setting-max_result_rows} Limit on the number of rows in the result. Also checked for subqueries, and on remote servers when running parts of a distributed query. @@ -107,7 +107,27 @@ Limit on the number of bytes in the result. The same as the previous setting. ## result_overflow_mode What to do if the volume of the result exceeds one of the limits: 'throw' or 'break'. By default, throw. -Using 'break' is similar to using LIMIT. + + +Using 'break' is similar to using LIMIT. Break interrupts execution only at the block level. This means that amount of returned rows is greater than [max_result_rows](#setting-max_result_rows), multiple of [max_block_size](settings.md#setting-max_block_size) and depends on [max_threads](settings.md#settings-max_threads). + +Пример: +```sql +SET max_threads = 3, max_block_size = 3333; +SET max_result_rows = 3334, result_overflow_mode = 'break'; + +SELECT * +FROM numbers_mt(100000) +FORMAT Null; +``` + +Результат: + +```text +6666 rows in set. ... +``` + + ## max_execution_time diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 4928fba1ca8..6afba6f4fa4 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -406,7 +406,7 @@ Possible values: Default value: 1. -## max_block_size +## max_block_size {#setting-max_block_size} In ClickHouse, data is processed by blocks (sets of column parts). The internal processing cycles for a single block are efficient enough, but there are noticeable expenditures on each block. The `max_block_size` setting is a recommendation for what size of block (in number of rows) to load from tables. The block size shouldn't be too small, so that the expenditures on each block are still noticeable, but not too large, so that the query with LIMIT that is completed after the first block is processed quickly. The goal is to avoid consuming too much memory when extracting a large number of columns in multiple threads, and to preserve at least some cache locality. diff --git a/docs/ru/operations/settings/query_complexity.md b/docs/ru/operations/settings/query_complexity.md index 3d0c8dd9c38..01ab8745840 100644 --- a/docs/ru/operations/settings/query_complexity.md +++ b/docs/ru/operations/settings/query_complexity.md @@ -97,7 +97,7 @@ Что делать, если количество строк, полученное перед сортировкой, превысило одно из ограничений: throw или break. По умолчанию: throw. -## max_result_rows +## max_result_rows {#setting-max_result_rows} Ограничение на количество строк результата. Проверяются также для подзапросов и на удалённых серверах при выполнении части распределённого запроса. @@ -109,7 +109,7 @@ Что делать, если объём результата превысил одно из ограничений: throw или break. По умолчанию: throw. -Использование break по смыслу похоже на LIMIT. Break прерывает выполнение только на уровне блока. Т.е. число строк которые вернет запрос будет больше чем ограничение [max_result_rows](#max_result_rows), кратно [max_block_size](settings.md#max_block_size) и зависит от [max_threads](settings.md#settings-max_threads). +Использование break по смыслу похоже на LIMIT. Break прерывает выполнение только на уровне блока. Т.е. число строк которые вернет запрос будет больше чем ограничение [max_result_rows](#setting-max_result_rows), кратно [max_block_size](settings.md#setting-max_block_size) и зависит от [max_threads](settings.md#settings-max_threads). Пример: ```sql diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 298dd7364c3..80486502dcb 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -377,7 +377,7 @@ Ok. Значение по умолчанию: 0. -## max_block_size +## max_block_size {#setting-max_block_size} Данные в ClickHouse обрабатываются по блокам (наборам кусочков столбцов). Внутренние циклы обработки для одного блока достаточно эффективны, но есть заметные издержки на каждый блок. Настройка `max_block_size` — это рекомендация, какой размер блока (в количестве строк) загружать из таблиц. Размер блока не должен быть слишком маленьким, чтобы затраты на каждый блок были заметны, но не слишком велики, чтобы запрос с LIMIT, который завершается после первого блока, обрабатывался быстро. Цель состоит в том, чтобы не использовалось слишком много оперативки при вынимании большого количества столбцов в несколько потоков; чтобы оставалась хоть какая-нибудь кэш-локальность. diff --git a/docs/toc_en.yml b/docs/toc_en.yml index fc8b9b99ee8..98c3e113fb6 100644 --- a/docs/toc_en.yml +++ b/docs/toc_en.yml @@ -75,6 +75,7 @@ nav: - 'MaterializedView': 'operations/table_engines/materializedview.md' - 'Memory': 'operations/table_engines/memory.md' - 'Buffer': 'operations/table_engines/buffer.md' + - 'Generate': 'operations/table_engines/generate.md' - 'Database Engines': - 'Introduction': 'database_engines/index.md' diff --git a/docs/toc_fa.yml b/docs/toc_fa.yml index 0e5604d1521..47197de62d9 100644 --- a/docs/toc_fa.yml +++ b/docs/toc_fa.yml @@ -109,6 +109,7 @@ nav: - 'MaterializedView': 'operations/table_engines/materializedview.md' - 'Memory': 'operations/table_engines/memory.md' - 'Buffer': 'operations/table_engines/buffer.md' + - 'Generate': 'operations/table_engines/generate.md' - 'SQL Reference': - 'hidden': 'query_language/index.md' diff --git a/docs/toc_ja.yml b/docs/toc_ja.yml index fea1f8780ce..916459b0e04 100644 --- a/docs/toc_ja.yml +++ b/docs/toc_ja.yml @@ -79,7 +79,7 @@ nav: - 'MaterializedView': 'operations/table_engines/materializedview.md' - 'Memory': 'operations/table_engines/memory.md' - 'Buffer': 'operations/table_engines/buffer.md' - + - 'Generate': 'operations/table_engines/generate.md' - 'SQL Reference': - 'hidden': 'query_language/index.md' - 'Syntax': 'query_language/syntax.md' @@ -143,7 +143,7 @@ nav: - 'hdfs': 'query_language/table_functions/hdfs.md' - 'input': 'query_language/table_functions/input.md' - 'generate': 'query_language/table_functions/generate.md' -- 'Dictionaries': + - 'Dictionaries': - 'Introduction': 'query_language/dicts/index.md' - 'External Dictionaries': - 'General Description': 'query_language/dicts/external_dicts.md' diff --git a/docs/toc_ru.yml b/docs/toc_ru.yml index 99af6d02545..60d73e0d075 100644 --- a/docs/toc_ru.yml +++ b/docs/toc_ru.yml @@ -80,6 +80,7 @@ nav: - 'MaterializedView': 'operations/table_engines/materializedview.md' - 'Memory': 'operations/table_engines/memory.md' - 'Buffer': 'operations/table_engines/buffer.md' + - 'Generate': 'operations/table_engines/generate.md' - 'Справка по SQL': - 'hidden': 'query_language/index.md' diff --git a/docs/toc_zh.yml b/docs/toc_zh.yml index bd5c4308bce..2fe4e9763b6 100644 --- a/docs/toc_zh.yml +++ b/docs/toc_zh.yml @@ -109,6 +109,7 @@ nav: - 'MaterializedView': 'operations/table_engines/materializedview.md' - 'Memory': 'operations/table_engines/memory.md' - 'Buffer': 'operations/table_engines/buffer.md' + - 'Generate': 'operations/table_engines/generate.md' - 'SQL语法': - 'hidden': 'query_language/index.md' From 0a41e5237964ff08e7424752ad6632f67f34520e Mon Sep 17 00:00:00 2001 From: BayoNet Date: Fri, 6 Mar 2020 15:45:07 +0300 Subject: [PATCH 089/712] Update docs/en/introduction/adopters.md Co-Authored-By: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> --- docs/en/introduction/adopters.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/introduction/adopters.md b/docs/en/introduction/adopters.md index e0d409d9cfc..d1c7a35cead 100644 --- a/docs/en/introduction/adopters.md +++ b/docs/en/introduction/adopters.md @@ -44,7 +44,7 @@ | [Pragma Innovation](http://www.pragma-innovation.fr/) | Telemetry and Big Data Analysis | Main product | — | — | [Slides in English, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup18/4_pragma_innovation.pdf) | | [QINGCLOUD](https://www.qingcloud.com/) | Cloud services | Main product | — | — | [Slides in Chinese, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/4.%20Cloud%20%2B%20TSDB%20for%20ClickHouse%20张健%20QingCloud.pdf) | | [Qrator](https://qrator.net) | DDoS protection | Main product | — | — | [Blog Post, March 2019](https://blog.qrator.net/en/clickhouse-ddos-mitigation_37/) | -| [Percent百分点](https://www.percent.cn/) | Analytics | Main Product | — | — | [Slides in Chinese, June 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/4.%20ClickHouse万亿数据双中心的设计与实践%20.pdf) | +| [Beijing PERCENT Information Technology Co., Ltd.](https://www.percent.cn/) | Analytics | Main Product | — | — | [Slides in Chinese, June 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/4.%20ClickHouse万亿数据双中心的设计与实践%20.pdf) | | [Rambler](https://rambler.ru) | Internet services | Analytics | — | — | [Talk in Russian, April 2018](https://medium.com/@ramblertop/разработка-api-clickhouse-для-рамблер-топ-100-f4c7e56f3141) | | [Tencent](https://www.tencent.com) | Messaging | Logging | — | — | [Talk in Chinese, November 2019](https://youtu.be/T-iVQRuw-QY?t=5050) | | [Traffic Stars](https://trafficstars.com/) | AD network | — | — | — | [Slides in Russian, May 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup15/lightning/ninja.pdf) | From 6134a046cb1a20d3df527a0c3391e5ab98cf2673 Mon Sep 17 00:00:00 2001 From: Hui Wang Date: Fri, 6 Mar 2020 14:31:27 +0100 Subject: [PATCH 090/712] ignore weak symbol under mac because it must be defined --- CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index c0d4cd5504d..2948225583c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -214,6 +214,10 @@ if (COMPILER_CLANG) # TODO investigate that set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-omit-frame-pointer") + if (OS_DARWIN) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,-U,_inside_main") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wl,-U,_inside_main") + endif() endif () option (ENABLE_LIBRARIES "Enable all libraries (Global default switch)" ON) From b3a6f6b3676a8a22f4a73b22162e918fb2fb2eb9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 6 Mar 2020 21:14:33 +0300 Subject: [PATCH 091/712] Revert "Simplification" This reverts commit f5518c0c439f0156812ae3f3c5bb1423c5c46d84. --- dbms/programs/server/HTTPHandler.cpp | 10 ++-- dbms/src/Interpreters/Context.cpp | 83 ++++++++++++++++++++-------- dbms/src/Interpreters/Context.h | 7 ++- 3 files changed, 70 insertions(+), 30 deletions(-) diff --git a/dbms/programs/server/HTTPHandler.cpp b/dbms/programs/server/HTTPHandler.cpp index b360b0a89f4..a4c59ff9e25 100644 --- a/dbms/programs/server/HTTPHandler.cpp +++ b/dbms/programs/server/HTTPHandler.cpp @@ -141,15 +141,15 @@ static Poco::Net::HTTPResponse::HTTPStatus exceptionCodeToHTTPStatus(int excepti } -static uint32_t parseSessionTimeout( +static std::chrono::steady_clock::duration parseSessionTimeout( const Poco::Util::AbstractConfiguration & config, const HTMLForm & params) { - uint32_t session_timeout = config.getInt("default_session_timeout", 60); + unsigned session_timeout = config.getInt("default_session_timeout", 60); if (params.has("session_timeout")) { - uint32_t max_session_timeout = config.getUInt("max_session_timeout", 3600); + unsigned max_session_timeout = config.getUInt("max_session_timeout", 3600); std::string session_timeout_str = params.get("session_timeout"); ReadBufferFromString buf(session_timeout_str); @@ -162,7 +162,7 @@ static uint32_t parseSessionTimeout( ErrorCodes::INVALID_SESSION_TIMEOUT); } - return session_timeout; + return std::chrono::seconds(session_timeout); } @@ -275,7 +275,7 @@ void HTTPHandler::processQuery( std::shared_ptr session; String session_id; - uint32_t session_timeout; + std::chrono::steady_clock::duration session_timeout; bool session_is_set = params.has("session_id"); const auto & config = server.config(); diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index aa688ff9dd5..dbc963e0a27 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -124,7 +124,7 @@ public: std::shared_ptr acquireSession( const String & session_id, Context & context, - uint32_t timeout, + std::chrono::steady_clock::duration timeout, bool throw_if_not_found) { std::unique_lock lock(mutex); @@ -162,7 +162,7 @@ public: void releaseSession(NamedSession & session) { std::unique_lock lock(mutex); - close_times.emplace(time(nullptr) + session.timeout, session.key); + scheduleCloseSession(session, lock); } private: @@ -178,11 +178,31 @@ private: } }; + /// TODO it's very complicated. Make simple std::map with time_t or boost::multi_index. using Container = std::unordered_map, SessionKeyHash>; - using CloseTimes = std::multimap; - + using CloseTimes = std::deque>; Container sessions; CloseTimes close_times; + std::chrono::steady_clock::duration close_interval = std::chrono::seconds(1); + std::chrono::steady_clock::time_point close_cycle_time = std::chrono::steady_clock::now(); + UInt64 close_cycle = 0; + + void scheduleCloseSession(NamedSession & session, std::unique_lock &) + { + /// Push it on a queue of sessions to close, on a position corresponding to the timeout. + /// (timeout is measured from current moment of time) + + const UInt64 close_index = session.timeout / close_interval + 1; + const auto new_close_cycle = close_cycle + close_index; + + if (session.close_cycle != new_close_cycle) + { + session.close_cycle = new_close_cycle; + if (close_times.size() < close_index + 1) + close_times.resize(close_index + 1); + close_times[close_index].emplace_back(session.key); + } + } void cleanThread() { @@ -192,32 +212,51 @@ private: while (true) { - closeSessions(lock); - if (cond.wait_for(lock, std::chrono::seconds(1), [this]() -> bool { return quit; })) + auto interval = closeSessions(lock); + + if (cond.wait_for(lock, interval, [this]() -> bool { return quit; })) break; } } /// Close sessions, that has been expired. Returns how long to wait for next session to be expired, if no new sessions will be added. - void closeSessions(std::unique_lock &) + std::chrono::steady_clock::duration closeSessions(std::unique_lock & lock) { - time_t now = time(nullptr); - for (auto it = close_times.begin(); it != close_times.end();) + const auto now = std::chrono::steady_clock::now(); + + /// The time to close the next session did not come + if (now < close_cycle_time) + return close_cycle_time - now; /// Will sleep until it comes. + + const auto current_cycle = close_cycle; + + ++close_cycle; + close_cycle_time = now + close_interval; + + if (close_times.empty()) + return close_interval; + + auto & sessions_to_close = close_times.front(); + + for (const auto & key : sessions_to_close) { - if (it->first >= now) - break; + const auto session = sessions.find(key); - const auto session_it = sessions.find(it->second); - it = close_times.erase(it); - - if (session_it == sessions.end()) - continue; - - if (session_it->second.unique()) - sessions.erase(session_it); - else - close_times.emplace(now + session_it->second->timeout, session_it->second->key); /// Does not invalidate iterators. + if (session != sessions.end() && session->second->close_cycle <= current_cycle) + { + if (!session->second.unique()) + { + /// Skip but move it to close on the next cycle. + session->second->timeout = std::chrono::steady_clock::duration{0}; + scheduleCloseSession(*session->second, lock); + } + else + sessions.erase(session); + } } + + close_times.pop_front(); + return close_interval; } std::mutex mutex; @@ -481,7 +520,7 @@ void Context::enableNamedSessions() shared->named_sessions.emplace(); } -std::shared_ptr Context::acquireNamedSession(const String & session_id, uint32_t timeout, bool session_check) +std::shared_ptr Context::acquireNamedSession(const String & session_id, std::chrono::steady_clock::duration timeout, bool session_check) { if (!shared->named_sessions) throw Exception("Support for named sessions is not enabled", ErrorCodes::NOT_IMPLEMENTED); diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index 13053e4c8f2..5b5b8bdabd5 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -426,7 +426,7 @@ public: /// The method must be called at the server startup. void enableNamedSessions(); - std::shared_ptr acquireNamedSession(const String & session_id, uint32_t timeout, bool session_check); + std::shared_ptr acquireNamedSession(const String & session_id, std::chrono::steady_clock::duration timeout, bool session_check); /// For methods below you may need to acquire a lock by yourself. std::unique_lock getLock() const; @@ -672,11 +672,12 @@ using NamedSessionKey = std::pair; struct NamedSession { NamedSessionKey key; + UInt64 close_cycle = 0; Context context; - uint32_t timeout; + std::chrono::steady_clock::duration timeout; NamedSessions & parent; - NamedSession(NamedSessionKey key_, Context & context_, uint32_t timeout_, NamedSessions & parent_) + NamedSession(NamedSessionKey key_, Context & context_, std::chrono::steady_clock::duration timeout_, NamedSessions & parent_) : key(key_), context(context_), timeout(timeout_), parent(parent_) { } From 42cc6f4e3c48da26483614c89cf1af87b8265057 Mon Sep 17 00:00:00 2001 From: Denis Zhuravlev Date: Fri, 6 Mar 2020 16:29:48 -0400 Subject: [PATCH 092/712] Update settings.md max_insert_threads corrected ru description --- docs/ru/operations/settings/settings.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index f71b24ff269..5f452a59888 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -552,7 +552,17 @@ log_query_threads=1 ## max_insert_threads {#settings-max_insert_threads} -Максимальное количество потоков для выполнения запроса `INSERT SELECT`. По умолчанию (0 - auto) определяется автоматически исходя из количества потоков данных которое выдает SELECT. Если параметр `max_insert_threads` задан (не 0), то будет использовано минимальное значение из заданного и автоматического. +Максимальное количество потоков для выполнения запроса `INSERT SELECT`. + +Возможные значения: + +- 0 (или 1) — `INSERT SELECT` не выполняется параллельно. +- Положительное целое число, больше 1. + +Значение по умолчанию: 0. + +Параллельный `INSERT SELECT` действует только в том случае, если часть SELECT выполняется параллельно, см. настройку [max_threads](#settings-max_threads). +Чем больше значение `max_insert_threads`, тем больше потребление оперативной памяти. ## max_compress_block_size From 44b6390c62ec3e7a722ff7dec48a2163f82be78a Mon Sep 17 00:00:00 2001 From: Denis Zhuravlev Date: Fri, 6 Mar 2020 16:35:31 -0400 Subject: [PATCH 093/712] Update settings.md max_insert_threads EN description --- docs/en/operations/settings/settings.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 4928fba1ca8..996af978643 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -569,6 +569,20 @@ For queries that are completed quickly because of a LIMIT, you can set a lower ' The smaller the `max_threads` value, the less memory is consumed. +## max_insert_threads {#settings-max_insert_threads} + +The maximum number of threads to execute the `INSERT SELECT` query. + +Possible values: + +- 0 (or 1) — `INSERT SELECT` no parallel execution. +- Positive integer. Bigger than 1. + +Default value: 0. + +Parallel `INSERT SELECT` has effect only if the SELECT part is run in parallel, see [max_threads](#settings-max_threads) setting. +Higher values will lead to higher memory usage. + ## max_compress_block_size The maximum size of blocks of uncompressed data before compressing for writing to a table. By default, 1,048,576 (1 MiB). If the size is reduced, the compression rate is significantly reduced, the compression and decompression speed increases slightly due to cache locality, and memory consumption is reduced. There usually isn't any reason to change this setting. From 367b358822cc2affaf7cead449423324c2d9b934 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 6 Mar 2020 23:38:19 +0300 Subject: [PATCH 094/712] remove Context::getTable() --- dbms/src/Functions/FunctionJoinGet.cpp | 2 +- dbms/src/Functions/hasColumnInTable.cpp | 4 ++-- dbms/src/Interpreters/Context.cpp | 12 ------------ dbms/src/Interpreters/Context.h | 1 - dbms/src/Interpreters/ExpressionAnalyzer.cpp | 4 ++-- dbms/src/Interpreters/InterpreterDescribeQuery.cpp | 11 +++++------ dbms/src/Interpreters/InterpreterExplainQuery.cpp | 3 ++- dbms/src/Interpreters/JoinedTables.cpp | 3 ++- dbms/src/Interpreters/getTableExpressions.cpp | 4 ++-- dbms/src/Interpreters/interpretSubquery.cpp | 7 +++---- dbms/src/Storages/StorageID.cpp | 14 ++++++++++++++ dbms/src/Storages/StorageID.h | 14 +++----------- dbms/src/Storages/getStructureOfRemoteTable.cpp | 9 ++++----- dbms/src/Storages/getStructureOfRemoteTable.h | 4 ++-- dbms/src/TableFunctions/TableFunctionRemote.cpp | 2 +- 15 files changed, 43 insertions(+), 51 deletions(-) diff --git a/dbms/src/Functions/FunctionJoinGet.cpp b/dbms/src/Functions/FunctionJoinGet.cpp index 83d0cca1694..0860deccb14 100644 --- a/dbms/src/Functions/FunctionJoinGet.cpp +++ b/dbms/src/Functions/FunctionJoinGet.cpp @@ -42,7 +42,7 @@ static auto getJoin(const ColumnsWithTypeAndName & arguments, const Context & co ++dot; } String table_name = join_name.substr(dot); - auto table = context.getTable(database_name, table_name); + auto table = DatabaseCatalog::instance().getTable({database_name, table_name}); auto storage_join = std::dynamic_pointer_cast(table); if (!storage_join) throw Exception{"Table " + join_name + " should have engine StorageJoin", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; diff --git a/dbms/src/Functions/hasColumnInTable.cpp b/dbms/src/Functions/hasColumnInTable.cpp index 228052e8d46..8445d3894af 100644 --- a/dbms/src/Functions/hasColumnInTable.cpp +++ b/dbms/src/Functions/hasColumnInTable.cpp @@ -113,7 +113,7 @@ void FunctionHasColumnInTable::executeImpl(Block & block, const ColumnNumbers & bool has_column; if (host_name.empty()) { - const StoragePtr & table = global_context.getTable(database_name, table_name); + const StoragePtr & table = DatabaseCatalog::instance().getTable({database_name, table_name}); has_column = table->hasColumn(column_name); } else @@ -128,7 +128,7 @@ void FunctionHasColumnInTable::executeImpl(Block & block, const ColumnNumbers & global_context.getTCPPort(), false); - auto remote_columns = getStructureOfRemoteTable(*cluster, database_name, table_name, global_context); + auto remote_columns = getStructureOfRemoteTable(*cluster, {database_name, table_name}, global_context); has_column = remote_columns.hasPhysical(column_name); } diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index d3b2ff25322..58c1fe147a5 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -834,7 +834,6 @@ const Block & Context::getScalar(const String & name) const Tables Context::getExternalTables() const { - //FIXME getTable() may acquire some locks. Better not to call it while holding context lock auto lock = getLock(); Tables res; @@ -855,17 +854,6 @@ Tables Context::getExternalTables() const } -StoragePtr Context::getTable(const String & database_name, const String & table_name) const -{ - auto resolved_id = resolveStorageID(StorageID(database_name, table_name)); - std::optional exc; - auto res = DatabaseCatalog::instance().getTableImpl(resolved_id, *this, &exc).second; - if (!res) - throw *exc; - return res; -} - - void Context::addExternalTable(const String & table_name, const StoragePtr & storage, const ASTPtr & ast) { auto external_db = DatabaseCatalog::instance().getDatabaseForTemporaryTables(); diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index 6b7c8305741..d409c7cc0ad 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -312,7 +312,6 @@ public: const Scalars & getScalars() const; const Block & getScalar(const String & name) const; Tables getExternalTables() const; - StoragePtr getTable(const String & database_name, const String & table_name) const; void addExternalTable(const String & table_name, const StoragePtr & storage, const ASTPtr & ast = {}); void addScalar(const String & name, const Block & block); bool hasScalar(const String & name) const; diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp index 3339f14ff6c..a6565912f6d 100644 --- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp +++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp @@ -322,8 +322,8 @@ SetPtr SelectQueryExpressionAnalyzer::isPlainStorageSetInSubquery(const ASTPtr & const auto * table = subquery_or_table_name->as(); if (!table) return nullptr; - const DatabaseAndTableWithAlias database_table(*table); - const auto storage = context.getTable(database_table.database, database_table.table); + auto table_id = StorageID::resolveFromAST(subquery_or_table_name, context); + const auto storage = DatabaseCatalog::instance().getTable(table_id); if (storage->getName() != "Set") return nullptr; const auto storage_set = std::dynamic_pointer_cast(storage); diff --git a/dbms/src/Interpreters/InterpreterDescribeQuery.cpp b/dbms/src/Interpreters/InterpreterDescribeQuery.cpp index 5159dc9a6e8..d96cd4f7961 100644 --- a/dbms/src/Interpreters/InterpreterDescribeQuery.cpp +++ b/dbms/src/Interpreters/InterpreterDescribeQuery.cpp @@ -86,14 +86,13 @@ BlockInputStreamPtr InterpreterDescribeQuery::executeImpl() { const auto & identifier = table_expression.database_and_table_name->as(); - String database_name; - String table_name; - std::tie(database_name, table_name) = IdentifierSemantic::extractDatabaseAndTable(identifier); + StorageID table_id = StorageID::createEmpty(); + std::tie(table_id.database_name, table_id.table_name) = IdentifierSemantic::extractDatabaseAndTable(identifier); - if (!database_name.empty() || !context.isExternalTableExist(table_name)) - context.checkAccess(AccessType::SHOW, database_name, table_name); + table_id = context.resolveStorageID(table_id); + context.checkAccess(AccessType::SHOW, table_id); - table = context.getTable(database_name, table_name); + table = DatabaseCatalog::instance().getTable(table_id); } auto table_lock = table->lockStructureForShare(false, context.getInitialQueryId()); diff --git a/dbms/src/Interpreters/InterpreterExplainQuery.cpp b/dbms/src/Interpreters/InterpreterExplainQuery.cpp index d6272a08f71..ef7f4384504 100644 --- a/dbms/src/Interpreters/InterpreterExplainQuery.cpp +++ b/dbms/src/Interpreters/InterpreterExplainQuery.cpp @@ -75,7 +75,8 @@ namespace if (const auto * identifier = expression.database_and_table_name->as()) { const auto & [database, table] = IdentifierSemantic::extractDatabaseAndTable(*identifier); - const auto & storage = data.context.getTable(database.empty() ? data.context.getCurrentDatabase() : database, table); + auto table_id = data.context.resolveStorageID({database, table}); + const auto & storage = DatabaseCatalog::instance().getTable(table_id); if (auto * storage_view = dynamic_cast(storage.get())) storage_view->getRuntimeViewQuery(&select_query, data.context, true); diff --git a/dbms/src/Interpreters/JoinedTables.cpp b/dbms/src/Interpreters/JoinedTables.cpp index d97e78a1669..810ccbb45a7 100644 --- a/dbms/src/Interpreters/JoinedTables.cpp +++ b/dbms/src/Interpreters/JoinedTables.cpp @@ -82,7 +82,8 @@ StoragePtr JoinedTables::getLeftTableStorage(Context & context) if (!storage) { /// Read from table. Even without table expression (implicit SELECT ... FROM system.one). - storage = context.getTable(database_name, table_name); + auto table_id = context.resolveStorageID({database_name, table_name}); + storage = DatabaseCatalog::instance().getTable(table_id); } return storage; diff --git a/dbms/src/Interpreters/getTableExpressions.cpp b/dbms/src/Interpreters/getTableExpressions.cpp index f32510d7299..63fd6d44c29 100644 --- a/dbms/src/Interpreters/getTableExpressions.cpp +++ b/dbms/src/Interpreters/getTableExpressions.cpp @@ -94,8 +94,8 @@ static NamesAndTypesList getColumnsFromTableExpression(const ASTTableExpression } else if (table_expression.database_and_table_name) { - DatabaseAndTableWithAlias database_table(table_expression.database_and_table_name); - const auto & table = context.getTable(database_table.database, database_table.table); + auto table_id = StorageID::resolveFromAST(table_expression.database_and_table_name, context); + const auto & table = DatabaseCatalog::instance().getTable(table_id); auto & columns = table->getColumns(); names_and_type_list = columns.getOrdinary(); materialized = columns.getMaterialized(); diff --git a/dbms/src/Interpreters/interpretSubquery.cpp b/dbms/src/Interpreters/interpretSubquery.cpp index 95bedfbfdfd..82d0f73ade9 100644 --- a/dbms/src/Interpreters/interpretSubquery.cpp +++ b/dbms/src/Interpreters/interpretSubquery.cpp @@ -84,7 +84,6 @@ std::shared_ptr interpretSubquery( const auto select_expression_list = select_query->select(); NamesAndTypesList columns; - /// get columns list for target table if (function) { @@ -95,10 +94,10 @@ std::shared_ptr interpretSubquery( } else { - DatabaseAndTableWithAlias database_table(*table); - const auto & storage = context.getTable(database_table.database, database_table.table); + auto table_id = StorageID::resolveFromAST(table_expression, context); + const auto & storage = DatabaseCatalog::instance().getTable(table_id); columns = storage->getColumns().getOrdinary(); - select_query->replaceDatabaseAndTable(database_table.database, database_table.table); + select_query->replaceDatabaseAndTable(table_id.database_name, table_id.table_name); } select_expression_list->children.reserve(columns.size()); diff --git a/dbms/src/Storages/StorageID.cpp b/dbms/src/Storages/StorageID.cpp index b4c09ddd440..abdee23e3aa 100644 --- a/dbms/src/Storages/StorageID.cpp +++ b/dbms/src/Storages/StorageID.cpp @@ -4,6 +4,7 @@ #include #include #include +#include namespace DB { @@ -16,6 +17,14 @@ StorageID::StorageID(const ASTQueryWithTableAndOutput & query, const Context & l assertNotEmpty(); } +String StorageID::getDatabaseName() const +{ + assertNotEmpty(); + if (database_name.empty()) + throw Exception("Database name is empty", ErrorCodes::UNKNOWN_DATABASE); + return database_name; +} + String StorageID::getNameForLogs() const { assertNotEmpty(); @@ -54,4 +63,9 @@ StorageID StorageID::resolveFromAST(const ASTPtr & table_identifier_node, const return context.tryResolveStorageID({database_table.database, database_table.table}); } +String StorageID::getFullTableName() const +{ + return backQuoteIfNeed(getDatabaseName()) + "." + backQuoteIfNeed(table_name); +} + } diff --git a/dbms/src/Storages/StorageID.h b/dbms/src/Storages/StorageID.h index dd0672dfb0a..ffe931822cc 100644 --- a/dbms/src/Storages/StorageID.h +++ b/dbms/src/Storages/StorageID.h @@ -3,7 +3,6 @@ #include #include #include - #include namespace DB @@ -12,6 +11,7 @@ namespace DB namespace ErrorCodes { extern const int LOGICAL_ERROR; + extern const int UNKNOWN_DATABASE; } static constexpr char const * TABLE_WITH_UUID_NAME_PLACEHOLDER = "_"; @@ -36,11 +36,7 @@ struct StorageID static StorageID resolveFromAST(const ASTPtr & table_identifier_node, const Context & context); - String getDatabaseName() const - { - assertNotEmpty(); - return database_name; - } + String getDatabaseName() const; String getTableName() const { @@ -48,11 +44,7 @@ struct StorageID return table_name; } - String getFullTableName() const - { - assertNotEmpty(); - return (database_name.empty() ? "" : database_name + ".") + table_name; - } + String getFullTableName() const; String getNameForLogs() const; diff --git a/dbms/src/Storages/getStructureOfRemoteTable.cpp b/dbms/src/Storages/getStructureOfRemoteTable.cpp index 21055c43c57..af19403cbbc 100644 --- a/dbms/src/Storages/getStructureOfRemoteTable.cpp +++ b/dbms/src/Storages/getStructureOfRemoteTable.cpp @@ -25,8 +25,7 @@ namespace ErrorCodes ColumnsDescription getStructureOfRemoteTable( const Cluster & cluster, - const std::string & database, - const std::string & table, + const StorageID & table_id, const Context & context, const ASTPtr & table_func_ptr) { @@ -50,10 +49,10 @@ ColumnsDescription getStructureOfRemoteTable( else { if (shard_info.isLocal()) - return context.getTable(database, table)->getColumns(); + return DatabaseCatalog::instance().getTable(table_id)->getColumns(); /// Request for a table description - query = "DESC TABLE " + backQuoteIfNeed(database) + "." + backQuoteIfNeed(table); + query = "DESC TABLE " + table_id.getFullTableName(); } ColumnsDescription res; @@ -73,7 +72,7 @@ ColumnsDescription getStructureOfRemoteTable( auto input = std::make_shared(shard_info.pool, query, sample_block, new_context); input->setPoolMode(PoolMode::GET_ONE); if (!table_func_ptr) - input->setMainTable(StorageID{database, table}); + input->setMainTable(table_id); input->readPrefix(); const DataTypeFactory & data_type_factory = DataTypeFactory::instance(); diff --git a/dbms/src/Storages/getStructureOfRemoteTable.h b/dbms/src/Storages/getStructureOfRemoteTable.h index 9f1769a7096..468c9c685d5 100644 --- a/dbms/src/Storages/getStructureOfRemoteTable.h +++ b/dbms/src/Storages/getStructureOfRemoteTable.h @@ -10,13 +10,13 @@ namespace DB class Cluster; class Context; +struct StorageID; /// Find the names and types of the table columns on any server in the cluster. /// Used to implement the `remote` table function and others. ColumnsDescription getStructureOfRemoteTable( const Cluster & cluster, - const std::string & database, - const std::string & table, + const StorageID & table_id, const Context & context, const ASTPtr & table_func_ptr = nullptr); diff --git a/dbms/src/TableFunctions/TableFunctionRemote.cpp b/dbms/src/TableFunctions/TableFunctionRemote.cpp index ffcec737fb7..7d86c6b4579 100644 --- a/dbms/src/TableFunctions/TableFunctionRemote.cpp +++ b/dbms/src/TableFunctions/TableFunctionRemote.cpp @@ -188,7 +188,7 @@ StoragePtr TableFunctionRemote::executeImpl(const ASTPtr & ast_function, const C secure); } - auto structure_remote_table = getStructureOfRemoteTable(*cluster, remote_database, remote_table, context, remote_table_function_ptr); + auto structure_remote_table = getStructureOfRemoteTable(*cluster, {remote_database, remote_table}, context, remote_table_function_ptr); StoragePtr res = remote_table_function_ptr ? StorageDistributed::createWithOwnCluster( From a5e4381a98dcc84a7fe1373fa5f728dcd2418a10 Mon Sep 17 00:00:00 2001 From: Denis Zhuravlev Date: Fri, 6 Mar 2020 16:42:17 -0400 Subject: [PATCH 095/712] Update settings.md --- docs/en/operations/settings/settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 996af978643..dd673ca0212 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -580,7 +580,7 @@ Possible values: Default value: 0. -Parallel `INSERT SELECT` has effect only if the SELECT part is run in parallel, see [max_threads](#settings-max_threads) setting. +Parallel `INSERT SELECT` has effect only if the `SELECT` part is executed in parallel, see [max_threads](#settings-max_threads) setting. Higher values will lead to higher memory usage. ## max_compress_block_size From 55fdce72eb3876ee493407d70e9a5d03f86b3213 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 6 Mar 2020 23:50:34 +0300 Subject: [PATCH 096/712] Update roadmap --- docs/ru/extended_roadmap.md | 118 ++++++++++++++++++++++-------------- 1 file changed, 73 insertions(+), 45 deletions(-) diff --git a/docs/ru/extended_roadmap.md b/docs/ru/extended_roadmap.md index 0c5d57c54d8..91e969c10ca 100644 --- a/docs/ru/extended_roadmap.md +++ b/docs/ru/extended_roadmap.md @@ -16,7 +16,7 @@ Задача "normalized z-Order curve" в перспективе может быть полезна для БК и Метрики, так как позволяет смешивать OrderID и PageID и избежать дублирования данных. В задаче также вводится способ индексации путём обращения функции нескольких аргументов на интервале, что имеет смысл для дальнейшего развития. -Изначально делал [Андрей Чулков](https://github.com/achulkov2), ВШЭ, теперь доделывает [Ольга Хвостикова](https://github.com/stavrolia), но сроки немного сдвинуты из-за задачи 25.9. Будем надеятся на реализацию к концу ноября. Впрочем, [Андрей Чулков](https://github.com/achulkov2) скоро сможет помочь её доделать. +Изначально делал [Андрей Чулков](https://github.com/achulkov2), ВШЭ, теперь (не) доделывает [Ольга Хвостикова](https://github.com/stavrolia), но сроки немного сдвинуты из-за задачи 25.9. Будем надеятся на лучшее. Upd. Доделывать будет другой человек. Приоритет не высокий. @@ -32,6 +32,8 @@ Q1. Делает [Александр Токмаков](https://github.com/tavplu Q1. И полностью immutable куски. Делает [Александр Сапин](https://github.com/alesapin). Готов приступить к задаче в конце ноября 2019. Нужно для Яндекс.Метрики. +Upd. Большая часть задачи реализована и добавлена в master. Есть незначительные технические долги. Остаётся реализация неблокирующего изменения метаданных таблицы. + ### 1.4. Нетранзитивные ALTER столбцов. Требует 1.3. Будет делать [Александр Сапин](https://github.com/alesapin). @@ -45,6 +47,8 @@ Q1. И полностью immutable куски. Делает [Александр Компактные куски - Q1, куски в оперативке Q1/Q2. +Компактные куски реализованы, ещё не включены по-умолчанию. Первым шагом включаем по-умолчанию для системных таблиц. + Делает [Антон Попов](https://github.com/CurtizJ), первый рабочий вариант в декабре. Пререквизит чтобы снизить сложность мелких INSERT, что в свою очередь нужно для 1.12, иначе задача 1.12 не сможет нормально работать. Особенно нужно для Яндекс.Облака. Данные в таблицах типа MergeTree в ClickHouse хранятся в виде набора независимых "кусков". Внутри куска, каждый столбец, а также индекс, хранится в отдельных файлах. Это сделано для возможности быстрых манипуляций со столбцами (пример - запрос ALTER DROP COLUMN). При вставке данных (INSERT), создаётся новый кусок. Для таблиц с большим количеством столбцов, запросы INSERT с маленьким количеством строк являются неэффективными, так как требуют создания большого количества файлов в файловой системе. Это является врождённой особенностью ClickHouse - одной из первой проблем, с которыми сталкиваются пользователи. Пользователям приходится буферизовывать данные и собирать их в более крупные пачки перед вставкой в ClickHouse. @@ -57,11 +61,12 @@ Q1. И полностью immutable куски. Делает [Александр Требует 1.6. -### 1.8. Перенос между разделами по TTL. +### 1.8. + Перенос между разделами по TTL. Делает [Владимир Чеботарёв](https://github.com/excitoon), Altinity. Декабрь 2019. Q1. Закоммичено, но есть технический долг, который исправляется сейчас. +Готово. ### 1.9. Использование TTL для прореживания данных. @@ -95,6 +100,8 @@ Q1. Закоммичено, но есть технический долг, ко ### 1.11. Виртуальная файловая система. +В процессе реализации, сейчас на VFS переведены Log, TinyLog, StripeLog, готовится MergeTree. + Q2. Нужно для Яндекс.Облака. Делает Александр, Яндекс.Облако, а также Олег Ершов, ВШЭ и Яндекс. @@ -110,6 +117,8 @@ Q2. Нужно для Яндекс.Облака. Требует 1.11. Желательно 1.6 и 1.18. Делает Александр, Яндекс.Облако (сначала часть для S3), а также Олег Ершов, ВШЭ и Яндекс. +Upd. Олег будет делать только часть про HDFS. + ### 1.13. Ускорение запросов с FINAL. Требует 2.1. Делает [Николай Кочетов](https://github.com/KochetovNicolai). Нужно для Яндекс.Метрики. @@ -174,6 +183,8 @@ Upd. Включили по-умолчанию. Удаление старого В очереди. Нужно для YQL. +Александр Токмаков исправил множество проблем с использованием Context и сейчас переносит каталог БД наружу. + ### 2.8. Декларативный парсер запросов. Средний приоритет. Нужно для YQL. @@ -277,6 +288,8 @@ Upd. Иван Блинков сделал эту задачу путём зам [#9075](https://github.com/ClickHouse/ClickHouse/pull/9075) Q1. Нужно для Метрики, в очереди. Никита Михайлов. +Upd. Задача на финальной стадии разработки. + ### 5.2. Автонастройка лимита на оперативку и размера кэшей. ### 5.3. + Встроенная ручка для Prometheus. @@ -288,20 +301,20 @@ Q1. Нужно для Метрики, в очереди. Никита Михай ### 5.5. + LTS релизы. Требует 7.5. Задачу хочет Метрика, Облако, БК, Маркет и Altinity. Первой LTS версией уже стала версия 19.14. -Метрика, БК, Маркет уже используют более свежие версии чем LTS. +Метрика, БК, Маркет, Altinity уже используют более свежие версии чем LTS. ## 6. Инструментирование. -### 6.1. Исправления сэмплирующего профайлера запросов. +### 6.1. + Исправления сэмплирующего профайлера запросов. Михаил Филимонов, Altinity. Ноябрь 2019. Сделано. -Осталось ещё проверить работоспособность профайлера в первом потоке (что важно для INSERT). Иван Лежанкин. Q1. +Осталось ещё проверить работоспособность профайлера в первом потоке (что важно для INSERT). Иван Лежанкин. Q1. Сделано. -### 6.2. Добавление memory profiler. +### 6.2. + Добавление memory profiler. [#6387](https://github.com/ClickHouse/ClickHouse/issues/6387) -Сравнительно простая задача, но только для опытных разработчиков. Нужна всем. Иван Лежанкин. Q1. +Сравнительно простая задача, но только для опытных разработчиков. Нужна всем. Иван Лежанкин. Q1. Сделано. ### 6.3. Учёт оперативки total расширить не только на запросы. @@ -313,6 +326,8 @@ Q1. Нужно для Метрики, в очереди. Никита Михай В Linux существует возможность получать в программе информацию о счётчиках производительности и событиях, относящихся к CPU и ядру ОС. Подробнее смотрите `man perf_event_open`. Предлагается добавить эти метрики в ClickHouse для инструментирования запросов. +Есть прототип. + ### 6.5. Эксперименты с LLVM X-Ray. Требует 2.2. @@ -357,9 +372,9 @@ Q1. Нужно для Метрики, в очереди. Никита Михай [Александр Сапин](https://github.com/alesapin). -### 7.6. Правильный статистический тест для comparison mode в clickhouse-performance-test. +### 7.6. + Правильный статистический тест для comparison mode в clickhouse-performance-test. -Задачу начал делать Дмитрий Рубашкин (ВШЭ). Сейчас продолжает [Александр Кузьменков](https://github.com/akuzm). +Задачу начал делать Дмитрий Рубашкин (ВШЭ). Сейчас продолжает [Александр Кузьменков](https://github.com/akuzm). Сделано, работает в CI. ### 7.7. Доделать тесты под MSan. @@ -441,15 +456,13 @@ Upd. В рамках данной задачи добавляем подстве [Александр Сапин](https://github.com/alesapin). Может делегировать эту задачу кому угодно. Upd. Сделано всё кроме инструкции на сайте. Для этого требуется создать директории testing/stable/prestable на repo.yandex.ru. Внезапно оказалось, что человек, отвечающий за это, в отпуске, и он не отвечает на вопрос, кто его заместитель. Q1. -### 7.18.1. Доделать бинарники под Mac. +### 7.18. + Доделать бинарники под Mac. Уже есть автосборка бинарников под Mac на каждый коммит и PR, но с недостатками. [Иван Лежанкин](https://github.com/abyss7). Требует 7.1, 7.2. Рекомендуется 7.14. Сейчас не хватает по крайней мере SSL и ICU. Нужно для Яндекс.Облака. Upd. Сделано SSL. Ориентируемся в Q1, но приоритет средний и может потеряться. -Not ready. There is no instruction on the main https://clickhouse.tech/ page neither in documentation. - -### 7.18. Поместить ссылку на собранные бинарники под Mac на сайт. +### 7.18.1. Поместить ссылку на собранные бинарники под Mac на сайт. Сейчас людям приходится делать несколько кликов, чтобы их скачать. [Иван Лежанкин](https://github.com/abyss7) или [Александр Сапин](https://github.com/alesapin). @@ -598,7 +611,9 @@ UPD: Все патчи Максима отправлены в master. Задач ### 8.1. Поддержка ALTER MODIFY SETTING для Kafka. -Altinity. +Также - возможность указать все настройки форматов в Kafka. + +Altinity. Никто не делает эту задачу. ### 8.2. Поддержка Mongo Atlas URI. @@ -618,7 +633,7 @@ Altinity. ### 8.6. Kerberos аутентификация для HDFS и Kafka. -Андрей Коняев, ArenaData. +Андрей Коняев, ArenaData. Он куда-то пропал. ### 8.7. + Исправление мелочи HDFS на очень старых ядрах Linux. @@ -661,7 +676,7 @@ Andrew Onyshchuk. Есть pull request. Q1. Сделано. Формат Apache Avro является компактным структурированным построчным бинарным форматом данных с внешней схемой. Этот формат часто используется совместно с Kafka и поддержка его в качестве одного из форматов ввода-вывода в ClickHouse является востребованной пользователями. -### 8.16.1. Поддержка формата JSONEachRow, засунутого в массив. +### 8.16.1. + Поддержка формата JSONEachRow, засунутого в массив. Павел Круглов, ВШЭ и Яндекс. Есть pull request. @@ -676,6 +691,7 @@ Andrew Onyshchuk. Есть pull request. Q1. Сделано. ### 8.16.4. Формат Regexp. Павел Круглов, ВШЭ и Яндекс. +Есть pull request. ### 8.17. ClickHouse как MySQL реплика. @@ -697,6 +713,8 @@ Maxim Fedotov, Wargaming + Yuri Baranov, Яндекс. Следующей по востребованности является система очередей RabbitMQ. Её поддержка в ClickHouse отсутствует. +Есть pull request в процессе разработки. + ### 8.20. Интеграция с SQS. Низкий приоритет. @@ -705,11 +723,14 @@ Maxim Fedotov, Wargaming + Yuri Baranov, Яндекс. Нужно для БК. Декабрь 2019. В декабре для БК сделан минимальный вариант этой задачи. +Максимальный вариант, вроде, никому не нужен. ### 8.22. Поддержка синтаксиса для переменных в стиле MySQL. При парсинге запроса преобразовывать синтаксис вида `@@version_full` в вызов функции `getGlobalVariable('version_full')`. Поддержать популярные MySQL переменные. Может быть поможет Юрий Баранов, если будет энтузиазм. +Upd. Юрий Баранов работает в Google, там запрещено разрабатывать ClickHouse. + ### 8.23. Подписка для импорта обновляемых и ротируемых логов в ФС. Желательно 2.15. @@ -745,6 +766,8 @@ ClickHouse предоставляет возможность обратитьс Код YT использует SIGILL вместо abort. Это, опять же, происходит при учениях. Нужно для БК и Метрики. Поиск причин - [Александр Сапин](https://github.com/alesapin). Дальшейшее исправление возможно на стороне YT. +Upd. Одну причину устранили, но ещё что-то неизвестное осталось. + ### 10.3. Возможность чтения данных из статических таблиц в YT словарях. Нужно для БК и Метрики. @@ -757,6 +780,8 @@ ClickHouse предоставляет возможность обратитьс Нужно для Метрики. +Для MySQL сделал Clément Rodriguez. + ### 10.6. Словари из Cassandra и Couchbase. ### 10.7. Поддержка Nullable в словарях. @@ -769,13 +794,13 @@ ClickHouse предоставляет возможность обратитьс ### 10.9. Уменьшение блокировок для cache словарей за счёт одновременных запросов одного и того же. -Никита Михайлов. Q1. Нужно для БК, но мотивация задачи находится под вопросом, так как есть рабочее предположение о том, что данная задача не устраняет причину проблемы. +Заменено в пользу 10.10, 10.11. -### 10.10. Возможность использования старых значений из cache словаря пока они перезапрашиваются. +### 10.10. + Возможность использования старых значений из cache словаря пока они перезапрашиваются. Никита Михайлов. Q1. Нужно для БК и Метрики. -### 10.11. Возможность исключительно асинхронных запросов в cache словарях. +### 10.11. + Возможность исключительно асинхронных запросов в cache словарях. Никита Михайлов. Q1. Нужно для БК и Метрики. Требует 10.10. @@ -817,7 +842,7 @@ ClickHouse предоставляет возможность обратитьс ### 11.3. Интеграционные тесты ODBC драйвера путём подключения ClickHouse к самому себе через ODBC. -Денис Глазачев, Altinity. +Михаил Филимонов, Altinity. Есть почти готовый pull request. ### 11.4. Исправление упячек с типами Date и Decimal в clickhouse-cpp. @@ -853,10 +878,10 @@ zhang2014, есть pull request. [Виталий Баранов](https://github.com/vitlibar). Финальная стадия разработки, рабочая версия в начале февраля 2019. Q1. Сейчас сделаны все интерфейсы в коде и запросы, но не сделаны варианты хранения прав кроме прототипа. -### 12.2. Управление пользователями и правами доступа с помощью SQL запросов. +### 12.2. + Управление пользователями и правами доступа с помощью SQL запросов. [Виталий Баранов](https://github.com/vitlibar). Финальная стадия разработки, рабочая версия в декабре 2019. -Q1. +Q1. Сделано управление правами полностью, но не реализовано их хранение, см. 12.1. ### 12.3. Подключение справочника пользователей и прав доступа из LDAP. @@ -890,6 +915,7 @@ Q1/Q2. Требует 13.2 или сможем сделать более неудобную реализацию раньше. Обсуждается вариант неудобной реализации. Пока средний приоритет, целимся на Q1/Q2. +Вариант реализации выбрал Александр Казаков. ## 14. Диалект SQL. @@ -926,8 +952,6 @@ zhang2014 ### 14.8. Модификаторы DISTINCT, ORDER BY для агрегатных функций. -Софья Борзенкова, ВШЭ. - В ClickHouse поддерживается вычисление COUNT(DISTINCT x). Предлагается добавить возможность использования модификатора DISTINCT для всех агрегатных функций. Например, AVG(DISTINCT x) - вычислить среднее значение для всех различных значений x. Под вопросом вариант, в котором фильтрация уникальных значений выполняется по одному выражению, а агрегация по другому. Результат некоторых агрегатных функций зависит от порядка данных. Предлагается реализовать модификатор ORDER BY, задающий порядок явно. Пример: groupArray(x ORDER BY y, z). @@ -992,7 +1016,7 @@ zhang2014. Артём Зуйков. Сейчас merge JOIN включается вручную опцией и всегда замедляет запросы. Хотим, чтобы он замедлял запросы только когда это неизбежно. Кстати, смысл merge JOIN появляется только совместно с 15.2 и 15.3. -Q1. +Q1. Сделали адаптивный вариант, но вроде он что-то всё-ещё замедляет. ### 15.1.1. Алгоритм two-level merge JOIN. @@ -1054,6 +1078,7 @@ ClickHouse не является geospatial СУБД. Тем не менее, в Похожая, но более сложная задача, которую ClickHouse пока не умеет решать - определение полигона среди множества полигонов, в которые попадают точки. Для примера: определение района города по географическим координатам. Для решения этой задачи нужно будет реализовать поддержку словарей с полигонами, в которых данные проиндексированы для быстрого поиска. Upd. Андрей сделал прототип интерфейса и реализацию-заглушку внутри него. +Upd. Андрей сделал прототип более оптимальной структуры данных. ### 17.2. GIS типы данных и операции. @@ -1086,6 +1111,8 @@ Upd. Андрей сделал прототип интерфейса и реал Предлагается реализовать в ClickHouse статистические тесты (Analysis of Variance, тесты нормальности распределения и т. п.) в виде агрегатных функций. Пример: `welchTTest(value, sample_idx)`. +Сделали прототип одного теста, есть pull request. + ### 18.3. Инфраструктура для тренировки моделей в ClickHouse. В очереди. Возможно, Александр Кожихов. У него сначала идёт задача 24.26. @@ -1115,9 +1142,11 @@ Upd. Андрей сделал прототип интерфейса и реал В прошлом году, Алексей добавил модельную реализацию (mock) интерфейса ZooKeeper для тестирования. Сейчас предлагается сделать реализацию поверх Etcd, а также расширить возможности тестовой реализации. +Upd. Алексей сделал какой-то вариант, но борется с тем, что ничего не работает. + ### 19.3. Подключение YT Cypress или YDB как альтернативы ZooKeeper. -Hold. Полезно для заказчиков внутри Яндекса, но есть риски. +Hold. Полезно для заказчиков внутри Яндекса, но есть риски. Эту задачу никто не будет делать. ### 19.4. internal_replication = 'auto'. @@ -1163,7 +1192,7 @@ Hold. Полезно для заказчиков внутри Яндекса, н Начинал Олег Ершов, доделывает Никита Михайлов, помогает [Александр Кузьменков](https://github.com/akuzm). Готово. -### 21.1.1. Избавление от лишнего копирование при параллельном парсинге форматов, если возможен mmap файла целиком. +### 21.1.1. Избавление от лишнего копирования при параллельном парсинге форматов, если возможен mmap файла целиком. ### 21.2. Параллельное форматирование форматов. @@ -1173,6 +1202,8 @@ Hold. Полезно для заказчиков внутри Яндекса, н Нужно всем (Zen, БК, DataLens, TestEnv...). Антон Попов, Q1/Q2. +Upd. Антон делает эту задачу. Большая часть уже реализована. + ### 21.4. Использование ORDER BY ключа для оптимизации GROUP BY и DISTINCT. Дмитрий Рубашкин, ВШЭ. Помогает Антон Попов. @@ -1183,7 +1214,7 @@ Hold. Полезно для заказчиков внутри Яндекса, н В прошлом году, аналогичное решение сделали для операции ORDER BY. -### 21.5. Распараллеливание INSERT при INSERT SELECT, если это необходимо. +### 21.5. + Распараллеливание INSERT при INSERT SELECT, если это необходимо. [Vxider](https://github.com/Vxider), ICT Есть pull request. @@ -1193,7 +1224,7 @@ Hold. Полезно для заказчиков внутри Яндекса, н ### 21.7. Кэш результатов запросов. [Achimbab](https://github.com/achimbab). -Есть pull request. +Есть pull request. Но это не совсем то. ### 21.8. Взаимная интеграция аллокатора и кэша. @@ -1248,8 +1279,6 @@ Amos Bird. ### 21.14. Оптимизация запросов с помощью constraints. -Мария Нефедова, ВШЭ. - Constraints позволяют задать выражение, истинность которого проверяется при вставке данных в таблицу. Предположение о том, что выражение истинно, может использоваться и для оптимизации запросов. Например, встретив в запросе точно такое же выражение, можно заменить его на константу 1. Если выражение содержит равенство, то встретив в запросе одну из частей равенства, её можно заменить на другую часть равенства, если это сделает проще чтение данных или вычисление выражения. Например, задан constraint: `URLDomain = domain(URL)`. Значит, выражение `domain(URL)` можно заменить на `URLDomain`. @@ -1334,9 +1363,11 @@ N.Vartolomei. ### 22.5. + Исправление редких срабатываний TSan в stress тестах в CI. -Александр Казаков. +Александр Казаков сделал эту задачу. -### 22.6. Изменение только DEFAULT в ALTER TABLE может поменять тип столбца. +### 22.6. + Изменение только DEFAULT в ALTER TABLE может поменять тип столбца. + +Александр Сапин сделал эту задачу. ### 22.7. + Row-Level Security не работает в случае наличия в запросе IN подзапросов. @@ -1371,7 +1402,7 @@ N.Vartolomei. Изначально было назначено на [Ивана Лежанкина](https://github.com/abyss7), но в результате сделал Александр Сапин. -### 22.14. Посмотреть, почему не работает StorageSet для MergeTree таблиц при некоторых условиях. +### 22.14. + Посмотреть, почему не работает StorageSet для MergeTree таблиц при некоторых условиях. Вроде бы сделал Никита Михайлов - проверить существующие issues на эту тему. @@ -1502,8 +1533,6 @@ Q1. [Николай Кочетов](https://github.com/KochetovNicolai). ### 24.2. Экспериментальные алгоритмы сжатия. -Анастасия Наумова, ВШЭ. - ClickHouse поддерживает LZ4 и ZSTD для сжатия данных. Эти алгоритмы являются парето-оптимальными по соотношению скорости и коэффициентам сжатия среди достаточно известных. Тем не менее, существуют менее известные алгоритмы сжатия, которые могут превзойти их по какому-либо критерию. Из потенциально более быстрых по сравнимом коэффициенте сжатия: Lizard, LZSSE, density. Из более сильных: bsc и csc. Необходимо изучить эти алгоритмы, добавить их поддержку в ClickHouse и исследовать их работу на тестовых датасетах. ### 24.3. Экспериментальные кодеки. @@ -1514,19 +1543,14 @@ ClickHouse поддерживает LZ4 и ZSTD для сжатия данных Внедрить их в ClickHouse в виде кодеков и изучить их работу на тестовых датасетах. -### 24.4. Шифрование в ClickHouse на уровне кусков данных. - -Yuchen Dong, ICT. +### 24.4. Шифрование в ClickHouse на уровне VFS. Данные в ClickHouse хранятся без шифрования. При наличии доступа к дискам, злоумышленник может прочитать данные. Предлагается реализовать два подхода к шифрованию: -1. Шифрование блоков данных. -Шифрование данных столбцов на диске требуется реализовать в виде кодеков. Это позволит применять шифрование к отдельным столбцам; применять его после сжатия данных (эффективно, но менее безопасно) или без сжатия. Потребуется проработать работу с ключами: получение ключей из отдельного сервиса, правильная работа с ключами в оперативке. Отдельным вопросом стоит шифрование индексов. +1. Шифрование на уровне VFS. ### 24.5. Поддержка функций шифрования для отдельных значений. -Yuchen Dong, ICT. - Смотрите также 24.5. 2. Шифрование отдельных значений. @@ -1608,6 +1632,8 @@ ClickHouse предоставляет достаточно богатый наб В компании nVidia сделали прототип offloading вычисления GROUP BY с некоторыми из агрегатных функций в ClickHouse и обещат предоставить исходники в публичный доступ для дальнейшего развития. Предлагается изучить этот прототип и расширить его применимость для более широкого сценария использования. В качестве альтернативы, предлагается изучить исходные коды системы `OmniSci` или `Alenka` или библиотеку `CUB` https://nvlabs.github.io/cub/ и применить некоторые из алгоритмов в ClickHouse. +Upd. В компании nVidia выложили прототип, теперь нужна интеграция в систему сборки. + ### 24.13. Stream запросы. Пререквизит для ClickHouse как CEP-системы. @@ -1769,7 +1795,7 @@ Amos Bird, но его решение слишком громоздкое и п ### 25.11. Митапы зарубежные: восток США (Нью Йорк, возможно Raleigh), возможно северо-запад (Сиэтл), Китай (Пекин снова, возможно митап для разработчиков или хакатон), Лондон. -[Иван Блинков](https://github.com/blinkov/) - организация. Две штуки в США запланированы. +[Иван Блинков](https://github.com/blinkov/) - организация. Две штуки в США запланированы. Upd. Два митапа в США и один в Европе проведены. ### 25.12. Статья "научная" - про устройство хранения данных и индексов или whitepaper по архитектуре. Есть вариант подать на VLDB. @@ -1800,6 +1826,8 @@ Amos Bird, но его решение слишком громоздкое и п Требуется проработать вопрос безопасности и изоляции инстансов (поднятие в контейнерах с ограничениями по сети), подключение тестовых датасетов с помощью copy-on-write файловой системы; органичения ресурсов. +Есть минимальный прототип. + ### 25.17. Взаимодействие с ВУЗами: ВШЭ, УрФУ, ICT Beijing. Алексей Миловидов и вся группа разработки @@ -1840,4 +1868,4 @@ UPD: не участвуем. ### 25.27. Обновить сайт ClickHouse. -Иван Блинков. Есть риски. +Иван Блинков. Нет рисков. Нужно для Яндекс.Облака. From 5db4222ff886f7070571fc5657fe0a4e9b4589e2 Mon Sep 17 00:00:00 2001 From: Denis Zhuravlev Date: Fri, 6 Mar 2020 19:25:22 -0400 Subject: [PATCH 097/712] Update settings.md insert_deduplicate , force_deduplicate_childrens RU description --- docs/ru/operations/settings/settings.md | 28 +++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 80486502dcb..ee7dc4e0654 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -842,6 +842,34 @@ ClickHouse генерирует исключение - [insert_quorum](#settings-insert_quorum) - [insert_quorum_timeout](#settings-insert_quorum_timeout) +## insert_deduplicate {#settings-insert_deduplicate} + +Включает и выключает дедупликацию для запросов `INSERT` (для Replicated* таблиц). + +Возможные значения: + +- 0 — выключена. +- 1 — включена. + +Значение по умолчанию: 1. + +По умолчанию блоки, вставляемые в реплицируемые таблицы оператором `INSERT`, дедуплицируются (см. [Репликация данных](../table_engines/replication.md). + +## force_deduplicate_childrens {#settings-force_deduplicate_childrens} + +Включает и выключает проверку дедупликации для материализованных представлений, которые получают данные из Replicated* таблиц. + +Возможные значения: + +- 0 — выключена. +- 1 — включена. + +Значение по умолчанию: 0. + +По умолчанию проверка дедупликации у материализованных представлений не производится, а наследуюется от Replicated* (основной) таблицы, за которой "следит" материализованное представление. +Т.е. если `INSERT` в основную таблицу д.б. пропущен (сдедуплирован), то автоматически не будет вставки и в материализованные представления. Это имплементировано для того, чтобы работали материализованные представления, которые сильно группируют данные основных `INSERT`, до такой степени что блоки вставляемые в материализованные представления получаются одинаковыми для разных `INSERT` в основную таблицу. +Одновременно это "ломает" идемпотентность вставки в материализованные представления. Т.е. если `INSERT` был успешен в основную таблицу и неуспешен в таблицу материализованного представления (напр. из-за сетевого сбоя при коммуникации с Zookeeper), клиент получит ошибку и попытается повторить `INSERT`. Но вставки в материализованные представления произведено не будет, потому что дедупликация сработает на основной таблице. Настройка `force_deduplicate_childrens` позволяет это изменить. Т.е. при повторном `INSERT` будет произведена дедупликация на таблице материализованного представления, и повторный инсерт вставит данные в таблицу материализованного представления, которые не удалось вставить из-за сбоя первого `INSERT`. + ## count_distinct_implementation {#settings-count_distinct_implementation} Задаёт, какая из функций `uniq*` используется при выполнении конструкции [COUNT(DISTINCT ...)](../../query_language/agg_functions/reference.md#agg_function-count). From 506575a8f79f1d8bcfcf5fe395327205a0e5d934 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sat, 7 Mar 2020 04:04:41 +0300 Subject: [PATCH 098/712] Update simple_join_query.xml --- dbms/tests/performance/simple_join_query.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/tests/performance/simple_join_query.xml b/dbms/tests/performance/simple_join_query.xml index 919cce33be6..8ef3d97460d 100644 --- a/dbms/tests/performance/simple_join_query.xml +++ b/dbms/tests/performance/simple_join_query.xml @@ -16,7 +16,7 @@ INSERT INTO join_table SELECT number AS A, toString(arrayMap(x->x, range(100))) S0, S0 AS S1, S0 AS S2, S0 AS S3 from numbers(500000) SELECT COUNT() FROM join_table LEFT JOIN join_table USING A - SELECT COUNT() FROM join_table LEFT JOIN (SELECT A FROM join_table) right USING A + SELECT COUNT() FROM join_table LEFT JOIN (SELECT A FROM join_table) AS right USING A SELECT COUNT() FROM join_table AS left LEFT JOIN join_table AS right ON left.A = right.A SELECT COUNT() FROM join_table AS left LEFT JOIN (SELECT A FROM join_table) AS right ON left.A = right.A From 5f55526b37e219effcbb7aef60d12f0e3f975ad3 Mon Sep 17 00:00:00 2001 From: Denis Zhuravlev Date: Fri, 6 Mar 2020 21:14:36 -0400 Subject: [PATCH 099/712] Update query_complexity.md explanation that `max_result_rows result_overflow_mode=break ` depends of max_block_size --- .../operations/settings/query_complexity.md | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/en/operations/settings/query_complexity.md b/docs/en/operations/settings/query_complexity.md index e5f993540a7..0b09599b8ae 100644 --- a/docs/en/operations/settings/query_complexity.md +++ b/docs/en/operations/settings/query_complexity.md @@ -107,7 +107,25 @@ Limit on the number of bytes in the result. The same as the previous setting. ## result_overflow_mode What to do if the volume of the result exceeds one of the limits: 'throw' or 'break'. By default, throw. -Using 'break' is similar to using LIMIT. + +Using 'break' is similar to using LIMIT. `Break` interrupts execution only at the block level. The query will return more rows than the limit [max_result_rows](#max_result_rows), multiple of [max_block_size](settings.md#max_block_size) and depends of [max_threads](settings.md#settings-max_threads). + +Example: + +```sql +SET max_threads = 3, max_block_size = 3333; +SET max_result_rows = 3334, result_overflow_mode = 'break'; + +SELECT * +FROM numbers_mt(100000) +FORMAT Null; +``` + +Result: + +```text +6666 rows in set. ... +``` ## max_execution_time From 6062485ed2bbed7c63311e5db1be07f085ab67e2 Mon Sep 17 00:00:00 2001 From: Denis Zhuravlev Date: Fri, 6 Mar 2020 21:25:23 -0400 Subject: [PATCH 100/712] Update query_complexity.md --- docs/en/operations/settings/query_complexity.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/settings/query_complexity.md b/docs/en/operations/settings/query_complexity.md index 01e59057c16..5b9db828d03 100644 --- a/docs/en/operations/settings/query_complexity.md +++ b/docs/en/operations/settings/query_complexity.md @@ -108,7 +108,7 @@ Limit on the number of bytes in the result. The same as the previous setting. What to do if the volume of the result exceeds one of the limits: 'throw' or 'break'. By default, throw. -Using 'break' is similar to using LIMIT. `Break` interrupts execution only at the block level. The query will return more rows than the limit [max_result_rows](#max_result_rows), multiple of [max_block_size](settings.md#max_block_size) and depends of [max_threads](settings.md#settings-max_threads). +Using 'break' is similar to using LIMIT. `Break` interrupts execution only at the block level. This means that amount of returned rows is greater than [max_result_rows](#setting-max_result_rows), multiple of [max_block_size](settings.md#setting-max_block_size) and depends on [max_threads](settings.md#settings-max_threads). Example: From c07cb384233bc0afc7b6e882bc5d23f0ce95ab3b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 7 Mar 2020 06:19:28 +0300 Subject: [PATCH 101/712] Fixed error --- dbms/tests/performance/if_to_multiif.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbms/tests/performance/if_to_multiif.xml b/dbms/tests/performance/if_to_multiif.xml index 03f528d6349..e1c45bbb69e 100644 --- a/dbms/tests/performance/if_to_multiif.xml +++ b/dbms/tests/performance/if_to_multiif.xml @@ -11,14 +11,14 @@ nonexistent_table_if_multiif - + - From e33592688296b4830cbb3251568d2ffa79117ba0 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 7 Mar 2020 06:28:03 +0300 Subject: [PATCH 102/712] Check XML validity --- utils/check-style/check-style | 3 +++ 1 file changed, 3 insertions(+) diff --git a/utils/check-style/check-style b/utils/check-style/check-style index 4734788b90e..471488287ab 100755 --- a/utils/check-style/check-style +++ b/utils/check-style/check-style @@ -48,3 +48,6 @@ find $ROOT_PATH/{dbms,base} -name '*.h' -or -name '*.cpp' | xargs grep -l -P 'Er # Three or more consecutive empty lines find $ROOT_PATH/{dbms,base} -name '*.h' -or -name '*.cpp' | while read file; do awk '/^$/ { ++i; if (i > 2) { print "More than two consecutive empty lines in file '$file'" } } /./ { i = 0 }' $file; done + +# Broken XML files (requires libxml2-utils) +find $ROOT_PATH/{dbms,base} -name '*.xml' | xargs xmllint --noout --nonet From 643367642c20b63c45ea7198e4457727d831e956 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 7 Mar 2020 06:52:50 +0300 Subject: [PATCH 103/712] Fixed race condition on queue_task_handle --- .../MergeTree/BackgroundProcessingPool.cpp | 14 +++++++++++--- .../Storages/MergeTree/BackgroundProcessingPool.h | 7 ++++++- dbms/src/Storages/StorageReplicatedMergeTree.cpp | 11 +++++++++-- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/dbms/src/Storages/MergeTree/BackgroundProcessingPool.cpp b/dbms/src/Storages/MergeTree/BackgroundProcessingPool.cpp index 64a179a8159..d922d9bd302 100644 --- a/dbms/src/Storages/MergeTree/BackgroundProcessingPool.cpp +++ b/dbms/src/Storages/MergeTree/BackgroundProcessingPool.cpp @@ -60,19 +60,27 @@ BackgroundProcessingPool::BackgroundProcessingPool(int size_, } -BackgroundProcessingPool::TaskHandle BackgroundProcessingPool::addTask(const Task & task) +BackgroundProcessingPool::TaskHandle BackgroundProcessingPool::createTask(const Task & task) { - TaskHandle res = std::make_shared(*this, task); + return std::make_shared(*this, task); +} +void BackgroundProcessingPool::startTask(const TaskHandle & task) +{ Poco::Timestamp current_time; { std::unique_lock lock(tasks_mutex); - res->iterator = tasks.emplace(current_time, res); + task->iterator = tasks.emplace(current_time, task); } wake_event.notify_all(); +} +BackgroundProcessingPool::TaskHandle BackgroundProcessingPool::addTask(const Task & task) +{ + TaskHandle res = createTask(task); + startTask(res); return res; } diff --git a/dbms/src/Storages/MergeTree/BackgroundProcessingPool.h b/dbms/src/Storages/MergeTree/BackgroundProcessingPool.h index 200cd434b2f..526cab0800e 100644 --- a/dbms/src/Storages/MergeTree/BackgroundProcessingPool.h +++ b/dbms/src/Storages/MergeTree/BackgroundProcessingPool.h @@ -82,9 +82,14 @@ public: return size; } - /// The task is started immediately. + /// Create task and start it. TaskHandle addTask(const Task & task); + /// Create task but not start it. + TaskHandle createTask(const Task & task); + /// Start the task that was created but not started. Precondition: task was not started. + void startTask(const TaskHandle & task); + void removeTask(const TaskHandle & task); ~BackgroundProcessingPool(); diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 0ed2527a981..052e218f30b 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -2902,9 +2902,16 @@ void StorageReplicatedMergeTree::startup() /// Wait while restarting_thread initializes LeaderElection (and so on) or makes first attmept to do it startup_event.wait(); - queue_task_handle = global_context.getBackgroundPool().addTask([this] { return queueTask(); }); + /// If we don't separate create/start steps, race condition will happen + /// between the assignment of queue_task_handle and queueTask that use the queue_task_handle. + queue_task_handle = global_context.getBackgroundPool().createTask([this] { return queueTask(); }); + queue_task_handle->startTask(); + if (areBackgroundMovesNeeded()) - move_parts_task_handle = global_context.getBackgroundMovePool().addTask([this] { return movePartsTask(); }); + { + move_parts_task_handle = global_context.getBackgroundMovePool().createTask([this] { return movePartsTask(); }); + move_parts_task_handle->startTask(); + } } From bfb502cf584227b8c8aaf31d6c10e296045ed110 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 7 Mar 2020 07:00:24 +0300 Subject: [PATCH 104/712] Fix race condition in executeMetadataAlter --- dbms/src/Storages/StorageReplicatedMergeTree.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 0ed2527a981..db2ffb415df 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -3174,7 +3174,6 @@ bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMer auto columns_from_entry = ColumnsDescription::parse(entry.columns_str); auto metadata_from_entry = ReplicatedMergeTreeTableMetadata::parse(entry.metadata_str); - auto metadata_diff = ReplicatedMergeTreeTableMetadata(*this).checkAndFindDiff(metadata_from_entry); MergeTreeData::DataParts parts; @@ -3183,7 +3182,6 @@ bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMer requests.emplace_back(zkutil::makeSetRequest(replica_path + "/columns", entry.columns_str, -1)); requests.emplace_back(zkutil::makeSetRequest(replica_path + "/metadata", entry.metadata_str, -1)); - zookeeper->multi(requests); { @@ -3192,6 +3190,7 @@ bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMer LOG_INFO(log, "Metadata changed in ZooKeeper. Applying changes locally."); + auto metadata_diff = ReplicatedMergeTreeTableMetadata(*this).checkAndFindDiff(metadata_from_entry); setTableStructure(std::move(columns_from_entry), metadata_diff); metadata_version = entry.alter_version; From ea7d0093c14d39321e6b399e39e02bc24c78225d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 7 Mar 2020 07:41:17 +0300 Subject: [PATCH 105/712] Fixed race condition in test --- .../0_stateless/00738_lock_for_inner_table.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/dbms/tests/queries/0_stateless/00738_lock_for_inner_table.sh b/dbms/tests/queries/0_stateless/00738_lock_for_inner_table.sh index 9e20c9469a3..467414a3ac5 100755 --- a/dbms/tests/queries/0_stateless/00738_lock_for_inner_table.sh +++ b/dbms/tests/queries/0_stateless/00738_lock_for_inner_table.sh @@ -9,17 +9,24 @@ DROP TABLE IF EXISTS mv; CREATE TABLE tab_00738(a Int) ENGINE = Log; CREATE MATERIALIZED VIEW mv ENGINE = Log AS SELECT a FROM tab_00738;" | ${CLICKHOUSE_CLIENT} -n -${CLICKHOUSE_CLIENT} --query "INSERT INTO tab_00738 SELECT number FROM numbers(10000000)" & +${CLICKHOUSE_CLIENT} --query_id test_00738 --query "INSERT INTO tab_00738 SELECT number FROM numbers(10000000)" & function drop() { - sleep 0.1 ${CLICKHOUSE_CLIENT} --query "DROP TABLE \`.inner.mv\`" -n } +function wait_for_query_to_start() +{ + while [[ $(${CLICKHOUSE_CLIENT} --query "SELECT count() FROM system.processes WHERE query_id = 'test_00738'") == 0 ]]; do sleep 0.001; done +} + +export -f wait_for_query_to_start +timeout 5 bash -c wait_for_query_to_start + drop & wait echo "DROP TABLE IF EXISTS tab_00738; -DROP TABLE IF EXISTS mv;" | ${CLICKHOUSE_CLIENT} -n +DROP TABLE IF EXISTS mv;" | ${CLICKHOUSE_CLIENT} -n From d4a724dcc7eeaf6c68ef4fb46ead9e259ba384b9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 7 Mar 2020 15:44:24 +0300 Subject: [PATCH 106/712] Fixed build --- dbms/src/Storages/StorageReplicatedMergeTree.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index e9e21daeb8a..48ba57d0c72 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -2904,13 +2904,17 @@ void StorageReplicatedMergeTree::startup() /// If we don't separate create/start steps, race condition will happen /// between the assignment of queue_task_handle and queueTask that use the queue_task_handle. - queue_task_handle = global_context.getBackgroundPool().createTask([this] { return queueTask(); }); - queue_task_handle->startTask(); + { + auto & pool = global_context.getBackgroundPool(); + queue_task_handle = pool.createTask([this] { return queueTask(); }); + pool.startTask(queue_task_handle); + } if (areBackgroundMovesNeeded()) { - move_parts_task_handle = global_context.getBackgroundMovePool().createTask([this] { return movePartsTask(); }); - move_parts_task_handle->startTask(); + auto & pool = global_context.getBackgroundMovePool(); + move_parts_task_handle = pool.createTask([this] { return movePartsTask(); }); + pool.startTask(move_parts_task_handle); } } From c658d85aa831a908d930d4a580c73bbbfdee2579 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 7 Mar 2020 22:04:47 +0300 Subject: [PATCH 107/712] Whitespaces --- dbms/src/Processors/LimitTransform.h | 1 - 1 file changed, 1 deletion(-) diff --git a/dbms/src/Processors/LimitTransform.h b/dbms/src/Processors/LimitTransform.h index c308c88db25..f37bf8aba15 100644 --- a/dbms/src/Processors/LimitTransform.h +++ b/dbms/src/Processors/LimitTransform.h @@ -12,7 +12,6 @@ private: InputPort & input; OutputPort & output; - size_t limit; size_t offset; size_t rows_read = 0; /// including the last read block From cfe2464e29d315ea409b50d2bad8449f9cd29e97 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 7 Mar 2020 22:32:04 +0300 Subject: [PATCH 108/712] Fixed errors --- dbms/src/Storages/StorageGenerateRandom.cpp | 59 +++++++++++++++------ 1 file changed, 43 insertions(+), 16 deletions(-) diff --git a/dbms/src/Storages/StorageGenerateRandom.cpp b/dbms/src/Storages/StorageGenerateRandom.cpp index 15634a9ae76..6f98dc1746c 100644 --- a/dbms/src/Storages/StorageGenerateRandom.cpp +++ b/dbms/src/Storages/StorageGenerateRandom.cpp @@ -49,7 +49,7 @@ void fillBufferWithRandomData(char * __restrict data, size_t size, pcg64_fast & /// The loop can be further optimized. UInt64 number = rng(); unalignedStore(data, number); - data += sizeof(UInt64); /// We assume that data has 15-byte padding (see PaddedPODArray) + data += sizeof(UInt64); /// We assume that data has at least 7-byte padding (see PaddedPODArray) } } @@ -63,23 +63,50 @@ ColumnPtr fillColumnWithRandomData( { case TypeIndex::String: { - auto size_column = ColumnUInt32::create(); - auto & sizes = size_column->getData(); + /// Mostly the same as the implementation of randomPrintableASCII function. - sizes.resize(limit); - for (UInt64 i = 0; i < limit; ++i) - sizes[i] = static_cast(rng()) % max_string_length; /// Slow + auto column = ColumnString::create(); + ColumnString::Chars & data_to = column->getChars(); + ColumnString::Offsets & offsets_to = column->getOffsets(); + offsets_to.resize(limit); - ColumnWithTypeAndName argument{std::move(size_column), std::make_shared(), "size"}; - - Block block + IColumn::Offset offset = 0; + for (size_t row_num = 0; row_num < limit; ++row_num) { - argument, - {nullptr, type, "result"} - }; + size_t length = rng() % (max_string_length + 1); /// Slow - FunctionFactory::instance().get("randomPrintableASCII", context)->build({argument})->execute(block, {0}, 1, limit); - return block.getByPosition(1).column; + IColumn::Offset next_offset = offset + length + 1; + data_to.resize(next_offset); + offsets_to[row_num] = next_offset; + + auto * data_to_ptr = data_to.data(); /// avoid assert on array indexing after end + for (size_t pos = offset, end = offset + length; pos < end; pos += 4) /// We have padding in column buffers that we can overwrite. + { + UInt64 rand = rng(); + + UInt16 rand1 = rand; + UInt16 rand2 = rand >> 16; + UInt16 rand3 = rand >> 32; + UInt16 rand4 = rand >> 48; + + /// Printable characters are from range [32; 126]. + /// https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ + + data_to_ptr[pos + 0] = 32 + ((rand1 * 95) >> 16); + data_to_ptr[pos + 1] = 32 + ((rand2 * 95) >> 16); + data_to_ptr[pos + 2] = 32 + ((rand3 * 95) >> 16); + data_to_ptr[pos + 3] = 32 + ((rand4 * 95) >> 16); + + /// NOTE gcc failed to vectorize this code (aliasing of char?) + /// TODO Implement SIMD optimizations from Danila Kutenin. + } + + data_to[offset + length] = 0; + + offset = next_offset; + } + + return column; } case TypeIndex::Enum8: @@ -129,7 +156,7 @@ ColumnPtr fillColumnWithRandomData( offsets.resize(limit); for (UInt64 i = 0; i < limit; ++i) { - offset += static_cast(rng()) % max_array_length; + offset += static_cast(rng()) % (max_array_length + 1); offsets[i] = offset; } @@ -252,7 +279,7 @@ ColumnPtr fillColumnWithRandomData( fillBufferWithRandomData(reinterpret_cast(column_concrete.getData().data()), limit * sizeof(Decimal32), rng); return column; } - case TypeIndex::Decimal64: + case TypeIndex::Decimal64: /// TODO Decimal may be generated out of range. { auto column = type->createColumn(); auto & column_concrete = typeid_cast &>(*column); From 7c791e1cf10dd503e426e9c0142b2f6da7b7fecf Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 7 Mar 2020 22:32:36 +0300 Subject: [PATCH 109/712] Remove infinite performance tests --- dbms/tests/performance/array_fill.xml | 2 +- dbms/tests/performance/base64_hits.xml | 2 +- dbms/tests/performance/bloom_filter.xml | 2 +- .../performance/mingroupby-orderbylimit1.xml | 2 +- dbms/tests/performance/sum_map.xml | 2 +- dbms/tests/performance/system_numbers.xml | 60 +++++++++---------- 6 files changed, 34 insertions(+), 36 deletions(-) diff --git a/dbms/tests/performance/array_fill.xml b/dbms/tests/performance/array_fill.xml index 25ed745158b..51204299fad 100644 --- a/dbms/tests/performance/array_fill.xml +++ b/dbms/tests/performance/array_fill.xml @@ -1,5 +1,5 @@ - once + loop diff --git a/dbms/tests/performance/base64_hits.xml b/dbms/tests/performance/base64_hits.xml index edf4321fa07..1a1be4842bc 100644 --- a/dbms/tests/performance/base64_hits.xml +++ b/dbms/tests/performance/base64_hits.xml @@ -1,5 +1,5 @@ - once + loop hits_100m_single diff --git a/dbms/tests/performance/bloom_filter.xml b/dbms/tests/performance/bloom_filter.xml index 073b3c722be..ef35af9965d 100644 --- a/dbms/tests/performance/bloom_filter.xml +++ b/dbms/tests/performance/bloom_filter.xml @@ -1,5 +1,5 @@ - once + loop diff --git a/dbms/tests/performance/mingroupby-orderbylimit1.xml b/dbms/tests/performance/mingroupby-orderbylimit1.xml index 85735e1ef25..ec69ffa2b8f 100644 --- a/dbms/tests/performance/mingroupby-orderbylimit1.xml +++ b/dbms/tests/performance/mingroupby-orderbylimit1.xml @@ -1,5 +1,5 @@ - once + loop diff --git a/dbms/tests/performance/sum_map.xml b/dbms/tests/performance/sum_map.xml index 69993ad58dc..ac1ccaae0fe 100644 --- a/dbms/tests/performance/sum_map.xml +++ b/dbms/tests/performance/sum_map.xml @@ -1,5 +1,5 @@ - once + loop diff --git a/dbms/tests/performance/system_numbers.xml b/dbms/tests/performance/system_numbers.xml index 296d6850250..94d8ef76fa1 100644 --- a/dbms/tests/performance/system_numbers.xml +++ b/dbms/tests/performance/system_numbers.xml @@ -3,8 +3,6 @@ - - 4000 12000 @@ -17,48 +15,48 @@ То есть, этот запрос представляет собой необычным образом написанный бесконечный цикл. Мы запускаем этот запрос и наблюдаем, с какой скоростью он выполняется. Через несколько секунд, когда скорость стабилизируется, прерываем выполнение. В качестве скорости выполнения запроса указывается количество обработанных исходных (прочитанных из таблицы) данных в единицу времени. -Например, в таблице system.numbers читаемые нами данные - это числа типа UInt64 (8 байт). Если мы обрабатываем миллиард таких чисел в секунду, то отобразится скорость - 8 GB/sec. --> -SELECT count() FROM system.numbers WHERE NOT ignore(rand()) -SELECT count() FROM system.numbers_mt WHERE NOT ignore(rand()) +Например, в таблице numbers читаемые нами данные - это числа типа UInt64 (8 байт). Если мы обрабатываем миллиард таких чисел в секунду, то отобразится скорость - 8 GB/sec. --> +SELECT count() FROM numbers(100000000) WHERE NOT ignore(rand()) +SELECT count() FROM numbers_mt(1600000000) WHERE NOT ignore(rand()) -SELECT count() FROM system.numbers WHERE NOT ignore(intHash64(number)) -SELECT count() FROM system.numbers_mt WHERE NOT ignore(intHash64(number)) +SELECT count() FROM numbers(100000000) WHERE NOT ignore(intHash64(number)) +SELECT count() FROM numbers_mt(1600000000) WHERE NOT ignore(intHash64(number)) -SELECT count() FROM system.numbers WHERE NOT ignore(intHash32(number)) -SELECT count() FROM system.numbers_mt WHERE NOT ignore(intHash32(number)) +SELECT count() FROM numbers(100000000) WHERE NOT ignore(intHash32(number)) +SELECT count() FROM numbers_mt(1600000000) WHERE NOT ignore(intHash32(number)) -SELECT count() FROM system.numbers WHERE NOT ignore(toString(number)) -SELECT count() FROM system.numbers_mt WHERE NOT ignore(toString(number)) +SELECT count() FROM numbers(100000000) WHERE NOT ignore(toString(number)) +SELECT count() FROM numbers_mt(1600000000) WHERE NOT ignore(toString(number)) -SELECT count() FROM system.numbers WHERE NOT ignore(reinterpretAsString(number)) -SELECT count() FROM system.numbers_mt WHERE NOT ignore(reinterpretAsString(number)) +SELECT count() FROM numbers(100000000) WHERE NOT ignore(reinterpretAsString(number)) +SELECT count() FROM numbers_mt(1600000000) WHERE NOT ignore(reinterpretAsString(number)) -SELECT count() FROM system.numbers WHERE NOT ignore(number / 7) -SELECT count() FROM system.numbers_mt WHERE NOT ignore(number / 7) +SELECT count() FROM numbers(100000000) WHERE NOT ignore(number / 7) +SELECT count() FROM numbers_mt(1600000000) WHERE NOT ignore(number / 7) -SELECT count() FROM system.numbers WHERE NOT ignore(number % 7) -SELECT count() FROM system.numbers_mt WHERE NOT ignore(number % 7) +SELECT count() FROM numbers(100000000) WHERE NOT ignore(number % 7) +SELECT count() FROM numbers_mt(1600000000) WHERE NOT ignore(number % 7) -SELECT count() FROM system.numbers WHERE NOT ignore(number % 34908756) -SELECT count() FROM system.numbers_mt WHERE NOT ignore(number % 34908756) +SELECT count() FROM numbers(100000000) WHERE NOT ignore(number % 34908756) +SELECT count() FROM numbers_mt(1600000000) WHERE NOT ignore(number % 34908756) -SELECT number % 1000 AS k, count() FROM system.numbers GROUP BY k -SELECT number % 1000 AS k, count() FROM system.numbers_mt GROUP BY k +SELECT number % 1000 AS k, count() FROM numbers(100000000) GROUP BY k +SELECT number % 1000 AS k, count() FROM numbers_mt(1600000000) GROUP BY k -SELECT number % 100000 AS k, count() FROM system.numbers GROUP BY k -SELECT number % 100000 AS k, count() FROM system.numbers_mt GROUP BY k +SELECT number % 100000 AS k, count() FROM numbers(100000000) GROUP BY k +SELECT number % 100000 AS k, count() FROM numbers_mt(1600000000) GROUP BY k -SELECT number % 1000000 AS k, count() FROM system.numbers GROUP BY k -SELECT number % 1000000 AS k, count() FROM system.numbers_mt GROUP BY k +SELECT number % 1000000 AS k, count() FROM numbers(100000000) GROUP BY k +SELECT number % 1000000 AS k, count() FROM numbers_mt(1600000000) GROUP BY k -SELECT number % 10000000 AS k, count() FROM system.numbers GROUP BY k -SELECT number % 10000000 AS k, count() FROM system.numbers_mt GROUP BY k +SELECT number % 10000000 AS k, count() FROM numbers(100000000) GROUP BY k +SELECT number % 10000000 AS k, count() FROM numbers_mt(1600000000) GROUP BY k -SELECT number % 500000000 AS k, count() FROM system.numbers GROUP BY k -SELECT number % 500000000 AS k, count() FROM system.numbers_mt GROUP BY k +SELECT number % 500000000 AS k, count() FROM numbers(100000000) GROUP BY k +SELECT number % 500000000 AS k, count() FROM numbers_mt(1600000000) GROUP BY k - + -SELECT count() FROM system.numbers WHERE NOT ignore(materialize('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx') AS s, concat(s,s,s,s,s,s,s,s,s,s) AS t, concat(t,t,t,t,t,t,t,t,t,t) AS u) SETTINGS max_block_size = 1000 +SELECT count() FROM numbers(100000000) WHERE NOT ignore(materialize('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx') AS s, concat(s,s,s,s,s,s,s,s,s,s) AS t, concat(t,t,t,t,t,t,t,t,t,t) AS u) SETTINGS max_block_size = 1000 From 8fa05b080c239d942d36b48d18434c779dc97010 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 7 Mar 2020 22:33:06 +0300 Subject: [PATCH 110/712] Renamed a test --- .../{system_numbers.xml => synthetic_hardware_benchmark.xml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename dbms/tests/performance/{system_numbers.xml => synthetic_hardware_benchmark.xml} (100%) diff --git a/dbms/tests/performance/system_numbers.xml b/dbms/tests/performance/synthetic_hardware_benchmark.xml similarity index 100% rename from dbms/tests/performance/system_numbers.xml rename to dbms/tests/performance/synthetic_hardware_benchmark.xml From 1645ce8696a252896118feea0ef0891d07f7cd9c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 7 Mar 2020 22:41:16 +0300 Subject: [PATCH 111/712] Removed outdated info --- dbms/tests/performance/README.md | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/dbms/tests/performance/README.md b/dbms/tests/performance/README.md index d436eb7bce3..a797b88a321 100644 --- a/dbms/tests/performance/README.md +++ b/dbms/tests/performance/README.md @@ -6,17 +6,9 @@ This directory contains `.xml`-files with performance tests for `clickhouse-perf First of all you should check existing tests don't cover your case. If there are no such tests than you should write your own. -There two types of performance tests: -* First is executed in loop, and have tag `loop` in config. -* Second one is executed only once and have tag `once` in config. +You have to specify `preconditions`. It contains table names. Only `hits_100m_single`, `hits_10m_single`, `test.hits` are available in CI. -Type `once` should be used only for endless queries. Even if your query really long (10 seconds+), it's better to choose `loop` test. - -After you have choosen type, you have to specify `preconditions`. It contains table names. Only `hits_100m_single`, `hits_10m_single`, `test.hits` are available in CI. - -The most important part of test is `stop_conditions`. For `loop` test you should always use `min_time_not_changing_for_ms` stop condition. For `once` test you can choose between `average_speed_not_changing_for_ms` and `max_speed_not_changing_for_ms`, but first is preferable. Also you should always specify `total_time_ms` metric. Endless tests will be ignored by CI. - -`loop` tests are always compared by `min_time` metric and `once` tests compared by `max_rows_per_second`. +The most important part of test is `stop_conditions`. Also you should always specify `total_time_ms` metric. Endless tests will be ignored by CI. You can use `substitions`, `create`, `fill` and `drop` queries to prepare test. You can find examples in this folder. From c30d1ba18b1fbdc6544906d4d2fb1152d4d52524 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 7 Mar 2020 22:44:18 +0300 Subject: [PATCH 112/712] Addition to prev. revision --- dbms/tests/performance/IPv4.xml | 1 - dbms/tests/performance/IPv6.xml | 1 - dbms/tests/performance/agg_functions_min_max_any.xml | 1 - dbms/tests/performance/and_function.xml | 1 - dbms/tests/performance/arithmetic.xml | 1 - dbms/tests/performance/array_element.xml | 1 - dbms/tests/performance/array_fill.xml | 1 - dbms/tests/performance/array_join.xml | 1 - dbms/tests/performance/base64.xml | 1 - dbms/tests/performance/base64_hits.xml | 1 - dbms/tests/performance/basename.xml | 1 - dbms/tests/performance/bitCount.xml | 1 - dbms/tests/performance/bit_operations_fixed_string.xml | 1 - .../performance/bit_operations_fixed_string_numbers.xml | 1 - dbms/tests/performance/bloom_filter.xml | 1 - dbms/tests/performance/bounding_ratio.xml | 1 - dbms/tests/performance/cidr.xml | 1 - dbms/tests/performance/codecs_float_insert.xml | 1 - dbms/tests/performance/codecs_float_select.xml | 1 - dbms/tests/performance/codecs_int_insert.xml | 1 - dbms/tests/performance/codecs_int_select.xml | 1 - dbms/tests/performance/collations.xml | 1 - dbms/tests/performance/column_column_comparison.xml | 1 - dbms/tests/performance/columns_hashing.xml | 1 - dbms/tests/performance/complex_array_creation.xml | 1 - dbms/tests/performance/concat_hits.xml | 1 - dbms/tests/performance/conditional.xml | 1 - dbms/tests/performance/consistent_hashes.xml | 1 - dbms/tests/performance/constant_column_comparison.xml | 1 - dbms/tests/performance/constant_column_search.xml | 1 - dbms/tests/performance/count.xml | 1 - dbms/tests/performance/cpu_synthetic.xml | 1 - dbms/tests/performance/cryptographic_hashes.xml | 1 - dbms/tests/performance/date_parsing.xml | 1 - dbms/tests/performance/date_time.xml | 1 - dbms/tests/performance/date_time_64.xml | 1 - dbms/tests/performance/decimal_aggregates.xml | 1 - dbms/tests/performance/early_constant_folding.xml | 1 - dbms/tests/performance/empty_string_deserialization.xml | 1 - dbms/tests/performance/empty_string_serialization.xml | 1 - dbms/tests/performance/entropy.xml | 1 - dbms/tests/performance/first_significant_subdomain.xml | 1 - dbms/tests/performance/fixed_string16.xml | 1 - dbms/tests/performance/float_formatting.xml | 1 - dbms/tests/performance/float_parsing.xml | 1 - dbms/tests/performance/format_date_time.xml | 1 - dbms/tests/performance/functions_coding.xml | 1 - dbms/tests/performance/functions_geo.xml | 1 - dbms/tests/performance/general_purpose_hashes.xml | 1 - dbms/tests/performance/general_purpose_hashes_on_UUID.xml | 1 - dbms/tests/performance/generate_table_function.xml | 2 -- dbms/tests/performance/great_circle_dist.xml | 1 - dbms/tests/performance/group_array_moving_sum.xml | 1 - dbms/tests/performance/h3.xml | 1 - dbms/tests/performance/if_array_num.xml | 1 - dbms/tests/performance/if_array_string.xml | 1 - dbms/tests/performance/if_string_const.xml | 1 - dbms/tests/performance/if_string_hits.xml | 1 - dbms/tests/performance/if_to_multiif.xml | 1 - dbms/tests/performance/information_value.xml | 1 - dbms/tests/performance/insert_values_with_expressions.xml | 1 - dbms/tests/performance/inserts_arrays_lowcardinality.xml | 1 - dbms/tests/performance/int_parsing.xml | 1 - dbms/tests/performance/jit_large_requests.xml | 1 - dbms/tests/performance/jit_small_requests.xml | 1 - dbms/tests/performance/joins_in_memory.xml | 1 - dbms/tests/performance/joins_in_memory_pmj.xml | 1 - dbms/tests/performance/json_extract_rapidjson.xml | 1 - dbms/tests/performance/json_extract_simdjson.xml | 1 - dbms/tests/performance/leftpad.xml | 1 - dbms/tests/performance/linear_regression.xml | 1 - dbms/tests/performance/logical_functions_large.xml | 1 - dbms/tests/performance/logical_functions_medium.xml | 1 - dbms/tests/performance/logical_functions_small.xml | 1 - dbms/tests/performance/math.xml | 1 - dbms/tests/performance/merge_table_streams.xml | 1 - dbms/tests/performance/merge_tree_huge_pk.xml | 1 - dbms/tests/performance/merge_tree_many_partitions.xml | 1 - dbms/tests/performance/merge_tree_many_partitions_2.xml | 1 - dbms/tests/performance/merge_tree_simple_select.xml | 1 - dbms/tests/performance/mingroupby-orderbylimit1.xml | 1 - dbms/tests/performance/modulo.xml | 1 - dbms/tests/performance/ngram_distance.xml | 1 - dbms/tests/performance/number_formatting_formats.xml | 1 - dbms/tests/performance/nyc_taxi.xml | 1 - dbms/tests/performance/order_by_decimals.xml | 1 - dbms/tests/performance/order_by_read_in_order.xml | 1 - dbms/tests/performance/order_by_single_column.xml | 1 - dbms/tests/performance/parallel_insert.xml | 1 - dbms/tests/performance/parse_engine_file.xml | 1 - dbms/tests/performance/prewhere.xml | 1 - dbms/tests/performance/random_printable_ascii.xml | 1 - dbms/tests/performance/range.xml | 1 - dbms/tests/performance/read_hits_with_aio.xml | 1 - dbms/tests/performance/right.xml | 1 - dbms/tests/performance/round_down.xml | 1 - dbms/tests/performance/round_methods.xml | 1 - dbms/tests/performance/scalar.xml | 1 - dbms/tests/performance/select_format.xml | 1 - dbms/tests/performance/set.xml | 1 - dbms/tests/performance/set_hits.xml | 1 - dbms/tests/performance/set_index.xml | 1 - dbms/tests/performance/simple_join_query.xml | 1 - dbms/tests/performance/slices_hits.xml | 1 - dbms/tests/performance/sort.xml | 1 - dbms/tests/performance/string_join.xml | 1 - dbms/tests/performance/string_set.xml | 1 - dbms/tests/performance/string_sort.xml | 1 - dbms/tests/performance/sum_map.xml | 1 - dbms/tests/performance/synthetic_hardware_benchmark.xml | 1 - dbms/tests/performance/trim_numbers.xml | 2 -- dbms/tests/performance/trim_urls.xml | 2 -- dbms/tests/performance/trim_whitespace.xml | 1 - dbms/tests/performance/uniq.xml | 1 - dbms/tests/performance/url_hits.xml | 2 -- .../tests/performance/vectorize_aggregation_combinators.xml | 2 -- dbms/tests/performance/visit_param_extract_raw.xml | 6 +----- dbms/tests/performance/website.xml | 1 - 118 files changed, 1 insertion(+), 127 deletions(-) diff --git a/dbms/tests/performance/IPv4.xml b/dbms/tests/performance/IPv4.xml index 41e519e0b9d..8f5b61d70c9 100644 --- a/dbms/tests/performance/IPv4.xml +++ b/dbms/tests/performance/IPv4.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/IPv6.xml b/dbms/tests/performance/IPv6.xml index 1c7c9d679ab..272d54c851b 100644 --- a/dbms/tests/performance/IPv6.xml +++ b/dbms/tests/performance/IPv6.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/agg_functions_min_max_any.xml b/dbms/tests/performance/agg_functions_min_max_any.xml index 8a132bb79a9..c63d0098262 100644 --- a/dbms/tests/performance/agg_functions_min_max_any.xml +++ b/dbms/tests/performance/agg_functions_min_max_any.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/and_function.xml b/dbms/tests/performance/and_function.xml index da9a3d271ab..1fb9dfa53f9 100644 --- a/dbms/tests/performance/and_function.xml +++ b/dbms/tests/performance/and_function.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/arithmetic.xml b/dbms/tests/performance/arithmetic.xml index 348273bebbf..3af85a260f3 100644 --- a/dbms/tests/performance/arithmetic.xml +++ b/dbms/tests/performance/arithmetic.xml @@ -1,5 +1,4 @@ - loop 10 diff --git a/dbms/tests/performance/array_element.xml b/dbms/tests/performance/array_element.xml index f4a33810fdd..4cc9bdc4a38 100644 --- a/dbms/tests/performance/array_element.xml +++ b/dbms/tests/performance/array_element.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/array_fill.xml b/dbms/tests/performance/array_fill.xml index 51204299fad..ccd2c5eba7c 100644 --- a/dbms/tests/performance/array_fill.xml +++ b/dbms/tests/performance/array_fill.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/array_join.xml b/dbms/tests/performance/array_join.xml index e5025695d15..c95681d4d29 100644 --- a/dbms/tests/performance/array_join.xml +++ b/dbms/tests/performance/array_join.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/base64.xml b/dbms/tests/performance/base64.xml index 232b4d20ba3..c25694841f1 100644 --- a/dbms/tests/performance/base64.xml +++ b/dbms/tests/performance/base64.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/base64_hits.xml b/dbms/tests/performance/base64_hits.xml index 1a1be4842bc..d585408fff8 100644 --- a/dbms/tests/performance/base64_hits.xml +++ b/dbms/tests/performance/base64_hits.xml @@ -1,5 +1,4 @@ - loop hits_100m_single diff --git a/dbms/tests/performance/basename.xml b/dbms/tests/performance/basename.xml index 6af67bc94c4..691fc38b8ca 100644 --- a/dbms/tests/performance/basename.xml +++ b/dbms/tests/performance/basename.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/bitCount.xml b/dbms/tests/performance/bitCount.xml index 60901885dbd..34fdb24c10b 100644 --- a/dbms/tests/performance/bitCount.xml +++ b/dbms/tests/performance/bitCount.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/bit_operations_fixed_string.xml b/dbms/tests/performance/bit_operations_fixed_string.xml index 53071b94d49..90df91f1025 100644 --- a/dbms/tests/performance/bit_operations_fixed_string.xml +++ b/dbms/tests/performance/bit_operations_fixed_string.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/bit_operations_fixed_string_numbers.xml b/dbms/tests/performance/bit_operations_fixed_string_numbers.xml index 2ba3ffb5d86..779aea19cdc 100644 --- a/dbms/tests/performance/bit_operations_fixed_string_numbers.xml +++ b/dbms/tests/performance/bit_operations_fixed_string_numbers.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/bloom_filter.xml b/dbms/tests/performance/bloom_filter.xml index ef35af9965d..079b7a43da3 100644 --- a/dbms/tests/performance/bloom_filter.xml +++ b/dbms/tests/performance/bloom_filter.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/bounding_ratio.xml b/dbms/tests/performance/bounding_ratio.xml index 4bf50f57290..0d0adfaea45 100644 --- a/dbms/tests/performance/bounding_ratio.xml +++ b/dbms/tests/performance/bounding_ratio.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/cidr.xml b/dbms/tests/performance/cidr.xml index 1ca7f691881..938734e3709 100644 --- a/dbms/tests/performance/cidr.xml +++ b/dbms/tests/performance/cidr.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/codecs_float_insert.xml b/dbms/tests/performance/codecs_float_insert.xml index 1bbbf6b2d92..706a2f3c0a0 100644 --- a/dbms/tests/performance/codecs_float_insert.xml +++ b/dbms/tests/performance/codecs_float_insert.xml @@ -1,6 +1,5 @@ - loop 10 diff --git a/dbms/tests/performance/codecs_float_select.xml b/dbms/tests/performance/codecs_float_select.xml index 1d3957c8da9..4c2f671a90e 100644 --- a/dbms/tests/performance/codecs_float_select.xml +++ b/dbms/tests/performance/codecs_float_select.xml @@ -1,6 +1,5 @@ - loop 10 diff --git a/dbms/tests/performance/codecs_int_insert.xml b/dbms/tests/performance/codecs_int_insert.xml index eea263e601a..1226d9020a0 100644 --- a/dbms/tests/performance/codecs_int_insert.xml +++ b/dbms/tests/performance/codecs_int_insert.xml @@ -1,6 +1,5 @@ - loop 10 diff --git a/dbms/tests/performance/codecs_int_select.xml b/dbms/tests/performance/codecs_int_select.xml index 40ebfd4d000..8054c2b2de4 100644 --- a/dbms/tests/performance/codecs_int_select.xml +++ b/dbms/tests/performance/codecs_int_select.xml @@ -1,6 +1,5 @@ - loop 10 diff --git a/dbms/tests/performance/collations.xml b/dbms/tests/performance/collations.xml index 1bec38dd103..03d77fa5e27 100644 --- a/dbms/tests/performance/collations.xml +++ b/dbms/tests/performance/collations.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/column_column_comparison.xml b/dbms/tests/performance/column_column_comparison.xml index 9d4446d7c2d..7559e03e506 100644 --- a/dbms/tests/performance/column_column_comparison.xml +++ b/dbms/tests/performance/column_column_comparison.xml @@ -7,7 +7,6 @@ hits_100m_single - loop diff --git a/dbms/tests/performance/columns_hashing.xml b/dbms/tests/performance/columns_hashing.xml index 138855dae89..ca330b0d435 100644 --- a/dbms/tests/performance/columns_hashing.xml +++ b/dbms/tests/performance/columns_hashing.xml @@ -8,7 +8,6 @@ hits_1000m_single - loop diff --git a/dbms/tests/performance/complex_array_creation.xml b/dbms/tests/performance/complex_array_creation.xml index 76e4910a1d7..abcea2671e7 100644 --- a/dbms/tests/performance/complex_array_creation.xml +++ b/dbms/tests/performance/complex_array_creation.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/concat_hits.xml b/dbms/tests/performance/concat_hits.xml index e2c6fc23c08..49ab27bf540 100644 --- a/dbms/tests/performance/concat_hits.xml +++ b/dbms/tests/performance/concat_hits.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/conditional.xml b/dbms/tests/performance/conditional.xml index eea43d6556a..2a4f4ccad6e 100644 --- a/dbms/tests/performance/conditional.xml +++ b/dbms/tests/performance/conditional.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/consistent_hashes.xml b/dbms/tests/performance/consistent_hashes.xml index 5929c6388d5..349f2c3b05f 100644 --- a/dbms/tests/performance/consistent_hashes.xml +++ b/dbms/tests/performance/consistent_hashes.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/constant_column_comparison.xml b/dbms/tests/performance/constant_column_comparison.xml index f32ed444a0c..34e6193675c 100644 --- a/dbms/tests/performance/constant_column_comparison.xml +++ b/dbms/tests/performance/constant_column_comparison.xml @@ -7,7 +7,6 @@ hits_100m_single - loop diff --git a/dbms/tests/performance/constant_column_search.xml b/dbms/tests/performance/constant_column_search.xml index 9953c2797a2..0c19624f5bc 100644 --- a/dbms/tests/performance/constant_column_search.xml +++ b/dbms/tests/performance/constant_column_search.xml @@ -7,7 +7,6 @@ hits_100m_single - loop diff --git a/dbms/tests/performance/count.xml b/dbms/tests/performance/count.xml index 3bb4a0d2cd5..0d8470bbf3a 100644 --- a/dbms/tests/performance/count.xml +++ b/dbms/tests/performance/count.xml @@ -1,6 +1,5 @@ - loop diff --git a/dbms/tests/performance/cpu_synthetic.xml b/dbms/tests/performance/cpu_synthetic.xml index dd7ac14ccf8..762115756de 100644 --- a/dbms/tests/performance/cpu_synthetic.xml +++ b/dbms/tests/performance/cpu_synthetic.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/cryptographic_hashes.xml b/dbms/tests/performance/cryptographic_hashes.xml index 7bafb25f299..6dcc05d17fb 100644 --- a/dbms/tests/performance/cryptographic_hashes.xml +++ b/dbms/tests/performance/cryptographic_hashes.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/date_parsing.xml b/dbms/tests/performance/date_parsing.xml index 8ecf3681804..f2e3954948e 100644 --- a/dbms/tests/performance/date_parsing.xml +++ b/dbms/tests/performance/date_parsing.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/date_time.xml b/dbms/tests/performance/date_time.xml index 4e9cd2c4abd..858b20d5784 100644 --- a/dbms/tests/performance/date_time.xml +++ b/dbms/tests/performance/date_time.xml @@ -1,6 +1,5 @@ - loop long diff --git a/dbms/tests/performance/date_time_64.xml b/dbms/tests/performance/date_time_64.xml index b345550b335..ccce5b54ecc 100644 --- a/dbms/tests/performance/date_time_64.xml +++ b/dbms/tests/performance/date_time_64.xml @@ -1,5 +1,4 @@ - loop default.hits_100m_single diff --git a/dbms/tests/performance/decimal_aggregates.xml b/dbms/tests/performance/decimal_aggregates.xml index 86830fedce6..c6639ea1053 100644 --- a/dbms/tests/performance/decimal_aggregates.xml +++ b/dbms/tests/performance/decimal_aggregates.xml @@ -1,5 +1,4 @@ - loop CREATE TABLE t (x UInt64, d32 Decimal32(3), d64 Decimal64(4), d128 Decimal128(5)) ENGINE = Memory INSERT INTO t SELECT number AS x, x AS d32, x AS d64, x d128 FROM numbers(1000000) diff --git a/dbms/tests/performance/early_constant_folding.xml b/dbms/tests/performance/early_constant_folding.xml index ad2d1619eb9..2cc68a7b8c6 100644 --- a/dbms/tests/performance/early_constant_folding.xml +++ b/dbms/tests/performance/early_constant_folding.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/empty_string_deserialization.xml b/dbms/tests/performance/empty_string_deserialization.xml index c56f67ab274..d4b4f338a85 100644 --- a/dbms/tests/performance/empty_string_deserialization.xml +++ b/dbms/tests/performance/empty_string_deserialization.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/empty_string_serialization.xml b/dbms/tests/performance/empty_string_serialization.xml index 46c4bc0275c..62b2e13f0af 100644 --- a/dbms/tests/performance/empty_string_serialization.xml +++ b/dbms/tests/performance/empty_string_serialization.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/entropy.xml b/dbms/tests/performance/entropy.xml index 45c9ccb840d..e5a96be165c 100644 --- a/dbms/tests/performance/entropy.xml +++ b/dbms/tests/performance/entropy.xml @@ -1,5 +1,4 @@ - loop test.hits diff --git a/dbms/tests/performance/first_significant_subdomain.xml b/dbms/tests/performance/first_significant_subdomain.xml index 705e70b86f9..d8ef4b01e54 100644 --- a/dbms/tests/performance/first_significant_subdomain.xml +++ b/dbms/tests/performance/first_significant_subdomain.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/fixed_string16.xml b/dbms/tests/performance/fixed_string16.xml index 398f09aba3d..6dd86ea7479 100644 --- a/dbms/tests/performance/fixed_string16.xml +++ b/dbms/tests/performance/fixed_string16.xml @@ -7,7 +7,6 @@ test.hits - loop diff --git a/dbms/tests/performance/float_formatting.xml b/dbms/tests/performance/float_formatting.xml index aaf2fad0c93..941e1bcaaa8 100644 --- a/dbms/tests/performance/float_formatting.xml +++ b/dbms/tests/performance/float_formatting.xml @@ -1,5 +1,4 @@ - loop long diff --git a/dbms/tests/performance/float_parsing.xml b/dbms/tests/performance/float_parsing.xml index e7779751fa4..f75de81c698 100644 --- a/dbms/tests/performance/float_parsing.xml +++ b/dbms/tests/performance/float_parsing.xml @@ -1,5 +1,4 @@ - loop long diff --git a/dbms/tests/performance/format_date_time.xml b/dbms/tests/performance/format_date_time.xml index aa070c40ec5..aed8fd7dc77 100644 --- a/dbms/tests/performance/format_date_time.xml +++ b/dbms/tests/performance/format_date_time.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/functions_coding.xml b/dbms/tests/performance/functions_coding.xml index 93e16a8a221..52ada24b5bc 100644 --- a/dbms/tests/performance/functions_coding.xml +++ b/dbms/tests/performance/functions_coding.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/functions_geo.xml b/dbms/tests/performance/functions_geo.xml index a4233b2fe57..207d39c52b7 100644 --- a/dbms/tests/performance/functions_geo.xml +++ b/dbms/tests/performance/functions_geo.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/general_purpose_hashes.xml b/dbms/tests/performance/general_purpose_hashes.xml index 94c8d5d4b2b..cc40c7fe1e3 100644 --- a/dbms/tests/performance/general_purpose_hashes.xml +++ b/dbms/tests/performance/general_purpose_hashes.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/general_purpose_hashes_on_UUID.xml b/dbms/tests/performance/general_purpose_hashes_on_UUID.xml index 964879148d0..3cb14e4c87c 100644 --- a/dbms/tests/performance/general_purpose_hashes_on_UUID.xml +++ b/dbms/tests/performance/general_purpose_hashes_on_UUID.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/generate_table_function.xml b/dbms/tests/performance/generate_table_function.xml index 4674b81af99..48b9b22df1c 100644 --- a/dbms/tests/performance/generate_table_function.xml +++ b/dbms/tests/performance/generate_table_function.xml @@ -1,9 +1,7 @@ - loop - 4000 10000 diff --git a/dbms/tests/performance/great_circle_dist.xml b/dbms/tests/performance/great_circle_dist.xml index 3edfc2c8008..3b88d00eb63 100644 --- a/dbms/tests/performance/great_circle_dist.xml +++ b/dbms/tests/performance/great_circle_dist.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/group_array_moving_sum.xml b/dbms/tests/performance/group_array_moving_sum.xml index 6939989c5b4..6da1752e1f6 100644 --- a/dbms/tests/performance/group_array_moving_sum.xml +++ b/dbms/tests/performance/group_array_moving_sum.xml @@ -1,6 +1,5 @@ - loop diff --git a/dbms/tests/performance/h3.xml b/dbms/tests/performance/h3.xml index 3a6d5940d0d..a09ac88f727 100644 --- a/dbms/tests/performance/h3.xml +++ b/dbms/tests/performance/h3.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/if_array_num.xml b/dbms/tests/performance/if_array_num.xml index d4c9c29dd99..4ae4db3afdc 100644 --- a/dbms/tests/performance/if_array_num.xml +++ b/dbms/tests/performance/if_array_num.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/if_array_string.xml b/dbms/tests/performance/if_array_string.xml index 235051fc905..95dfb809230 100644 --- a/dbms/tests/performance/if_array_string.xml +++ b/dbms/tests/performance/if_array_string.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/if_string_const.xml b/dbms/tests/performance/if_string_const.xml index 5b06440473f..5ab8455c948 100644 --- a/dbms/tests/performance/if_string_const.xml +++ b/dbms/tests/performance/if_string_const.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/if_string_hits.xml b/dbms/tests/performance/if_string_hits.xml index 267c8b039e5..ec9ea39f7cf 100644 --- a/dbms/tests/performance/if_string_hits.xml +++ b/dbms/tests/performance/if_string_hits.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/if_to_multiif.xml b/dbms/tests/performance/if_to_multiif.xml index e1c45bbb69e..373318c316c 100644 --- a/dbms/tests/performance/if_to_multiif.xml +++ b/dbms/tests/performance/if_to_multiif.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/information_value.xml b/dbms/tests/performance/information_value.xml index f5b73a18abc..6f94d828eb9 100644 --- a/dbms/tests/performance/information_value.xml +++ b/dbms/tests/performance/information_value.xml @@ -1,5 +1,4 @@ - loop hits_100m_single diff --git a/dbms/tests/performance/insert_values_with_expressions.xml b/dbms/tests/performance/insert_values_with_expressions.xml index 66fe2aef18b..4464066c16e 100644 --- a/dbms/tests/performance/insert_values_with_expressions.xml +++ b/dbms/tests/performance/insert_values_with_expressions.xml @@ -1,5 +1,4 @@ - loop 1 diff --git a/dbms/tests/performance/inserts_arrays_lowcardinality.xml b/dbms/tests/performance/inserts_arrays_lowcardinality.xml index a453cfb07f8..bca5c858576 100644 --- a/dbms/tests/performance/inserts_arrays_lowcardinality.xml +++ b/dbms/tests/performance/inserts_arrays_lowcardinality.xml @@ -1,5 +1,4 @@ - loop 5 diff --git a/dbms/tests/performance/int_parsing.xml b/dbms/tests/performance/int_parsing.xml index 51f740523ba..8a6475546bf 100644 --- a/dbms/tests/performance/int_parsing.xml +++ b/dbms/tests/performance/int_parsing.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/jit_large_requests.xml b/dbms/tests/performance/jit_large_requests.xml index 54aa2af65b1..03ef588ca87 100644 --- a/dbms/tests/performance/jit_large_requests.xml +++ b/dbms/tests/performance/jit_large_requests.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/jit_small_requests.xml b/dbms/tests/performance/jit_small_requests.xml index d65e14cb97e..edf9311eb05 100644 --- a/dbms/tests/performance/jit_small_requests.xml +++ b/dbms/tests/performance/jit_small_requests.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/joins_in_memory.xml b/dbms/tests/performance/joins_in_memory.xml index f624030d7d4..1d3b14ae962 100644 --- a/dbms/tests/performance/joins_in_memory.xml +++ b/dbms/tests/performance/joins_in_memory.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/joins_in_memory_pmj.xml b/dbms/tests/performance/joins_in_memory_pmj.xml index 0352268c846..19383467fa1 100644 --- a/dbms/tests/performance/joins_in_memory_pmj.xml +++ b/dbms/tests/performance/joins_in_memory_pmj.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/json_extract_rapidjson.xml b/dbms/tests/performance/json_extract_rapidjson.xml index 8a2718d4a56..42b89456c9c 100644 --- a/dbms/tests/performance/json_extract_rapidjson.xml +++ b/dbms/tests/performance/json_extract_rapidjson.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/json_extract_simdjson.xml b/dbms/tests/performance/json_extract_simdjson.xml index f3a38912b0f..1e0c992802e 100644 --- a/dbms/tests/performance/json_extract_simdjson.xml +++ b/dbms/tests/performance/json_extract_simdjson.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/leftpad.xml b/dbms/tests/performance/leftpad.xml index a0717adbbd8..eb0b09c72ed 100644 --- a/dbms/tests/performance/leftpad.xml +++ b/dbms/tests/performance/leftpad.xml @@ -8,7 +8,6 @@ hashfile - loop diff --git a/dbms/tests/performance/linear_regression.xml b/dbms/tests/performance/linear_regression.xml index c358e21af05..87fa034d851 100644 --- a/dbms/tests/performance/linear_regression.xml +++ b/dbms/tests/performance/linear_regression.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/logical_functions_large.xml b/dbms/tests/performance/logical_functions_large.xml index b90023c3f60..a87b41ec916 100644 --- a/dbms/tests/performance/logical_functions_large.xml +++ b/dbms/tests/performance/logical_functions_large.xml @@ -1,6 +1,5 @@ 1 - loop diff --git a/dbms/tests/performance/logical_functions_medium.xml b/dbms/tests/performance/logical_functions_medium.xml index 0f6de7ea23e..087917040d9 100644 --- a/dbms/tests/performance/logical_functions_medium.xml +++ b/dbms/tests/performance/logical_functions_medium.xml @@ -1,6 +1,5 @@ 1 - loop diff --git a/dbms/tests/performance/logical_functions_small.xml b/dbms/tests/performance/logical_functions_small.xml index a2c90346ed6..ed6ab2afde6 100644 --- a/dbms/tests/performance/logical_functions_small.xml +++ b/dbms/tests/performance/logical_functions_small.xml @@ -1,6 +1,5 @@ 1 - loop diff --git a/dbms/tests/performance/math.xml b/dbms/tests/performance/math.xml index 280f0821964..6ab497749f1 100644 --- a/dbms/tests/performance/math.xml +++ b/dbms/tests/performance/math.xml @@ -1,6 +1,5 @@ - loop diff --git a/dbms/tests/performance/merge_table_streams.xml b/dbms/tests/performance/merge_table_streams.xml index f1816e85097..084fa2da575 100644 --- a/dbms/tests/performance/merge_table_streams.xml +++ b/dbms/tests/performance/merge_table_streams.xml @@ -1,5 +1,4 @@ - loop hits_100m_single diff --git a/dbms/tests/performance/merge_tree_huge_pk.xml b/dbms/tests/performance/merge_tree_huge_pk.xml index e39ff7501f7..1636fd52e2d 100644 --- a/dbms/tests/performance/merge_tree_huge_pk.xml +++ b/dbms/tests/performance/merge_tree_huge_pk.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/merge_tree_many_partitions.xml b/dbms/tests/performance/merge_tree_many_partitions.xml index 6eb110bfab9..33bb12ed22b 100644 --- a/dbms/tests/performance/merge_tree_many_partitions.xml +++ b/dbms/tests/performance/merge_tree_many_partitions.xml @@ -1,5 +1,4 @@ - loop CREATE TABLE bad_partitions (x UInt64) ENGINE = MergeTree PARTITION BY x ORDER BY x INSERT INTO bad_partitions SELECT * FROM numbers(10000) diff --git a/dbms/tests/performance/merge_tree_many_partitions_2.xml b/dbms/tests/performance/merge_tree_many_partitions_2.xml index 35d158abf83..42bb0ac29c9 100644 --- a/dbms/tests/performance/merge_tree_many_partitions_2.xml +++ b/dbms/tests/performance/merge_tree_many_partitions_2.xml @@ -1,5 +1,4 @@ - loop CREATE TABLE bad_partitions (a UInt64, b UInt64, c UInt64, d UInt64, e UInt64, f UInt64, g UInt64, h UInt64, i UInt64, j UInt64, k UInt64, l UInt64, m UInt64, n UInt64, o UInt64, p UInt64, q UInt64, r UInt64, s UInt64, t UInt64, u UInt64, v UInt64, w UInt64, x UInt64, y UInt64, z UInt64) ENGINE = MergeTree PARTITION BY x ORDER BY x INSERT INTO bad_partitions (x) SELECT * FROM numbers_mt(3000) diff --git a/dbms/tests/performance/merge_tree_simple_select.xml b/dbms/tests/performance/merge_tree_simple_select.xml index 5600e12a5db..f38a5241cb5 100644 --- a/dbms/tests/performance/merge_tree_simple_select.xml +++ b/dbms/tests/performance/merge_tree_simple_select.xml @@ -1,6 +1,5 @@ - loop diff --git a/dbms/tests/performance/mingroupby-orderbylimit1.xml b/dbms/tests/performance/mingroupby-orderbylimit1.xml index ec69ffa2b8f..34cd992558b 100644 --- a/dbms/tests/performance/mingroupby-orderbylimit1.xml +++ b/dbms/tests/performance/mingroupby-orderbylimit1.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/modulo.xml b/dbms/tests/performance/modulo.xml index 8e6674d0980..e31de5c1701 100644 --- a/dbms/tests/performance/modulo.xml +++ b/dbms/tests/performance/modulo.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/ngram_distance.xml b/dbms/tests/performance/ngram_distance.xml index 78da4d55d0e..e90f49155b1 100644 --- a/dbms/tests/performance/ngram_distance.xml +++ b/dbms/tests/performance/ngram_distance.xml @@ -8,7 +8,6 @@ hits_10m_single - loop 20000000000 diff --git a/dbms/tests/performance/number_formatting_formats.xml b/dbms/tests/performance/number_formatting_formats.xml index aa9929464fb..c2a9a9c081d 100644 --- a/dbms/tests/performance/number_formatting_formats.xml +++ b/dbms/tests/performance/number_formatting_formats.xml @@ -1,5 +1,4 @@ - loop CREATE TABLE IF NOT EXISTS table_{format} (x UInt64) ENGINE = File(`{format}`) diff --git a/dbms/tests/performance/nyc_taxi.xml b/dbms/tests/performance/nyc_taxi.xml index 7648e377433..92a1dd59441 100644 --- a/dbms/tests/performance/nyc_taxi.xml +++ b/dbms/tests/performance/nyc_taxi.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/order_by_decimals.xml b/dbms/tests/performance/order_by_decimals.xml index c6a7e7f72df..5479181fb08 100644 --- a/dbms/tests/performance/order_by_decimals.xml +++ b/dbms/tests/performance/order_by_decimals.xml @@ -4,7 +4,6 @@ comparison - loop diff --git a/dbms/tests/performance/order_by_read_in_order.xml b/dbms/tests/performance/order_by_read_in_order.xml index a99dd89846e..e37e4df4681 100644 --- a/dbms/tests/performance/order_by_read_in_order.xml +++ b/dbms/tests/performance/order_by_read_in_order.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/order_by_single_column.xml b/dbms/tests/performance/order_by_single_column.xml index ed247641ca8..148b14e8959 100644 --- a/dbms/tests/performance/order_by_single_column.xml +++ b/dbms/tests/performance/order_by_single_column.xml @@ -8,7 +8,6 @@ hits_100m_single - loop diff --git a/dbms/tests/performance/parallel_insert.xml b/dbms/tests/performance/parallel_insert.xml index 44a2964f881..6da1a2cc020 100644 --- a/dbms/tests/performance/parallel_insert.xml +++ b/dbms/tests/performance/parallel_insert.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/parse_engine_file.xml b/dbms/tests/performance/parse_engine_file.xml index 8a0054bdd7f..fb10fa97915 100644 --- a/dbms/tests/performance/parse_engine_file.xml +++ b/dbms/tests/performance/parse_engine_file.xml @@ -1,5 +1,4 @@ - loop CREATE TABLE IF NOT EXISTS table_{format} ENGINE = File({format}) AS test.hits diff --git a/dbms/tests/performance/prewhere.xml b/dbms/tests/performance/prewhere.xml index e4c9cc749ff..e3350d765ee 100644 --- a/dbms/tests/performance/prewhere.xml +++ b/dbms/tests/performance/prewhere.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/random_printable_ascii.xml b/dbms/tests/performance/random_printable_ascii.xml index 5fca705464e..320ffeac796 100644 --- a/dbms/tests/performance/random_printable_ascii.xml +++ b/dbms/tests/performance/random_printable_ascii.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/range.xml b/dbms/tests/performance/range.xml index ee61a22b0cf..95b8455057e 100644 --- a/dbms/tests/performance/range.xml +++ b/dbms/tests/performance/range.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/read_hits_with_aio.xml b/dbms/tests/performance/read_hits_with_aio.xml index 5fa3f70ed86..850fd0fbadc 100644 --- a/dbms/tests/performance/read_hits_with_aio.xml +++ b/dbms/tests/performance/read_hits_with_aio.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/right.xml b/dbms/tests/performance/right.xml index 55095d251f7..73030e52f21 100644 --- a/dbms/tests/performance/right.xml +++ b/dbms/tests/performance/right.xml @@ -1,5 +1,4 @@ - loop hits_100m_single diff --git a/dbms/tests/performance/round_down.xml b/dbms/tests/performance/round_down.xml index 880b625af28..353f169ae8d 100644 --- a/dbms/tests/performance/round_down.xml +++ b/dbms/tests/performance/round_down.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/round_methods.xml b/dbms/tests/performance/round_methods.xml index 54bd1e4af17..0e560b2eae6 100644 --- a/dbms/tests/performance/round_methods.xml +++ b/dbms/tests/performance/round_methods.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/scalar.xml b/dbms/tests/performance/scalar.xml index d1bc661c58f..e8e487a80da 100644 --- a/dbms/tests/performance/scalar.xml +++ b/dbms/tests/performance/scalar.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/select_format.xml b/dbms/tests/performance/select_format.xml index 189b35a2700..b8df874304f 100644 --- a/dbms/tests/performance/select_format.xml +++ b/dbms/tests/performance/select_format.xml @@ -1,5 +1,4 @@ - loop CREATE TABLE IF NOT EXISTS table_{format} ENGINE = File({format}, '/dev/null') AS test.hits diff --git a/dbms/tests/performance/set.xml b/dbms/tests/performance/set.xml index 75b87d38abe..8d50dbbce23 100644 --- a/dbms/tests/performance/set.xml +++ b/dbms/tests/performance/set.xml @@ -1,5 +1,4 @@ - loop long diff --git a/dbms/tests/performance/set_hits.xml b/dbms/tests/performance/set_hits.xml index a5e61625604..09860aa1cd7 100644 --- a/dbms/tests/performance/set_hits.xml +++ b/dbms/tests/performance/set_hits.xml @@ -1,5 +1,4 @@ - loop hits_10m_single diff --git a/dbms/tests/performance/set_index.xml b/dbms/tests/performance/set_index.xml index 4e24b7ccd79..1d1b2460e85 100644 --- a/dbms/tests/performance/set_index.xml +++ b/dbms/tests/performance/set_index.xml @@ -1,5 +1,4 @@ - loop CREATE TABLE test_in (`a` UInt32) ENGINE = MergeTree() ORDER BY a INSERT INTO test_in SELECT number FROM numbers(500000000) diff --git a/dbms/tests/performance/simple_join_query.xml b/dbms/tests/performance/simple_join_query.xml index 8ef3d97460d..8f62ffdfd00 100644 --- a/dbms/tests/performance/simple_join_query.xml +++ b/dbms/tests/performance/simple_join_query.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/slices_hits.xml b/dbms/tests/performance/slices_hits.xml index ad01a607b8a..1745df3328c 100644 --- a/dbms/tests/performance/slices_hits.xml +++ b/dbms/tests/performance/slices_hits.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/sort.xml b/dbms/tests/performance/sort.xml index 3aa7ed5788f..da7f2fd5410 100644 --- a/dbms/tests/performance/sort.xml +++ b/dbms/tests/performance/sort.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/string_join.xml b/dbms/tests/performance/string_join.xml index 228fe3182b8..3988845641c 100644 --- a/dbms/tests/performance/string_join.xml +++ b/dbms/tests/performance/string_join.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/string_set.xml b/dbms/tests/performance/string_set.xml index cf6261d6d60..95612fb2d34 100644 --- a/dbms/tests/performance/string_set.xml +++ b/dbms/tests/performance/string_set.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/string_sort.xml b/dbms/tests/performance/string_sort.xml index b72b073172a..6a4e68270f9 100644 --- a/dbms/tests/performance/string_sort.xml +++ b/dbms/tests/performance/string_sort.xml @@ -4,7 +4,6 @@ hits_100m_single - loop diff --git a/dbms/tests/performance/sum_map.xml b/dbms/tests/performance/sum_map.xml index ac1ccaae0fe..a88983fdbea 100644 --- a/dbms/tests/performance/sum_map.xml +++ b/dbms/tests/performance/sum_map.xml @@ -1,5 +1,4 @@ - loop diff --git a/dbms/tests/performance/synthetic_hardware_benchmark.xml b/dbms/tests/performance/synthetic_hardware_benchmark.xml index 94d8ef76fa1..055f8f67ee5 100644 --- a/dbms/tests/performance/synthetic_hardware_benchmark.xml +++ b/dbms/tests/performance/synthetic_hardware_benchmark.xml @@ -1,5 +1,4 @@ - once diff --git a/dbms/tests/performance/trim_numbers.xml b/dbms/tests/performance/trim_numbers.xml index 997272e95f6..62e26f8245a 100644 --- a/dbms/tests/performance/trim_numbers.xml +++ b/dbms/tests/performance/trim_numbers.xml @@ -1,12 +1,10 @@ - loop 10000 - 5000 20000 diff --git a/dbms/tests/performance/trim_urls.xml b/dbms/tests/performance/trim_urls.xml index 23dd3f77f6e..f29d878682f 100644 --- a/dbms/tests/performance/trim_urls.xml +++ b/dbms/tests/performance/trim_urls.xml @@ -1,5 +1,4 @@ - loop hits_100m_single @@ -10,7 +9,6 @@ 10000 - 5000 20000 diff --git a/dbms/tests/performance/trim_whitespace.xml b/dbms/tests/performance/trim_whitespace.xml index 2038d8f5647..8ec4aeaa54e 100644 --- a/dbms/tests/performance/trim_whitespace.xml +++ b/dbms/tests/performance/trim_whitespace.xml @@ -1,5 +1,4 @@ - loop create table if not exists whitespaces diff --git a/dbms/tests/performance/uniq.xml b/dbms/tests/performance/uniq.xml index 9de2ecdf72b..0b7c8e58c86 100644 --- a/dbms/tests/performance/uniq.xml +++ b/dbms/tests/performance/uniq.xml @@ -1,5 +1,4 @@ - loop hits_100m_single diff --git a/dbms/tests/performance/url_hits.xml b/dbms/tests/performance/url_hits.xml index d4e504cd1b8..f9383eb3910 100644 --- a/dbms/tests/performance/url_hits.xml +++ b/dbms/tests/performance/url_hits.xml @@ -1,5 +1,4 @@ - loop hits_100m_single @@ -10,7 +9,6 @@ 10000 - 5000 20000 diff --git a/dbms/tests/performance/vectorize_aggregation_combinators.xml b/dbms/tests/performance/vectorize_aggregation_combinators.xml index 73024f454f9..49af4ae0f07 100644 --- a/dbms/tests/performance/vectorize_aggregation_combinators.xml +++ b/dbms/tests/performance/vectorize_aggregation_combinators.xml @@ -1,13 +1,11 @@ - loop 30000 - 6000 60000 diff --git a/dbms/tests/performance/visit_param_extract_raw.xml b/dbms/tests/performance/visit_param_extract_raw.xml index 0faa43088e7..5db6b11a5e0 100644 --- a/dbms/tests/performance/visit_param_extract_raw.xml +++ b/dbms/tests/performance/visit_param_extract_raw.xml @@ -1,14 +1,10 @@ - once - - 4000 10000 - param @@ -20,5 +16,5 @@ - SELECT count() FROM system.numbers WHERE NOT ignore(visitParamExtractRaw(materialize({param}), 'myparam')) + SELECT count() FROM numbers(2000000) WHERE NOT ignore(visitParamExtractRaw(materialize({param}), 'myparam')) diff --git a/dbms/tests/performance/website.xml b/dbms/tests/performance/website.xml index 83a1c3607c7..ef97d118b85 100644 --- a/dbms/tests/performance/website.xml +++ b/dbms/tests/performance/website.xml @@ -1,5 +1,4 @@ - loop hits_10m_single From edc9f9886c284a93477f72f5ff941b35d4ffbadf Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 7 Mar 2020 22:52:07 +0300 Subject: [PATCH 113/712] Remove unused features from performance test --- .../performance-test/PerformanceTest.cpp | 21 ++---- .../performance-test/PerformanceTestInfo.cpp | 17 ----- .../performance-test/PerformanceTestInfo.h | 7 -- .../performance-test/ReportBuilder.cpp | 73 +++++++------------ .../performance-test/StopConditionsSet.cpp | 3 - .../performance-test/StopConditionsSet.h | 1 - dbms/programs/performance-test/TestStats.cpp | 63 ---------------- dbms/programs/performance-test/TestStats.h | 17 ----- .../performance-test/TestStopConditions.h | 2 - .../performance-test/executeQuery.cpp | 2 - 10 files changed, 36 insertions(+), 170 deletions(-) diff --git a/dbms/programs/performance-test/PerformanceTest.cpp b/dbms/programs/performance-test/PerformanceTest.cpp index e2c5c0d8741..a2e0aa933b8 100644 --- a/dbms/programs/performance-test/PerformanceTest.cpp +++ b/dbms/programs/performance-test/PerformanceTest.cpp @@ -305,22 +305,17 @@ void PerformanceTest::runQueries( statistics.startWatches(); try { - executeQuery(connection, query, statistics, stop_conditions, interrupt_listener, context, test_info.settings); - - if (test_info.exec_type == ExecutionType::Loop) + LOG_INFO(log, "Will run query in loop"); + for (size_t iteration = 0; !statistics.got_SIGINT; ++iteration) { - LOG_INFO(log, "Will run query in loop"); - for (size_t iteration = 1; !statistics.got_SIGINT; ++iteration) + stop_conditions.reportIterations(iteration); + if (stop_conditions.areFulfilled()) { - stop_conditions.reportIterations(iteration); - if (stop_conditions.areFulfilled()) - { - LOG_INFO(log, "Stop conditions fulfilled"); - break; - } - - executeQuery(connection, query, statistics, stop_conditions, interrupt_listener, context, test_info.settings); + LOG_INFO(log, "Stop conditions fulfilled"); + break; } + + executeQuery(connection, query, statistics, stop_conditions, interrupt_listener, context, test_info.settings); } } catch (const Exception & e) diff --git a/dbms/programs/performance-test/PerformanceTestInfo.cpp b/dbms/programs/performance-test/PerformanceTestInfo.cpp index b0f877abfc7..ef48ffae9df 100644 --- a/dbms/programs/performance-test/PerformanceTestInfo.cpp +++ b/dbms/programs/performance-test/PerformanceTestInfo.cpp @@ -54,7 +54,6 @@ PerformanceTestInfo::PerformanceTestInfo( extractQueries(config); extractAuxiliaryQueries(config); processSubstitutions(config); - getExecutionType(config); getStopConditions(config); } @@ -141,22 +140,6 @@ void PerformanceTestInfo::processSubstitutions(XMLConfigurationPtr config) } } -void PerformanceTestInfo::getExecutionType(XMLConfigurationPtr config) -{ - if (!config->has("type")) - throw Exception("Missing type property in config: " + test_name, - ErrorCodes::BAD_ARGUMENTS); - - std::string config_exec_type = config->getString("type"); - if (config_exec_type == "loop") - exec_type = ExecutionType::Loop; - else if (config_exec_type == "once") - exec_type = ExecutionType::Once; - else - throw Exception("Unknown type " + config_exec_type + " in :" + test_name, - ErrorCodes::BAD_ARGUMENTS); -} - void PerformanceTestInfo::getStopConditions(XMLConfigurationPtr config) { diff --git a/dbms/programs/performance-test/PerformanceTestInfo.h b/dbms/programs/performance-test/PerformanceTestInfo.h index 8e6b1c5f43a..d40f5d3f19f 100644 --- a/dbms/programs/performance-test/PerformanceTestInfo.h +++ b/dbms/programs/performance-test/PerformanceTestInfo.h @@ -12,11 +12,6 @@ namespace DB { -enum class ExecutionType -{ - Loop, - Once -}; using XMLConfiguration = Poco::Util::XMLConfiguration; using XMLConfigurationPtr = Poco::AutoPtr; @@ -34,7 +29,6 @@ public: Strings queries; Settings settings; - ExecutionType exec_type; StringToVector substitutions; size_t times_to_run; @@ -47,7 +41,6 @@ private: void applySettings(XMLConfigurationPtr config); void extractQueries(XMLConfigurationPtr config); void processSubstitutions(XMLConfigurationPtr config); - void getExecutionType(XMLConfigurationPtr config); void getStopConditions(XMLConfigurationPtr config); void extractAuxiliaryQueries(XMLConfigurationPtr config); }; diff --git a/dbms/programs/performance-test/ReportBuilder.cpp b/dbms/programs/performance-test/ReportBuilder.cpp index c95b4d56a1e..ef4417f1713 100644 --- a/dbms/programs/performance-test/ReportBuilder.cpp +++ b/dbms/programs/performance-test/ReportBuilder.cpp @@ -17,13 +17,6 @@ namespace DB namespace { -std::string getMainMetric(const PerformanceTestInfo & test_info) -{ - if (test_info.exec_type == ExecutionType::Loop) - return "min_time"; - else - return "rows_per_second"; -} bool isASCIIString(const std::string & str) { @@ -120,50 +113,40 @@ std::string ReportBuilder::buildFullReport( runJSON.set("exception", "Some exception occurred with non ASCII message. This may produce invalid JSON. Try reproduce locally."); } - if (test_info.exec_type == ExecutionType::Loop) + /// in seconds + runJSON.set("min_time", statistics.min_time / double(1000)); + + if (statistics.sampler.size() != 0) { - /// in seconds - runJSON.set("min_time", statistics.min_time / double(1000)); - - if (statistics.sampler.size() != 0) + JSONString quantiles(4); /// here, 4 is the size of \t padding + for (double percent = 10; percent <= 90; percent += 10) { - JSONString quantiles(4); /// here, 4 is the size of \t padding - for (double percent = 10; percent <= 90; percent += 10) - { - std::string quantile_key = std::to_string(percent / 100.0); - while (quantile_key.back() == '0') - quantile_key.pop_back(); + std::string quantile_key = std::to_string(percent / 100.0); + while (quantile_key.back() == '0') + quantile_key.pop_back(); - quantiles.set(quantile_key, - statistics.sampler.quantileInterpolated(percent / 100.0)); - } - quantiles.set("0.95", - statistics.sampler.quantileInterpolated(95 / 100.0)); - quantiles.set("0.99", - statistics.sampler.quantileInterpolated(99 / 100.0)); - quantiles.set("0.999", - statistics.sampler.quantileInterpolated(99.9 / 100.0)); - quantiles.set("0.9999", - statistics.sampler.quantileInterpolated(99.99 / 100.0)); - - runJSON.set("quantiles", quantiles.asString()); + quantiles.set(quantile_key, + statistics.sampler.quantileInterpolated(percent / 100.0)); } + quantiles.set("0.95", + statistics.sampler.quantileInterpolated(95 / 100.0)); + quantiles.set("0.99", + statistics.sampler.quantileInterpolated(99 / 100.0)); + quantiles.set("0.999", + statistics.sampler.quantileInterpolated(99.9 / 100.0)); + quantiles.set("0.9999", + statistics.sampler.quantileInterpolated(99.99 / 100.0)); - runJSON.set("total_time", statistics.total_time); - - if (statistics.total_time != 0) - { - runJSON.set("queries_per_second", static_cast(statistics.queries) / statistics.total_time); - runJSON.set("rows_per_second", static_cast(statistics.total_rows_read) / statistics.total_time); - runJSON.set("bytes_per_second", static_cast(statistics.total_bytes_read) / statistics.total_time); - } + runJSON.set("quantiles", quantiles.asString()); } - else + + runJSON.set("total_time", statistics.total_time); + + if (statistics.total_time != 0) { - runJSON.set("max_rows_per_second", statistics.max_rows_speed); - runJSON.set("max_bytes_per_second", statistics.max_bytes_speed); - runJSON.set("avg_rows_per_second", statistics.avg_rows_speed_value); - runJSON.set("avg_bytes_per_second", statistics.avg_bytes_speed_value); + runJSON.set("queries_per_second", static_cast(statistics.queries) / statistics.total_time); + runJSON.set("rows_per_second", static_cast(statistics.total_rows_read) / statistics.total_time); + runJSON.set("bytes_per_second", static_cast(statistics.total_bytes_read) / statistics.total_time); } runJSON.set("memory_usage", statistics.memory_usage); @@ -197,7 +180,7 @@ std::string ReportBuilder::buildCompactReport( output << "run " << std::to_string(number_of_launch + 1) << ": "; - std::string main_metric = getMainMetric(test_info); + std::string main_metric = "min_time"; output << main_metric << " = "; size_t index = number_of_launch * test_info.queries.size() + query_index; diff --git a/dbms/programs/performance-test/StopConditionsSet.cpp b/dbms/programs/performance-test/StopConditionsSet.cpp index ab334b71244..9d0df07631b 100644 --- a/dbms/programs/performance-test/StopConditionsSet.cpp +++ b/dbms/programs/performance-test/StopConditionsSet.cpp @@ -28,8 +28,6 @@ void StopConditionsSet::loadFromConfig(const ConfigurationPtr & stop_conditions_ min_time_not_changing_for_ms.value = stop_conditions_view->getUInt64(key); else if (key == "max_speed_not_changing_for_ms") max_speed_not_changing_for_ms.value = stop_conditions_view->getUInt64(key); - else if (key == "average_speed_not_changing_for_ms") - average_speed_not_changing_for_ms.value = stop_conditions_view->getUInt64(key); else throw Exception("Met unknown stop condition: " + key, ErrorCodes::LOGICAL_ERROR); @@ -45,7 +43,6 @@ void StopConditionsSet::reset() iterations.fulfilled = false; min_time_not_changing_for_ms.fulfilled = false; max_speed_not_changing_for_ms.fulfilled = false; - average_speed_not_changing_for_ms.fulfilled = false; fulfilled_count = 0; } diff --git a/dbms/programs/performance-test/StopConditionsSet.h b/dbms/programs/performance-test/StopConditionsSet.h index ad29c748a76..db34c9e4b51 100644 --- a/dbms/programs/performance-test/StopConditionsSet.h +++ b/dbms/programs/performance-test/StopConditionsSet.h @@ -30,7 +30,6 @@ struct StopConditionsSet StopCondition iterations; StopCondition min_time_not_changing_for_ms; StopCondition max_speed_not_changing_for_ms; - StopCondition average_speed_not_changing_for_ms; size_t initialized_count = 0; size_t fulfilled_count = 0; diff --git a/dbms/programs/performance-test/TestStats.cpp b/dbms/programs/performance-test/TestStats.cpp index 4a3ec281d90..5268f8bb328 100644 --- a/dbms/programs/performance-test/TestStats.cpp +++ b/dbms/programs/performance-test/TestStats.cpp @@ -67,41 +67,6 @@ void TestStats::update_min_time(UInt64 min_time_candidate) } } -void TestStats::update_max_speed( - size_t max_speed_candidate, - Stopwatch & max_speed_watch, - UInt64 & max_speed) -{ - if (max_speed_candidate > max_speed) - { - max_speed = max_speed_candidate; - max_speed_watch.restart(); - } -} - - -void TestStats::update_average_speed( - double new_speed_info, - Stopwatch & avg_speed_watch, - size_t & number_of_info_batches, - double precision, - double & avg_speed_first, - double & avg_speed_value) -{ - avg_speed_value = ((avg_speed_value * number_of_info_batches) + new_speed_info); - ++number_of_info_batches; - avg_speed_value /= number_of_info_batches; - - if (avg_speed_first == 0) - avg_speed_first = avg_speed_value; - - auto [min, max] = std::minmax(avg_speed_value, avg_speed_first); - if (1 - min / max >= precision) - { - avg_speed_first = avg_speed_value; - avg_speed_watch.restart(); - } -} void TestStats::add(size_t rows_read_inc, size_t bytes_read_inc) { @@ -109,26 +74,6 @@ void TestStats::add(size_t rows_read_inc, size_t bytes_read_inc) total_bytes_read += bytes_read_inc; last_query_rows_read += rows_read_inc; last_query_bytes_read += bytes_read_inc; - - double new_rows_speed = last_query_rows_read / watch_per_query.elapsedSeconds(); - double new_bytes_speed = last_query_bytes_read / watch_per_query.elapsedSeconds(); - - /// Update rows speed - update_max_speed(new_rows_speed, max_rows_speed_watch, max_rows_speed); - update_average_speed(new_rows_speed, - avg_rows_speed_watch, - number_of_rows_speed_info_batches, - avg_rows_speed_precision, - avg_rows_speed_first, - avg_rows_speed_value); - /// Update bytes speed - update_max_speed(new_bytes_speed, max_bytes_speed_watch, max_bytes_speed); - update_average_speed(new_bytes_speed, - avg_bytes_speed_watch, - number_of_bytes_speed_info_batches, - avg_bytes_speed_precision, - avg_bytes_speed_first, - avg_bytes_speed_value); } void TestStats::updateQueryInfo() @@ -144,10 +89,6 @@ TestStats::TestStats() watch.reset(); watch_per_query.reset(); min_time_watch.reset(); - max_rows_speed_watch.reset(); - max_bytes_speed_watch.reset(); - avg_rows_speed_watch.reset(); - avg_bytes_speed_watch.reset(); } @@ -156,10 +97,6 @@ void TestStats::startWatches() watch.start(); watch_per_query.start(); min_time_watch.start(); - max_rows_speed_watch.start(); - max_bytes_speed_watch.start(); - avg_rows_speed_watch.start(); - avg_bytes_speed_watch.start(); } } diff --git a/dbms/programs/performance-test/TestStats.h b/dbms/programs/performance-test/TestStats.h index b38ffa7386a..c88e50727c4 100644 --- a/dbms/programs/performance-test/TestStats.h +++ b/dbms/programs/performance-test/TestStats.h @@ -13,10 +13,6 @@ struct TestStats Stopwatch watch; Stopwatch watch_per_query; Stopwatch min_time_watch; - Stopwatch max_rows_speed_watch; - Stopwatch max_bytes_speed_watch; - Stopwatch avg_rows_speed_watch; - Stopwatch avg_bytes_speed_watch; bool last_query_was_cancelled = false; std::string query_id; @@ -62,19 +58,6 @@ struct TestStats void update_min_time(UInt64 min_time_candidate); - void update_average_speed( - double new_speed_info, - Stopwatch & avg_speed_watch, - size_t & number_of_info_batches, - double precision, - double & avg_speed_first, - double & avg_speed_value); - - void update_max_speed( - size_t max_speed_candidate, - Stopwatch & max_speed_watch, - UInt64 & max_speed); - void add(size_t rows_read_inc, size_t bytes_read_inc); void updateQueryInfo(); diff --git a/dbms/programs/performance-test/TestStopConditions.h b/dbms/programs/performance-test/TestStopConditions.h index 2dcbcce4674..0bdfa094641 100644 --- a/dbms/programs/performance-test/TestStopConditions.h +++ b/dbms/programs/performance-test/TestStopConditions.h @@ -32,8 +32,6 @@ public: DEFINE_REPORT_FUNC(reportBytesReadUncompressed, bytes_read_uncompressed) DEFINE_REPORT_FUNC(reportIterations, iterations) DEFINE_REPORT_FUNC(reportMinTimeNotChangingFor, min_time_not_changing_for_ms) - DEFINE_REPORT_FUNC(reportMaxSpeedNotChangingFor, max_speed_not_changing_for_ms) - DEFINE_REPORT_FUNC(reportAverageSpeedNotChangingFor, average_speed_not_changing_for_ms) #undef REPORT diff --git a/dbms/programs/performance-test/executeQuery.cpp b/dbms/programs/performance-test/executeQuery.cpp index db82a48d0c1..6f0f4cbe3c5 100644 --- a/dbms/programs/performance-test/executeQuery.cpp +++ b/dbms/programs/performance-test/executeQuery.cpp @@ -21,8 +21,6 @@ void checkFulfilledConditionsAndUpdate( stop_conditions.reportBytesReadUncompressed(statistics.total_bytes_read); stop_conditions.reportTotalTime(statistics.watch.elapsed() / (1000 * 1000)); stop_conditions.reportMinTimeNotChangingFor(statistics.min_time_watch.elapsed() / (1000 * 1000)); - stop_conditions.reportMaxSpeedNotChangingFor(statistics.max_rows_speed_watch.elapsed() / (1000 * 1000)); - stop_conditions.reportAverageSpeedNotChangingFor(statistics.avg_rows_speed_watch.elapsed() / (1000 * 1000)); if (stop_conditions.areFulfilled()) { From 9a10457cb6bcec6f8c11ecfb3e216fb99df78038 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sat, 7 Mar 2020 22:56:38 +0300 Subject: [PATCH 114/712] Remove `indexHint` function (#9542) * Remove indexHint function * Fixed build * Update KeyCondition.cpp * Update KeyCondition.cpp --- dbms/src/Functions/indexHint.cpp | 63 ----------- .../registerFunctionsMiscellaneous.cpp | 2 - dbms/src/Interpreters/ActionsVisitor.cpp | 10 -- .../RequiredSourceColumnsVisitor.cpp | 3 +- dbms/src/Storages/MergeTree/KeyCondition.cpp | 12 +- .../Storages/MergeTree/MergeTreeIndexSet.cpp | 4 +- .../MergeTree/MergeTreeWhereOptimizer.cpp | 4 - dbms/src/Storages/MergeTree/RPNBuilder.h | 5 +- .../functions/other_functions.md | 105 +----------------- .../functions/other_functions.md | 105 ------------------ .../functions/other_functions.md | 96 ---------------- 11 files changed, 9 insertions(+), 400 deletions(-) delete mode 100644 dbms/src/Functions/indexHint.cpp diff --git a/dbms/src/Functions/indexHint.cpp b/dbms/src/Functions/indexHint.cpp deleted file mode 100644 index 0da2398ebd9..00000000000 --- a/dbms/src/Functions/indexHint.cpp +++ /dev/null @@ -1,63 +0,0 @@ -#include -#include -#include - - -namespace DB -{ - - -/** The `indexHint` function takes any number of any arguments and always returns one. - * - * This function has a special meaning (see ExpressionAnalyzer, KeyCondition) - * - the expressions inside it are not evaluated; - * - but when analyzing the index (selecting ranges for reading), this function is treated the same way, - * as if instead of using it the expression itself would be. - * - * Example: WHERE something AND indexHint(CounterID = 34) - * - do not read or calculate CounterID = 34, but select ranges in which the CounterID = 34 expression can be true. - * - * The function can be used for debugging purposes, as well as for (hidden from the user) query conversions. - */ -class FunctionIndexHint : public IFunction -{ -public: - static constexpr auto name = "indexHint"; - static FunctionPtr create(const Context &) - { - return std::make_shared(); - } - - bool isVariadic() const override - { - return true; - } - size_t getNumberOfArguments() const override - { - return 0; - } - - bool useDefaultImplementationForNulls() const override { return false; } - - String getName() const override - { - return name; - } - DataTypePtr getReturnTypeImpl(const DataTypes & /*arguments*/) const override - { - return std::make_shared(); - } - - void executeImpl(Block & block, const ColumnNumbers &, size_t result, size_t input_rows_count) override - { - block.getByPosition(result).column = DataTypeUInt8().createColumnConst(input_rows_count, 1u); - } -}; - - -void registerFunctionIndexHint(FunctionFactory & factory) -{ - factory.registerFunction(); -} - -} diff --git a/dbms/src/Functions/registerFunctionsMiscellaneous.cpp b/dbms/src/Functions/registerFunctionsMiscellaneous.cpp index 6b8b030d2c3..44e26542c7d 100644 --- a/dbms/src/Functions/registerFunctionsMiscellaneous.cpp +++ b/dbms/src/Functions/registerFunctionsMiscellaneous.cpp @@ -28,7 +28,6 @@ void registerFunctionSleepEachRow(FunctionFactory &); void registerFunctionMaterialize(FunctionFactory &); void registerFunctionIgnore(FunctionFactory &); void registerFunctionIgnoreExceptNull(FunctionFactory &); -void registerFunctionIndexHint(FunctionFactory &); void registerFunctionIdentity(FunctionFactory &); void registerFunctionArrayJoin(FunctionFactory &); void registerFunctionReplicate(FunctionFactory &); @@ -87,7 +86,6 @@ void registerFunctionsMiscellaneous(FunctionFactory & factory) registerFunctionMaterialize(factory); registerFunctionIgnore(factory); registerFunctionIgnoreExceptNull(factory); - registerFunctionIndexHint(factory); registerFunctionIdentity(factory); registerFunctionArrayJoin(factory); registerFunctionReplicate(factory); diff --git a/dbms/src/Interpreters/ActionsVisitor.cpp b/dbms/src/Interpreters/ActionsVisitor.cpp index ea13bb57b14..3a20ae6ce24 100644 --- a/dbms/src/Interpreters/ActionsVisitor.cpp +++ b/dbms/src/Interpreters/ActionsVisitor.cpp @@ -400,16 +400,6 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & } } - /// A special function `indexHint`. Everything that is inside it is not calculated - /// (and is used only for index analysis, see KeyCondition). - if (node.name == "indexHint") - { - data.addAction(ExpressionAction::addColumn(ColumnWithTypeAndName( - ColumnConst::create(ColumnUInt8::create(1, 1), 1), std::make_shared(), - column_name.get(ast)))); - return; - } - if (AggregateFunctionFactory::instance().isAggregateFunctionName(node.name)) return; diff --git a/dbms/src/Interpreters/RequiredSourceColumnsVisitor.cpp b/dbms/src/Interpreters/RequiredSourceColumnsVisitor.cpp index b5f9c83db50..5a740805560 100644 --- a/dbms/src/Interpreters/RequiredSourceColumnsVisitor.cpp +++ b/dbms/src/Interpreters/RequiredSourceColumnsVisitor.cpp @@ -51,9 +51,8 @@ bool RequiredSourceColumnsMatcher::needChildVisit(const ASTPtr & node, const AST if (const auto * f = node->as()) { - /// "indexHint" is a special function for index analysis. Everything that is inside it is not calculated. @sa KeyCondition /// "lambda" visit children itself. - if (f->name == "indexHint" || f->name == "lambda") + if (f->name == "lambda") return false; } diff --git a/dbms/src/Storages/MergeTree/KeyCondition.cpp b/dbms/src/Storages/MergeTree/KeyCondition.cpp index 0bd2389e941..a936ead568d 100644 --- a/dbms/src/Storages/MergeTree/KeyCondition.cpp +++ b/dbms/src/Storages/MergeTree/KeyCondition.cpp @@ -281,11 +281,11 @@ static const std::map inverse_relations = { bool isLogicalOperator(const String & func_name) { - return (func_name == "and" || func_name == "or" || func_name == "not" || func_name == "indexHint"); + return (func_name == "and" || func_name == "or" || func_name == "not"); } /// The node can be one of: -/// - Logical operator (AND, OR, NOT and indexHint() - logical NOOP) +/// - Logical operator (AND, OR, NOT) /// - An "atom" (relational operator, constant, expression) /// - A logical constant expression /// - Any other function @@ -302,8 +302,7 @@ ASTPtr cloneASTWithInversionPushDown(const ASTPtr node, const bool need_inversio const auto result_node = makeASTFunction(func->name); - /// indexHint() is a special case - logical NOOP function - if (result_node->name != "indexHint" && need_inversion) + if (need_inversion) { result_node->name = (result_node->name == "and") ? "or" : "and"; } @@ -887,9 +886,6 @@ bool KeyCondition::tryParseAtomFromAST(const ASTPtr & node, const Context & cont bool KeyCondition::tryParseLogicalOperatorFromAST(const ASTFunction * func, RPNElement & out) { /// Functions AND, OR, NOT. - /** Also a special function `indexHint` - works as if instead of calling a function there are just parentheses - * (or, the same thing - calling the function `and` from one argument). - */ const ASTs & args = func->arguments->children; if (func->name == "not") @@ -901,7 +897,7 @@ bool KeyCondition::tryParseLogicalOperatorFromAST(const ASTFunction * func, RPNE } else { - if (func->name == "and" || func->name == "indexHint") + if (func->name == "and") out.function = RPNElement::FUNCTION_AND; else if (func->name == "or") out.function = RPNElement::FUNCTION_OR; diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexSet.cpp b/dbms/src/Storages/MergeTree/MergeTreeIndexSet.cpp index a379463210b..946b4d80c99 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexSet.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexSet.cpp @@ -363,7 +363,7 @@ bool MergeTreeIndexConditionSet::operatorFromAST(ASTPtr & node) const func->name = "__bitSwapLastTwo"; } - else if (func->name == "and" || func->name == "indexHint") + else if (func->name == "and") { auto last_arg = args.back(); args.pop_back(); @@ -419,7 +419,7 @@ bool MergeTreeIndexConditionSet::checkASTUseless(const ASTPtr & node, bool atomi const ASTs & args = func->arguments->children; - if (func->name == "and" || func->name == "indexHint") + if (func->name == "and") return checkASTUseless(args[0], atomic) && checkASTUseless(args[1], atomic); else if (func->name == "or") return checkASTUseless(args[0], atomic) || checkASTUseless(args[1], atomic); diff --git a/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp b/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp index 311a09f3461..e18c9197217 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp @@ -332,10 +332,6 @@ bool MergeTreeWhereOptimizer::cannotBeMoved(const ASTPtr & ptr) const if ("globalIn" == function_ptr->name || "globalNotIn" == function_ptr->name) return true; - - /// indexHint is a special function that it does not make sense to transfer to PREWHERE - if ("indexHint" == function_ptr->name) - return true; } else if (auto opt_name = IdentifierSemantic::getColumnName(ptr)) { diff --git a/dbms/src/Storages/MergeTree/RPNBuilder.h b/dbms/src/Storages/MergeTree/RPNBuilder.h index 234f7b06ffe..2e457147cf4 100644 --- a/dbms/src/Storages/MergeTree/RPNBuilder.h +++ b/dbms/src/Storages/MergeTree/RPNBuilder.h @@ -91,9 +91,6 @@ private: bool operatorFromAST(const ASTFunction * func, RPNElement & out) { /// Functions AND, OR, NOT. - /** Also a special function `indexHint` - works as if instead of calling a function there are just parentheses - * (or, the same thing - calling the function `and` from one argument). - */ const ASTs & args = typeid_cast(*func->arguments).children; if (func->name == "not") @@ -105,7 +102,7 @@ private: } else { - if (func->name == "and" || func->name == "indexHint") + if (func->name == "and") out.function = RPNElement::FUNCTION_AND; else if (func->name == "or") out.function = RPNElement::FUNCTION_OR; diff --git a/docs/en/query_language/functions/other_functions.md b/docs/en/query_language/functions/other_functions.md index b3066f88685..e851cf804d8 100644 --- a/docs/en/query_language/functions/other_functions.md +++ b/docs/en/query_language/functions/other_functions.md @@ -734,109 +734,6 @@ SELECT defaultValueOfArgumentType( CAST(1 AS Nullable(Int8) ) ) └───────────────────────────────────────────────────────┘ ``` -## indexHint {#indexhint} - -The function is intended for debugging and introspection purposes. The function ignores it's argument and always returns 1. Arguments are not even evaluated. - -But for the purpose of index analysis, the argument of this function is analyzed as if it was present directly without being wrapped inside `indexHint` function. This allows to select data in index ranges by the corresponding condition but without further filtering by this condition. The index in ClickHouse is sparse and using `indexHint` will yield more data than specifying the same condition directly. - -**Syntax** - -```sql -SELECT * FROM table WHERE indexHint() -``` - -**Returned value** - -1. Type: [Uint8](https://clickhouse.yandex/docs/en/data_types/int_uint/#diapazony-uint). - -**Example** - -Here is the example of test data from the table [ontime](../../getting_started/example_datasets/ontime.md). - -Input table: - -```sql -SELECT count() FROM ontime -``` - -```text -┌─count()─┐ -│ 4276457 │ -└─────────┘ -``` - -The table has indexes on the fields `(FlightDate, (Year, FlightDate))`. - -Create a query, where the index is not used. - -Query: - -```sql -SELECT FlightDate AS k, count() FROM ontime GROUP BY k ORDER BY k -``` - -ClickHouse processed the entire table (`Processed 4.28 million rows`). - -Result: - -```text -┌──────────k─┬─count()─┐ -│ 2017-01-01 │ 13970 │ -│ 2017-01-02 │ 15882 │ -........................ -│ 2017-09-28 │ 16411 │ -│ 2017-09-29 │ 16384 │ -│ 2017-09-30 │ 12520 │ -└────────────┴─────────┘ -``` - -To apply the index, select a specific date. - -Query: - -```sql -SELECT FlightDate AS k, count() FROM ontime WHERE k = '2017-09-15' GROUP BY k ORDER BY k -``` - -By using the index, ClickHouse processed a significantly smaller number of rows (`Processed 32.74 thousand rows`). - -Result: - -```text -┌──────────k─┬─count()─┐ -│ 2017-09-15 │ 16428 │ -└────────────┴─────────┘ -``` - -Now wrap the expression `k = '2017-09-15'` into `indexHint` function. - -Query: - -```sql -SELECT - FlightDate AS k, - count() -FROM ontime -WHERE indexHint(k = '2017-09-15') -GROUP BY k -ORDER BY k ASC -``` - -ClickHouse used the index in the same way as the previous time (`Processed 32.74 thousand rows`). -The expression `k = '2017-09-15'` was not used when generating the result. -In examle the `indexHint` function allows to see adjacent dates. - -Result: - -```text -┌──────────k─┬─count()─┐ -│ 2017-09-14 │ 7071 │ -│ 2017-09-15 │ 16428 │ -│ 2017-09-16 │ 1077 │ -│ 2017-09-30 │ 8167 │ -└────────────┴─────────┘ -``` ## replicate {#other_functions-replicate} @@ -1005,7 +902,7 @@ joinGet(join_storage_table_name, `value_column`, join_keys) Returns list of values corresponded to list of keys. -If certain doesn't exist in source table then `0` or `null` will be returned based on [join_use_nulls](../../operations/settings/settings.md#join_use_nulls) setting. +If certain doesn't exist in source table then `0` or `null` will be returned based on [join_use_nulls](../../operations/settings/settings.md#join_use_nulls) setting. More info about `join_use_nulls` in [Join operation](../../operations/table_engines/join.md). diff --git a/docs/ru/query_language/functions/other_functions.md b/docs/ru/query_language/functions/other_functions.md index a988c14b56d..e85eaac6f99 100644 --- a/docs/ru/query_language/functions/other_functions.md +++ b/docs/ru/query_language/functions/other_functions.md @@ -685,111 +685,6 @@ SELECT defaultValueOfArgumentType( CAST(1 AS Nullable(Int8) ) ) └───────────────────────────────────────────────────────┘ ``` -## indexHint {#indexhint} - -Возвращает все данные из диапазона, в который попадают данные, соответствующие указанному выражению. -Переданное выражение не будет вычислено. Выбор диапазона производится по индексу. -Индекс в ClickHouse разреженный, при чтении диапазона в ответ попадают «лишние» соседние данные. - -**Синтаксис** - -```sql -SELECT * FROM table WHERE indexHint() -``` - -**Возвращаемое значение** - -Возвращает диапазон индекса, в котором выполняется заданное условие. - -Тип: [Uint8](https://clickhouse.yandex/docs/ru/data_types/int_uint/#diapazony-uint). - -**Пример** - -Рассмотрим пример с использованием тестовых данных таблицы [ontime](../../getting_started/example_datasets/ontime.md). - -Исходная таблица: - -```sql -SELECT count() FROM ontime -``` - -```text -┌─count()─┐ -│ 4276457 │ -└─────────┘ -``` - -В таблице есть индексы по полям `(FlightDate, (Year, FlightDate))`. - -Выполним выборку по дате, где индекс не используется. - -Запрос: - -```sql -SELECT FlightDate AS k, count() FROM ontime GROUP BY k ORDER BY k -``` - -ClickHouse обработал всю таблицу (`Processed 4.28 million rows`). - -Результат: - -```text -┌──────────k─┬─count()─┐ -│ 2017-01-01 │ 13970 │ -│ 2017-01-02 │ 15882 │ -........................ -│ 2017-09-28 │ 16411 │ -│ 2017-09-29 │ 16384 │ -│ 2017-09-30 │ 12520 │ -└────────────┴─────────┘ -``` - -Для подключения индекса выбираем конкретную дату. - -Запрос: - -```sql -SELECT FlightDate AS k, count() FROM ontime WHERE k = '2017-09-15' GROUP BY k ORDER BY k -``` - -При использовании индекса ClickHouse обработал значительно меньшее количество строк (`Processed 32.74 thousand rows`). - -Результат: - -```text -┌──────────k─┬─count()─┐ -│ 2017-09-15 │ 16428 │ -└────────────┴─────────┘ -``` - -Передадим в функцию `indexHint` выражение `k = '2017-09-15'`. - -Запрос: - -```sql -SELECT - FlightDate AS k, - count() -FROM ontime -WHERE indexHint(k = '2017-09-15') -GROUP BY k -ORDER BY k ASC -``` - -ClickHouse применил индекс по аналогии с примером выше (`Processed 32.74 thousand rows`). -Выражение `k = '2017-09-15'` не используется при формировании результата. -Функция `indexHint` позволяет увидеть соседние данные. - -Результат: - -```text -┌──────────k─┬─count()─┐ -│ 2017-09-14 │ 7071 │ -│ 2017-09-15 │ 16428 │ -│ 2017-09-16 │ 1077 │ -│ 2017-09-30 │ 8167 │ -└────────────┴─────────┘ -``` ## replicate {#other_functions-replicate} diff --git a/docs/zh/query_language/functions/other_functions.md b/docs/zh/query_language/functions/other_functions.md index a93079f4af3..613a13e48be 100644 --- a/docs/zh/query_language/functions/other_functions.md +++ b/docs/zh/query_language/functions/other_functions.md @@ -503,102 +503,6 @@ SELECT defaultValueOfArgumentType(CAST(1, 'Nullable(Int8)')) 1 rows in set. Elapsed: 0.002 sec. ``` -## indexHint - -输出符合索引选择范围内的所有数据,同时不实用参数中的表达式进行过滤。 - -传递给函数的表达式参数将不会被计算,但ClickHouse使用参数中的表达式进行索引过滤。 - -**返回值** - -- 1。 - -**示例** - -这是一个包含[ontime](../../getting_started/example_datasets/ontime.md)测试数据集的测试表。 - -``` -SELECT count() FROM ontime - -┌─count()─┐ -│ 4276457 │ -└─────────┘ -``` - -该表使用`(FlightDate, (Year, FlightDate))`作为索引。 - -对该表进行如下的查询: - -``` -:) SELECT FlightDate AS k, count() FROM ontime GROUP BY k ORDER BY k - -SELECT - FlightDate AS k, - count() -FROM ontime -GROUP BY k -ORDER BY k ASC - -┌──────────k─┬─count()─┐ -│ 2017-01-01 │ 13970 │ -│ 2017-01-02 │ 15882 │ -........................ -│ 2017-09-28 │ 16411 │ -│ 2017-09-29 │ 16384 │ -│ 2017-09-30 │ 12520 │ -└────────────┴─────────┘ - -273 rows in set. Elapsed: 0.072 sec. Processed 4.28 million rows, 8.55 MB (59.00 million rows/s., 118.01 MB/s.) -``` - -在这个查询中,由于没有使用索引,所以ClickHouse将处理整个表的所有数据(`Processed 4.28 million rows`)。使用下面的查询尝试使用索引进行查询: - -``` -:) SELECT FlightDate AS k, count() FROM ontime WHERE k = '2017-09-15' GROUP BY k ORDER BY k - -SELECT - FlightDate AS k, - count() -FROM ontime -WHERE k = '2017-09-15' -GROUP BY k -ORDER BY k ASC - -┌──────────k─┬─count()─┐ -│ 2017-09-15 │ 16428 │ -└────────────┴─────────┘ - -1 rows in set. Elapsed: 0.014 sec. Processed 32.74 thousand rows, 65.49 KB (2.31 million rows/s., 4.63 MB/s.) -``` - -在最后一行的显示中,通过索引ClickHouse处理的行数明显减少(`Processed 32.74 thousand rows`)。 - -现在将表达式`k = '2017-09-15'`传递给`indexHint`函数: - -``` -:) SELECT FlightDate AS k, count() FROM ontime WHERE indexHint(k = '2017-09-15') GROUP BY k ORDER BY k - -SELECT - FlightDate AS k, - count() -FROM ontime -WHERE indexHint(k = '2017-09-15') -GROUP BY k -ORDER BY k ASC - -┌──────────k─┬─count()─┐ -│ 2017-09-14 │ 7071 │ -│ 2017-09-15 │ 16428 │ -│ 2017-09-16 │ 1077 │ -│ 2017-09-30 │ 8167 │ -└────────────┴─────────┘ - -4 rows in set. Elapsed: 0.004 sec. Processed 32.74 thousand rows, 65.49 KB (8.97 million rows/s., 17.94 MB/s.) -``` - -对于这个请求,根据ClickHouse显示ClickHouse与上一次相同的方式应用了索引(`Processed 32.74 thousand rows`)。但是,最终返回的结果集中并没有根据`k = '2017-09-15'`表达式进行过滤结果。 - -由于ClickHouse中使用稀疏索引,因此在读取范围时(本示例中为相邻日期),“额外”的数据将包含在索引结果中。使用`indexHint`函数可以查看到它们。 ## replicate From 1991464cb6d4e5bfde773dcee8dc88bf23fa6330 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sat, 7 Mar 2020 22:56:58 +0300 Subject: [PATCH 115/712] Remove `findClusterIndex`, `findClusterValue` functions (#9543) * Remove findCluster* functions * Fixed test * Fixed test --- dbms/src/Functions/FunctionsFindCluster.cpp | 14 - dbms/src/Functions/FunctionsFindCluster.h | 302 ------------------ dbms/src/Functions/registerFunctions.cpp | 2 - .../0_stateless/00809_add_days_segfault.sql | 2 +- 4 files changed, 1 insertion(+), 319 deletions(-) delete mode 100644 dbms/src/Functions/FunctionsFindCluster.cpp delete mode 100644 dbms/src/Functions/FunctionsFindCluster.h diff --git a/dbms/src/Functions/FunctionsFindCluster.cpp b/dbms/src/Functions/FunctionsFindCluster.cpp deleted file mode 100644 index 4f7caf8d536..00000000000 --- a/dbms/src/Functions/FunctionsFindCluster.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include -#include - - -namespace DB -{ - -void registerFunctionsFindCluster(FunctionFactory & factory) -{ - factory.registerFunction(); - factory.registerFunction(); -} - -} diff --git a/dbms/src/Functions/FunctionsFindCluster.h b/dbms/src/Functions/FunctionsFindCluster.h deleted file mode 100644 index 26eb6564020..00000000000 --- a/dbms/src/Functions/FunctionsFindCluster.h +++ /dev/null @@ -1,302 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include - -#include -#include - -#include - -#include - - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; - extern const int ILLEGAL_COLUMN; -} - -enum ClusterOperation -{ - FindClusterIndex = 0, - FindCentroidValue = 1 -}; - -/// The centroid values are converted to Float64 for easier coding of -/// distance calculations. -/// -/// We assume to have 10th to 100th centroids, usually of type Float64, as a typical use case. -/// While it is possible to sort centroids and use a modification of a binary search to find the -/// nearest centroid, we think for arrays of 10th to 100th this might be an overkill. -/// -/// Also, even though centroids of other types are feasible, this first implementation -/// lacks support of them for simplicity. Date, DateTime and Strings (eg. with the -/// Levenshtein distance) could be theoretically supported, as well as custom distance -/// functions (eg. Hamming distance) using Clickhouse lambdas. - -// Centroids array has the same size as number of clusters. -inline size_t find_centroid(Float64 x, std::vector & centroids) -{ - // Centroids array has to have at least one element, and if it has only one element, - // it is also the result of this Function. - Float64 distance = std::abs(centroids[0] - x); - size_t index = 0; - - // Check if we have more clusters and if we have, whether some is closer to src[i] - for (size_t j = 1; j < centroids.size(); ++j) - { - Float64 next_distance = std::abs(centroids[j] - x); - - if (next_distance < distance) - { - distance = next_distance; - index = j; - } - } - - // Index of the closest cluster, or 0 in case of just one cluster - return index; -} - -/** findClusterIndex(x, centroids_array) - find index of element in centroids_array with the value nearest to x - * findClusterValue(x, centroids_array) - find value of element in centroids_array with the value nearest to x - * - * Types: - * findClusterIndex(T, Array(T)) -> UInt64 - * findClusterValue(T, Array(T)) -> T - * - * T can be any numeric type. - * centroids_array must be constant - */ -class FunctionFindClusterIndex : public IFunction -{ -public: - static constexpr auto name = "findClusterIndex"; - static FunctionPtr create(const Context &) - { - return std::make_shared(); - } - - String getName() const override - { - return FunctionFindClusterIndex::name; - } - - bool isVariadic() const override - { - return true; - } - - size_t getNumberOfArguments() const override - { - return 0; - } - - bool useDefaultImplementationForConstants() const override { return true; } - ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; } - - DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override - { - const auto args_size = arguments.size(); - if (args_size != 2) - throw Exception{"Number of arguments for function " + getName() + " doesn't match: passed " + toString(args_size) + ", should be 2", - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH}; - - const auto type_x = arguments[0]; - - if (!isNativeNumber(type_x)) - throw Exception{"Unsupported type " + type_x->getName() + " of first argument of function " + getName() + " must be a numeric type", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - - const DataTypeArray * type_arr_from = checkAndGetDataType(arguments[1].get()); - - if (!type_arr_from) - throw Exception{"Second argument of function " + getName() + " must be literal array", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - - return std::make_shared(); - } - - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) override - { - const auto in_untyped = block.getByPosition(arguments[0]).column.get(); - const auto centroids_array_untyped = block.getByPosition(arguments[1]).column.get(); - auto column_result = block.getByPosition(result).type->createColumn(); - auto out_untyped = column_result.get(); - - if (!isColumnConst(*centroids_array_untyped)) - throw Exception{"Second argument of function " + getName() + " must be literal array", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - - executeImplTyped(in_untyped, out_untyped, centroids_array_untyped); - - block.getByPosition(result).column = std::move(column_result); - } - -protected: - virtual ClusterOperation getOperation() - { - return ClusterOperation::FindClusterIndex; - } - - virtual void executeImplTyped(const IColumn* in_untyped, IColumn* out_untyped, const IColumn* centroids_array_untyped) - { - if (!executeOperation(in_untyped, out_untyped, centroids_array_untyped) - && !executeOperation(in_untyped, out_untyped, centroids_array_untyped) - && !executeOperation(in_untyped, out_untyped, centroids_array_untyped) - && !executeOperation(in_untyped, out_untyped, centroids_array_untyped) - && !executeOperation(in_untyped, out_untyped, centroids_array_untyped) - && !executeOperation(in_untyped, out_untyped, centroids_array_untyped) - && !executeOperation(in_untyped, out_untyped, centroids_array_untyped) - && !executeOperation(in_untyped, out_untyped, centroids_array_untyped) - && !executeOperation(in_untyped, out_untyped, centroids_array_untyped) - && !executeOperation(in_untyped, out_untyped, centroids_array_untyped)) - { - throw Exception{"Function " + getName() + " expects both x and centroids_array of a numeric type." - " Passed arguments are " + in_untyped->getName() + " and " + centroids_array_untyped->getName(), ErrorCodes::ILLEGAL_COLUMN}; - - } - } - - // Match the type of the centrods array and convert them to Float64, because we - // don't want to have problems calculating negative distances of UInts - template - bool fillCentroids(const IColumn * centroids_array_untyped, std::vector & centroids) - { - const ColumnConst * const_centroids_array = checkAndGetColumnConst>(centroids_array_untyped); - - if (!const_centroids_array) - return false; - - Array array = const_centroids_array->getValue(); - if (array.empty()) - throw Exception{"Centroids array must be not empty", ErrorCodes::ILLEGAL_COLUMN}; - - for (size_t k = 0; k < array.size(); ++k) - { - const Field & tmp_field = array[k]; - NearestFieldType value; - if (!tmp_field.tryGet(value)) - return false; - - centroids.push_back(Float64(value)); - } - return true; - } - - template - bool executeOperation(const IColumn * in_untyped, IColumn * out_untyped, const IColumn * centroids_array_untyped) - { - // Match the type of the output - auto out = typeid_cast *>(out_untyped); - - if (!out) - return false; - - PaddedPODArray & dst = out->getData(); - - // try to match the type of the input column - if (!executeOperationTyped(in_untyped, dst, centroids_array_untyped) - && !executeOperationTyped(in_untyped, dst, centroids_array_untyped) - && !executeOperationTyped(in_untyped, dst, centroids_array_untyped) - && !executeOperationTyped(in_untyped, dst, centroids_array_untyped) - && !executeOperationTyped(in_untyped, dst, centroids_array_untyped) - && !executeOperationTyped(in_untyped, dst, centroids_array_untyped) - && !executeOperationTyped(in_untyped, dst, centroids_array_untyped) - && !executeOperationTyped(in_untyped, dst, centroids_array_untyped) - && !executeOperationTyped(in_untyped, dst, centroids_array_untyped) - && !executeOperationTyped(in_untyped, dst, centroids_array_untyped)) - { - return false; - } - - return true; - } - - template - bool executeOperationTyped(const IColumn * in_untyped, PaddedPODArray & dst, const IColumn * centroids_array_untyped) - { - const auto maybe_const = in_untyped->convertToFullColumnIfConst(); - in_untyped = maybe_const.get(); - - const auto in_vector = checkAndGetColumn>(in_untyped); - if (in_vector) - { - const PaddedPODArray & src = in_vector->getData(); - - std::vector centroids; - if (!fillCentroids(centroids_array_untyped, centroids)) - return false; - - for (size_t i = 0; i < src.size(); ++i) - { - size_t index = find_centroid(Float64(src[i]), centroids); - if (getOperation() == ClusterOperation::FindClusterIndex) - // Note that array indexes start with 1 in Clickhouse - dst.push_back(UInt64(index + 1)); - else if (getOperation() == ClusterOperation::FindCentroidValue) - dst.push_back(centroids[index]); - else - throw Exception{"Unexpected error in findCluster* function", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - } - - return true; - } - return false; - } - -}; - -class FunctionFindClusterValue : public FunctionFindClusterIndex -{ -public: - static constexpr auto name = "findClusterValue"; - static FunctionPtr create(const Context &) - { - return std::make_shared(); - } - - DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override - { - FunctionFindClusterIndex::getReturnTypeImpl(arguments); - const DataTypeArray * type_arr_from = checkAndGetDataType(arguments[1].get()); - return type_arr_from->getNestedType(); - } - - String getName() const override - { - return FunctionFindClusterValue::name; - } - -protected: - ClusterOperation getOperation() override - { - return ClusterOperation::FindCentroidValue; - } - - void executeImplTyped(const IColumn* in_untyped, IColumn* out_untyped, const IColumn* centroids_array_untyped) override - { - if (!executeOperation(in_untyped, out_untyped, centroids_array_untyped) - && !executeOperation(in_untyped, out_untyped, centroids_array_untyped) - && !executeOperation(in_untyped, out_untyped, centroids_array_untyped) - && !executeOperation(in_untyped, out_untyped, centroids_array_untyped) - && !executeOperation(in_untyped, out_untyped, centroids_array_untyped) - && !executeOperation(in_untyped, out_untyped, centroids_array_untyped) - && !executeOperation(in_untyped, out_untyped, centroids_array_untyped) - && !executeOperation(in_untyped, out_untyped, centroids_array_untyped) - && !executeOperation(in_untyped, out_untyped, centroids_array_untyped) - && !executeOperation(in_untyped, out_untyped, centroids_array_untyped)) - { - throw Exception{"Function " + getName() + " expects both x and centroids_array of a numeric type." - "Passed arguments are " + in_untyped->getName() + " and " + centroids_array_untyped->getName(), ErrorCodes::ILLEGAL_COLUMN}; - } - } -}; - -} diff --git a/dbms/src/Functions/registerFunctions.cpp b/dbms/src/Functions/registerFunctions.cpp index 652e9a8b8af..233018c7f16 100644 --- a/dbms/src/Functions/registerFunctions.cpp +++ b/dbms/src/Functions/registerFunctions.cpp @@ -35,7 +35,6 @@ void registerFunctionsMath(FunctionFactory &); void registerFunctionsGeo(FunctionFactory &); void registerFunctionsIntrospection(FunctionFactory &); void registerFunctionsNull(FunctionFactory &); -void registerFunctionsFindCluster(FunctionFactory &); void registerFunctionsJSON(FunctionFactory &); void registerFunctionsConsistentHashing(FunctionFactory & factory); @@ -74,7 +73,6 @@ void registerFunctions() registerFunctionsMath(factory); registerFunctionsGeo(factory); registerFunctionsNull(factory); - registerFunctionsFindCluster(factory); registerFunctionsJSON(factory); registerFunctionsIntrospection(factory); registerFunctionsConsistentHashing(factory); diff --git a/dbms/tests/queries/0_stateless/00809_add_days_segfault.sql b/dbms/tests/queries/0_stateless/00809_add_days_segfault.sql index b087f7bbde5..3d2e11ece77 100644 --- a/dbms/tests/queries/0_stateless/00809_add_days_segfault.sql +++ b/dbms/tests/queries/0_stateless/00809_add_days_segfault.sql @@ -8,5 +8,5 @@ SET send_logs_level = 'none'; SELECT ignore(addDays((CAST((96.338) AS DateTime)), -3)); SELECT ignore(subtractDays((CAST((-5263074.47) AS DateTime)), -737895)); -SELECT quantileDeterministic([], findClusterIndex(( SELECT subtractDays((CAST((566450.398706) AS DateTime)), 54) ) )), '\0', []; -- { serverError 42 } +SELECT quantileDeterministic([], identity(( SELECT subtractDays((CAST((566450.398706) AS DateTime)), 54) ) )), '\0', []; -- { serverError 43 } SELECT sequenceCount((CAST((( SELECT NULL ) AS rg, ( SELECT ( SELECT [], ' Date: Sat, 7 Mar 2020 23:35:55 +0300 Subject: [PATCH 116/712] Support for NULL as random seed; reordered parameters for convenience --- dbms/src/Storages/StorageGenerateRandom.cpp | 26 +++++++++---------- dbms/src/Storages/StorageGenerateRandom.h | 3 ++- .../TableFunctionGenerateRandom.cpp | 16 ++++++------ docs/en/operations/table_engines/generate.md | 4 +-- .../table_functions/generate.md | 6 ++--- 5 files changed, 28 insertions(+), 27 deletions(-) diff --git a/dbms/src/Storages/StorageGenerateRandom.cpp b/dbms/src/Storages/StorageGenerateRandom.cpp index 6f98dc1746c..e2e04b9a638 100644 --- a/dbms/src/Storages/StorageGenerateRandom.cpp +++ b/dbms/src/Storages/StorageGenerateRandom.cpp @@ -352,10 +352,10 @@ private: StorageGenerateRandom::StorageGenerateRandom(const StorageID & table_id_, const ColumnsDescription & columns_, - UInt64 max_array_length_, UInt64 max_string_length_, UInt64 random_seed_) + UInt64 max_array_length_, UInt64 max_string_length_, std::optional random_seed_) : IStorage(table_id_), max_array_length(max_array_length_), max_string_length(max_string_length_) { - random_seed = random_seed_ ? random_seed_ : randomSeed(); + random_seed = random_seed_ ? *random_seed_ : randomSeed(); setColumns(columns_); } @@ -367,25 +367,25 @@ void registerStorageGenerateRandom(StorageFactory & factory) ASTs & engine_args = args.engine_args; if (engine_args.size() > 3) - throw Exception("Storage GenerateRandom requires at most three arguments: "\ - "max_array_length, max_string_length, random_seed.", - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + throw Exception("Storage GenerateRandom requires at most three arguments: " + "random_seed, max_string_length, max_array_length.", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - UInt64 max_array_length_ = 10; - UInt64 max_string_length_ = 10; - UInt64 random_seed_ = 0; // zero for random + std::optional random_seed; + UInt64 max_string_length = 10; + UInt64 max_array_length = 10; - /// Parsing second argument if present if (engine_args.size() >= 1) - max_array_length_ = engine_args[0]->as().value.safeGet(); + random_seed = engine_args[2]->as().value.safeGet(); if (engine_args.size() >= 2) - max_string_length_ = engine_args[1]->as().value.safeGet(); + max_string_length = engine_args[0]->as().value.safeGet(); if (engine_args.size() == 3) - random_seed_ = engine_args[2]->as().value.safeGet(); + max_array_length = engine_args[1]->as().value.safeGet(); - return StorageGenerateRandom::create(args.table_id, args.columns, max_array_length_, max_string_length_, random_seed_); + + return StorageGenerateRandom::create(args.table_id, args.columns, max_array_length, max_string_length, random_seed); }); } diff --git a/dbms/src/Storages/StorageGenerateRandom.h b/dbms/src/Storages/StorageGenerateRandom.h index 7622099dcbb..f39ca1f18c1 100644 --- a/dbms/src/Storages/StorageGenerateRandom.h +++ b/dbms/src/Storages/StorageGenerateRandom.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -29,7 +30,7 @@ private: protected: StorageGenerateRandom(const StorageID & table_id_, const ColumnsDescription & columns_, - UInt64 max_array_length, UInt64 max_string_length, UInt64 random_seed); + UInt64 max_array_length, UInt64 max_string_length, std::optional random_seed); }; } diff --git a/dbms/src/TableFunctions/TableFunctionGenerateRandom.cpp b/dbms/src/TableFunctions/TableFunctionGenerateRandom.cpp index 327e941508a..df615729cb8 100644 --- a/dbms/src/TableFunctions/TableFunctionGenerateRandom.cpp +++ b/dbms/src/TableFunctions/TableFunctionGenerateRandom.cpp @@ -36,30 +36,30 @@ StoragePtr TableFunctionGenerateRandom::executeImpl(const ASTPtr & ast_function, if (args.size() < 1) throw Exception("Table function '" + getName() + "' requires at least one argument: " - " structure(, max_array_length, max_string_length, random_seed).", + " structure, [random_seed, max_string_length, max_array_length].", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); if (args.size() > 4) throw Exception("Table function '" + getName() + "' requires at most four arguments: " - " structure, max_array_length, max_string_length, random_seed.", + " structure, [random_seed, max_string_length, max_array_length].", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); /// Parsing first argument as table structure and creating a sample block std::string structure = args[0]->as().value.safeGet(); - UInt64 max_array_length = 10; UInt64 max_string_length = 10; - UInt64 random_seed = 0; // zero for random + UInt64 max_array_length = 10; + std::optional random_seed = 0; // zero for random - /// Parsing second argument if present if (args.size() >= 2) - max_array_length = args[1]->as().value.safeGet(); + random_seed = args[3]->as().value.safeGet(); if (args.size() >= 3) - max_string_length = args[2]->as().value.safeGet(); + max_string_length = args[1]->as().value.safeGet(); if (args.size() == 4) - random_seed = args[3]->as().value.safeGet(); + max_array_length = args[2]->as().value.safeGet(); + ColumnsDescription columns = parseColumnsListFromString(structure, context); diff --git a/docs/en/operations/table_engines/generate.md b/docs/en/operations/table_engines/generate.md index bdf52f84ac1..fd98b3c9d18 100644 --- a/docs/en/operations/table_engines/generate.md +++ b/docs/en/operations/table_engines/generate.md @@ -10,7 +10,7 @@ Usage examples: ## Usage in ClickHouse Server ```sql -ENGINE = GenerateRandom(max_array_length, max_string_length, random_seed) +ENGINE = GenerateRandom(random_seed, max_string_length, max_array_length) ``` The `max_array_length` and `max_string_length` parameters specify maximum length of all @@ -25,7 +25,7 @@ It supports all [DataTypes](../../data_types/index.md) that can be stored in a t **1.** Set up the `generate_engine_table` table: ```sql -CREATE TABLE generate_engine_table (name String, value UInt32) ENGINE=GenerateRandom(3, 5, 1) +CREATE TABLE generate_engine_table (name String, value UInt32) ENGINE = GenerateRandom(1, 5, 3) ``` **2.** Query the data: diff --git a/docs/en/query_language/table_functions/generate.md b/docs/en/query_language/table_functions/generate.md index 2f43bf453db..cca435557ac 100644 --- a/docs/en/query_language/table_functions/generate.md +++ b/docs/en/query_language/table_functions/generate.md @@ -5,7 +5,7 @@ Allows to populate test tables with data. Supports all data types that can be stored in table except `LowCardinality` and `AggregateFunction`. ```sql -generateRandom('name TypeName[, name TypeName]...', 'limit'[, 'max_array_length'[, 'max_string_length'[, 'random_seed']]]); +generateRandom('name TypeName[, name TypeName]...', [, 'random_seed'[, 'max_string_length'[, 'max_array_length']]]); ``` **Parameters** @@ -15,7 +15,7 @@ generateRandom('name TypeName[, name TypeName]...', 'limit'[, 'max_array_length' - `limit` — Number of rows to generate. - `max_array_length` — Maximum array length for all generated arrays. Defaults to `10`. - `max_string_length` — Maximum string length for all generated strings. Defaults to `10`. -- `random_seed` — Specify random seed manually to produce stable results. Defaults to `0` — seed is randomly generated. +- `random_seed` — Specify random seed manually to produce stable results. If NULL — seed is randomly generated. **Returned Value** @@ -25,7 +25,7 @@ A table object with requested schema. ```sql -SELECT * FROM generateRandom('a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3), UUID)', 3, 2, 10, 1); +SELECT * FROM generateRandom('a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3), UUID)', 1, 10, 2); ``` ```text ┌─a────────┬────────────d─┬─c──────────────────────────────────────────────────────────────────┐ From 609aef269d9d18a6150c0dc1c8f27c3f03dd6fa8 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 7 Mar 2020 23:54:27 +0300 Subject: [PATCH 117/712] Support for NULL as random seed; reordered parameters for convenience --- dbms/src/Storages/StorageGenerateRandom.cpp | 6 +++--- dbms/src/TableFunctions/TableFunctionGenerateRandom.cpp | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dbms/src/Storages/StorageGenerateRandom.cpp b/dbms/src/Storages/StorageGenerateRandom.cpp index e2e04b9a638..9efc832bf81 100644 --- a/dbms/src/Storages/StorageGenerateRandom.cpp +++ b/dbms/src/Storages/StorageGenerateRandom.cpp @@ -376,13 +376,13 @@ void registerStorageGenerateRandom(StorageFactory & factory) UInt64 max_array_length = 10; if (engine_args.size() >= 1) - random_seed = engine_args[2]->as().value.safeGet(); + random_seed = engine_args[0]->as().value.safeGet(); if (engine_args.size() >= 2) - max_string_length = engine_args[0]->as().value.safeGet(); + max_string_length = engine_args[1]->as().value.safeGet(); if (engine_args.size() == 3) - max_array_length = engine_args[1]->as().value.safeGet(); + max_array_length = engine_args[2]->as().value.safeGet(); return StorageGenerateRandom::create(args.table_id, args.columns, max_array_length, max_string_length, random_seed); diff --git a/dbms/src/TableFunctions/TableFunctionGenerateRandom.cpp b/dbms/src/TableFunctions/TableFunctionGenerateRandom.cpp index df615729cb8..6431ca3bd54 100644 --- a/dbms/src/TableFunctions/TableFunctionGenerateRandom.cpp +++ b/dbms/src/TableFunctions/TableFunctionGenerateRandom.cpp @@ -52,13 +52,13 @@ StoragePtr TableFunctionGenerateRandom::executeImpl(const ASTPtr & ast_function, std::optional random_seed = 0; // zero for random if (args.size() >= 2) - random_seed = args[3]->as().value.safeGet(); + random_seed = args[1]->as().value.safeGet(); if (args.size() >= 3) - max_string_length = args[1]->as().value.safeGet(); + max_string_length = args[2]->as().value.safeGet(); if (args.size() == 4) - max_array_length = args[2]->as().value.safeGet(); + max_array_length = args[3]->as().value.safeGet(); ColumnsDescription columns = parseColumnsListFromString(structure, context); From 757c37c03ee7cdd2cacf133f0475fca19330bb1f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 8 Mar 2020 00:09:45 +0300 Subject: [PATCH 118/712] Support for NULL as random seed; reordered parameters for convenience --- dbms/src/Storages/StorageGenerateRandom.cpp | 10 +++++++--- .../TableFunctions/TableFunctionGenerateRandom.cpp | 12 ++++++++---- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/dbms/src/Storages/StorageGenerateRandom.cpp b/dbms/src/Storages/StorageGenerateRandom.cpp index 9efc832bf81..7584b417a93 100644 --- a/dbms/src/Storages/StorageGenerateRandom.cpp +++ b/dbms/src/Storages/StorageGenerateRandom.cpp @@ -376,13 +376,17 @@ void registerStorageGenerateRandom(StorageFactory & factory) UInt64 max_array_length = 10; if (engine_args.size() >= 1) - random_seed = engine_args[0]->as().value.safeGet(); + { + const Field & value = engine_args[0]->as().value; + if (!value.isNull()) + random_seed = value.safeGet(); + } if (engine_args.size() >= 2) - max_string_length = engine_args[1]->as().value.safeGet(); + max_string_length = engine_args[1]->as().value.safeGet(); if (engine_args.size() == 3) - max_array_length = engine_args[2]->as().value.safeGet(); + max_array_length = engine_args[2]->as().value.safeGet(); return StorageGenerateRandom::create(args.table_id, args.columns, max_array_length, max_string_length, random_seed); diff --git a/dbms/src/TableFunctions/TableFunctionGenerateRandom.cpp b/dbms/src/TableFunctions/TableFunctionGenerateRandom.cpp index 6431ca3bd54..16c957aecbe 100644 --- a/dbms/src/TableFunctions/TableFunctionGenerateRandom.cpp +++ b/dbms/src/TableFunctions/TableFunctionGenerateRandom.cpp @@ -45,20 +45,24 @@ StoragePtr TableFunctionGenerateRandom::executeImpl(const ASTPtr & ast_function, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); /// Parsing first argument as table structure and creating a sample block - std::string structure = args[0]->as().value.safeGet(); + std::string structure = args[0]->as().value.safeGet(); UInt64 max_string_length = 10; UInt64 max_array_length = 10; std::optional random_seed = 0; // zero for random if (args.size() >= 2) - random_seed = args[1]->as().value.safeGet(); + { + const Field & value = args[1]->as().value; + if (!value.isNull()) + random_seed = value.safeGet(); + } if (args.size() >= 3) - max_string_length = args[2]->as().value.safeGet(); + max_string_length = args[2]->as().value.safeGet(); if (args.size() == 4) - max_array_length = args[3]->as().value.safeGet(); + max_array_length = args[3]->as().value.safeGet(); ColumnsDescription columns = parseColumnsListFromString(structure, context); From 4a5d61f43f099e55fdd83d81f08cc63fdfc62b77 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 8 Mar 2020 01:07:09 +0300 Subject: [PATCH 119/712] Fixed errors --- dbms/src/Storages/StorageGenerateRandom.cpp | 10 +++++----- .../src/TableFunctions/TableFunctionGenerateRandom.cpp | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dbms/src/Storages/StorageGenerateRandom.cpp b/dbms/src/Storages/StorageGenerateRandom.cpp index 7584b417a93..ee0a0b23e62 100644 --- a/dbms/src/Storages/StorageGenerateRandom.cpp +++ b/dbms/src/Storages/StorageGenerateRandom.cpp @@ -41,7 +41,7 @@ namespace ErrorCodes namespace { -void fillBufferWithRandomData(char * __restrict data, size_t size, pcg64_fast & rng) +void fillBufferWithRandomData(char * __restrict data, size_t size, pcg64 & rng) { char * __restrict end = data + size; while (data < end) @@ -55,7 +55,7 @@ void fillBufferWithRandomData(char * __restrict data, size_t size, pcg64_fast & ColumnPtr fillColumnWithRandomData( - const DataTypePtr type, UInt64 limit, UInt64 max_array_length, UInt64 max_string_length, pcg64_fast & rng, const Context & context) + const DataTypePtr type, UInt64 limit, UInt64 max_array_length, UInt64 max_string_length, pcg64 & rng, const Context & context) { TypeIndex idx = type->getTypeId(); @@ -343,7 +343,7 @@ private: UInt64 max_string_length; Block block_header; - pcg64_fast rng; + pcg64 rng; const Context & context; }; @@ -355,7 +355,7 @@ StorageGenerateRandom::StorageGenerateRandom(const StorageID & table_id_, const UInt64 max_array_length_, UInt64 max_string_length_, std::optional random_seed_) : IStorage(table_id_), max_array_length(max_array_length_), max_string_length(max_string_length_) { - random_seed = random_seed_ ? *random_seed_ : randomSeed(); + random_seed = random_seed_ ? sipHash64(*random_seed_) : randomSeed(); setColumns(columns_); } @@ -416,7 +416,7 @@ Pipes StorageGenerateRandom::read( } /// Will create more seed values for each source from initial seed. - pcg64_fast generate(random_seed); + pcg64 generate(random_seed); for (UInt64 i = 0; i < num_streams; ++i) pipes.emplace_back(std::make_shared(max_block_size, max_array_length, max_string_length, generate(), block_header, context)); diff --git a/dbms/src/TableFunctions/TableFunctionGenerateRandom.cpp b/dbms/src/TableFunctions/TableFunctionGenerateRandom.cpp index 16c957aecbe..7b5177b0090 100644 --- a/dbms/src/TableFunctions/TableFunctionGenerateRandom.cpp +++ b/dbms/src/TableFunctions/TableFunctionGenerateRandom.cpp @@ -49,7 +49,7 @@ StoragePtr TableFunctionGenerateRandom::executeImpl(const ASTPtr & ast_function, UInt64 max_string_length = 10; UInt64 max_array_length = 10; - std::optional random_seed = 0; // zero for random + std::optional random_seed; if (args.size() >= 2) { From 82eb71fd899218f0c9e799f0aba355396ba03ca3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 8 Mar 2020 01:13:01 +0300 Subject: [PATCH 120/712] Added missing code --- dbms/src/Storages/StorageGenerateRandom.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/dbms/src/Storages/StorageGenerateRandom.cpp b/dbms/src/Storages/StorageGenerateRandom.cpp index ee0a0b23e62..12c5d1ac09f 100644 --- a/dbms/src/Storages/StorageGenerateRandom.cpp +++ b/dbms/src/Storages/StorageGenerateRandom.cpp @@ -12,6 +12,8 @@ #include #include #include +#include +#include #include #include #include @@ -295,6 +297,14 @@ ColumnPtr fillColumnWithRandomData( fillBufferWithRandomData(reinterpret_cast(column_concrete.getData().data()), limit * sizeof(Decimal128), rng); return column; } + case TypeIndex::FixedString: + { + size_t n = typeid_cast(*type).getN(); + auto column = ColumnFixedString::create(n); + column->getChars().resize(limit * n); + fillBufferWithRandomData(reinterpret_cast(column->getChars().data()), limit * n, rng); + return column; + } case TypeIndex::DateTime64: { auto column = type->createColumn(); From ec893a999866743913fd54344aa7641294c4cef3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 8 Mar 2020 01:13:21 +0300 Subject: [PATCH 121/712] Adjusted parameters in test --- .../01087_table_function_generate.sql | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/dbms/tests/queries/0_stateless/01087_table_function_generate.sql b/dbms/tests/queries/0_stateless/01087_table_function_generate.sql index 6891dd94520..0e856b79972 100644 --- a/dbms/tests/queries/0_stateless/01087_table_function_generate.sql +++ b/dbms/tests/queries/0_stateless/01087_table_function_generate.sql @@ -10,7 +10,7 @@ SELECT ui32, i32, ui16, i16, ui8, i8 -FROM generateRandom('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8', 10, 10, 1) +FROM generateRandom('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8', 1, 10, 10) LIMIT 10; SELECT '-'; SELECT @@ -19,7 +19,7 @@ FROM generateRandom('i Enum8(\'hello\' = 1, \'world\' = 5)') LIMIT 1; SELECT i -FROM generateRandom('i Enum8(\'hello\' = 1, \'world\' = 5)', 10, 10, 1) +FROM generateRandom('i Enum8(\'hello\' = 1, \'world\' = 5)', 1, 10, 10) LIMIT 10; SELECT '-'; SELECT @@ -28,7 +28,7 @@ FROM generateRandom('i Array(Nullable(Enum8(\'hello\' = 1, \'world\' = 5)))') LIMIT 1; SELECT i -FROM generateRandom('i Array(Nullable(Enum8(\'hello\' = 1, \'world\' = 5)))', 10, 10, 1) +FROM generateRandom('i Array(Nullable(Enum8(\'hello\' = 1, \'world\' = 5)))', 1, 10, 10) LIMIT 10; SELECT '-'; SELECT @@ -37,7 +37,7 @@ FROM generateRandom('i Nullable(Enum16(\'h\' = 1, \'w\' = 5 , \'o\' = -200)))') LIMIT 1; SELECT i -FROM generateRandom('i Nullable(Enum16(\'h\' = 1, \'w\' = 5 , \'o\' = -200)))', 10, 10, 1) +FROM generateRandom('i Nullable(Enum16(\'h\' = 1, \'w\' = 5 , \'o\' = -200)))', 1, 10, 10) LIMIT 10; SELECT '-'; SELECT @@ -46,7 +46,7 @@ FROM generateRandom('d Date, dt DateTime, dtm DateTime(\'Europe/Moscow\')') LIMIT 1; SELECT d, dt, dtm -FROM generateRandom('d Date, dt DateTime, dtm DateTime(\'Europe/Moscow\')', 10, 10, 1) +FROM generateRandom('d Date, dt DateTime, dtm DateTime(\'Europe/Moscow\')', 1, 10, 10) LIMIT 10; SELECT '-'; SELECT @@ -55,7 +55,7 @@ FROM generateRandom('dt64 DateTime64, dts64 DateTime64(6), dtms64 DateTime64(6 , LIMIT 1; SELECT dt64, dts64, dtms64 -FROM generateRandom('dt64 DateTime64, dts64 DateTime64(6), dtms64 DateTime64(6 ,\'Europe/Moscow\')', 10, 10, 1) +FROM generateRandom('dt64 DateTime64, dts64 DateTime64(6), dtms64 DateTime64(6 ,\'Europe/Moscow\')', 1, 10, 10) LIMIT 10; SELECT '-'; SELECT @@ -64,7 +64,7 @@ FROM generateRandom('f32 Float32, f64 Float64') LIMIT 1; SELECT f32, f64 -FROM generateRandom('f32 Float32, f64 Float64', 10, 10, 1) +FROM generateRandom('f32 Float32, f64 Float64', 1, 10, 10) LIMIT 10; SELECT '-'; SELECT @@ -73,7 +73,7 @@ FROM generateRandom('d32 Decimal32(4), d64 Decimal64(8), d128 Decimal128(16)') LIMIT 1; SELECT d32, d64, d128 -FROM generateRandom('d32 Decimal32(4), d64 Decimal64(8), d128 Decimal128(16)', 10, 10, 1) +FROM generateRandom('d32 Decimal32(4), d64 Decimal64(8), d128 Decimal128(16)', 1, 10, 10) LIMIT 10; SELECT '-'; SELECT @@ -82,7 +82,7 @@ FROM generateRandom('i Tuple(Int32, Int64)') LIMIT 1; SELECT i -FROM generateRandom('i Tuple(Int32, Int64)', 10, 10, 1) +FROM generateRandom('i Tuple(Int32, Int64)', 1, 10, 10) LIMIT 10; SELECT '-'; SELECT @@ -91,7 +91,7 @@ FROM generateRandom('i Array(Int8)') LIMIT 1; SELECT i -FROM generateRandom('i Array(Int8)', 10, 10, 1) +FROM generateRandom('i Array(Int8)', 1, 10, 10) LIMIT 10; SELECT '-'; SELECT @@ -100,7 +100,7 @@ FROM generateRandom('i Array(Nullable(Int32))') LIMIT 1; SELECT i -FROM generateRandom('i Array(Nullable(Int32))', 10, 10, 1) +FROM generateRandom('i Array(Nullable(Int32))', 1, 10, 10) LIMIT 10; SELECT '-'; SELECT @@ -109,7 +109,7 @@ FROM generateRandom('i Tuple(Int32, Array(Int64))') LIMIT 1; SELECT i -FROM generateRandom('i Tuple(Int32, Array(Int64))', 10, 10, 1) +FROM generateRandom('i Tuple(Int32, Array(Int64))', 1, 10, 10) LIMIT 10; SELECT '-'; SELECT @@ -118,7 +118,7 @@ FROM generateRandom('i Nullable(String)', 1) LIMIT 1; SELECT i -FROM generateRandom('i Nullable(String)', 10, 10, 1) +FROM generateRandom('i Nullable(String)', 1, 10, 10) LIMIT 10; SELECT '-'; SELECT @@ -127,7 +127,7 @@ FROM generateRandom('i Array(String)') LIMIT 1; SELECT i -FROM generateRandom('i Array(String)', 10, 10, 1) +FROM generateRandom('i Array(String)', 1, 10, 10) LIMIT 10; SELECT '-'; @@ -137,7 +137,7 @@ FROM generateRandom('i UUID') LIMIT 1; SELECT i -FROM generateRandom('i UUID', 10, 10, 1) +FROM generateRandom('i UUID', 1, 10, 10) LIMIT 10; SELECT '-'; SELECT @@ -146,7 +146,7 @@ FROM generateRandom('i Array(Nullable(UUID))') LIMIT 1; SELECT i -FROM generateRandom('i Array(Nullable(UUID))', 10, 10, 1) +FROM generateRandom('i Array(Nullable(UUID))', 1, 10, 10) LIMIT 10; SELECT '-'; SELECT @@ -155,7 +155,7 @@ FROM generateRandom('i FixedString(4)') LIMIT 1; SELECT i -FROM generateRandom('i FixedString(4)', 10, 10, 1) +FROM generateRandom('i FixedString(4)', 1, 10, 10) LIMIT 10; SELECT '-'; SELECT @@ -164,12 +164,12 @@ FROM generateRandom('i String') LIMIT 1; SELECT i -FROM generateRandom('i String', 10, 10, 1) +FROM generateRandom('i String', 1, 10, 10) LIMIT 10; SELECT '-'; DROP TABLE IF EXISTS test_table; CREATE TABLE test_table(a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3), UUID)) ENGINE=Memory; -INSERT INTO test_table SELECT * FROM generateRandom('a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3), UUID)', 2, 10, 1) +INSERT INTO test_table SELECT * FROM generateRandom('a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3), UUID)', 1, 10, 2) LIMIT 10; SELECT * FROM test_table; @@ -180,7 +180,7 @@ SELECT '-'; DROP TABLE IF EXISTS test_table_2; CREATE TABLE test_table_2(a Array(Int8), b UInt32, c Nullable(String), d Decimal32(4), e Nullable(Enum16('h' = 1, 'w' = 5 , 'o' = -200)), f Float64, g Tuple(Date, DateTime, DateTime64, UUID), h FixedString(2)) ENGINE=Memory; -INSERT INTO test_table_2 SELECT * FROM generateRandom('a Array(Int8), b UInt32, c Nullable(String), d Decimal32(4), e Nullable(Enum16(\'h\' = 1, \'w\' = 5 , \'o\' = -200)), f Float64, g Tuple(Date, DateTime, DateTime64, UUID), h FixedString(2)', 3, 5, 10) +INSERT INTO test_table_2 SELECT * FROM generateRandom('a Array(Int8), b UInt32, c Nullable(String), d Decimal32(4), e Nullable(Enum16(\'h\' = 1, \'w\' = 5 , \'o\' = -200)), f Float64, g Tuple(Date, DateTime, DateTime64, UUID), h FixedString(2)', 10, 5, 3) LIMIT 10; SELECT * FROM test_table_2; From bfe44d171ed4d23eaf7f277bde7577b3884b2033 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 8 Mar 2020 01:17:45 +0300 Subject: [PATCH 122/712] Updated test --- .../01087_table_function_generate.reference | 364 +++++++++--------- .../01087_table_function_generate.sql | 4 +- 2 files changed, 184 insertions(+), 184 deletions(-) diff --git a/dbms/tests/queries/0_stateless/01087_table_function_generate.reference b/dbms/tests/queries/0_stateless/01087_table_function_generate.reference index 08081a34bc9..291e5b1689a 100644 --- a/dbms/tests/queries/0_stateless/01087_table_function_generate.reference +++ b/dbms/tests/queries/0_stateless/01087_table_function_generate.reference @@ -1,238 +1,238 @@ UInt64 Int64 UInt32 Int32 UInt16 Int16 UInt8 Int8 -5443401583997919274 956654340036924402 2956613447 2041372187 46025 26509 247 -34 -14051730854243326159 340055300607421421 579798001 915264595 58925 22498 36 -57 -12126660396637528292 -9182366379883086416 535113873 -1583603936 45790 6066 230 91 -5198178071978083704 -3549936112074464250 3354362520 -1732019372 41330 -27737 13 -47 -9045663333607591872 -5069075924065328373 741246230 -1830932765 29642 -11720 41 7 -18192666371709191624 -5005976579831091773 671021725 1851158245 38613 -27838 57 3 -4333039311970693040 -7294587049092886539 2106347821 2101852759 24058 9107 85 94 -1398111012802844853 1131449717368086026 1687614855 -1193084417 9803 -18141 198 115 -15838944643191192696 6226099517671026657 1300309956 468322781 17216 -2375 184 -102 -15170414162889419078 3337938833953948518 3603117877 -1297530274 25534 8264 36 16 +2804162938822577320 -2776833771540858 3467776823 1163715250 23903 13655 137 -41 +7885388429666205427 -1363628932535403038 484159052 -308788249 56810 -22227 51 -41 +4357435422797280898 1355609803008819271 4126129912 -852056475 64304 -11401 139 86 +5935810273536892891 -804738887697332962 3109335413 -80126721 258 12889 18 88 +368066018677693974 -4927165984347126295 1015254922 2026080544 44305 21973 16 0 +8124171311239967992 -1179703908046100129 1720727300 -138469036 61343 10573 252 -32 +15657812979985370729 -5733276247123822513 3254757884 -500590428 45913 19153 105 -102 +18371568619324220532 -6793779541583578394 1686821450 -455892108 49050 -28603 248 80 +821735343441964030 3148260644406230976 256251035 -885069056 58858 -29361 58 61 +9558594037060121162 -2907172753635797124 4276198376 1947296644 26801 -13531 204 -66 - Enum8(\'hello\' = 1, \'world\' = 5) +hello world -world +hello world hello hello world -world -world hello -world +hello +hello - Array(Nullable(Enum8(\'hello\' = 1, \'world\' = 5))) -['world','world','hello','hello','world','world','world'] +['hello','hello','world','hello',NULL,'world','world','hello','hello','hello'] +['world','hello'] +['hello','hello','world','hello','world',NULL,NULL,NULL] +['world','world','hello','world','world','world',NULL,'hello','world','world'] +[] +['hello','hello','world','hello','hello','world','hello','world'] +['world',NULL,'world','world','hello','hello','world'] ['world'] -['world','hello','world'] -[] -[] -['world','hello','hello','hello','world'] -['hello'] -['world','hello','hello','world','hello'] -['hello','world','hello','hello','world','world'] -['world','hello','world','hello','hello','world','world'] +['world','world'] +[NULL,'hello','hello','world','world','world','hello','world','world','world'] - Nullable(Enum16(\'o\' = -200, \'h\' = 1, \'w\' = 5)) -w -h -h o -w +h +h +h +h w o -w h +\N o - Date DateTime DateTime(\'Europe/Moscow\') -2031-03-05 2034-09-09 02:49:47 2061-06-26 03:46:01 -1972-10-06 1999-01-02 11:09:55 2064-03-18 05:47:09 -2004-01-16 2055-12-02 15:29:20 2090-08-18 23:04:46 -2061-07-14 2051-03-20 20:58:44 1973-04-20 21:20:34 -2063-04-13 2048-01-31 01:02:11 2051-02-07 03:11:54 -2106-02-07 2028-08-29 13:37:25 2054-10-20 03:48:21 -2026-11-24 2036-08-09 02:59:19 2065-10-12 06:39:38 -2106-02-07 2068-04-17 13:07:59 2101-04-03 08:48:59 -1997-11-15 1984-11-03 12:39:41 1998-04-01 17:38:08 -2008-09-11 2064-12-25 16:23:42 2031-10-18 03:20:14 +2106-02-07 2050-12-17 02:46:35 2096-02-16 22:18:22 +2106-02-07 2013-10-17 23:35:26 1976-01-24 12:52:48 +2039-08-16 1974-11-17 23:22:46 1980-03-04 21:02:50 +1997-04-11 1972-09-18 23:44:08 2040-07-10 14:46:42 +2103-11-03 2044-11-23 20:57:12 1970-10-09 02:30:14 +2066-11-19 2029-12-10 03:13:55 2106-01-30 21:52:44 +2064-08-14 2016-07-14 11:33:45 2096-12-12 00:40:50 +2046-09-13 2085-07-10 18:51:14 2096-01-15 16:31:33 +2008-03-16 2047-05-16 23:28:36 2103-02-11 16:44:39 +2000-07-07 2105-07-19 19:29:06 1980-01-02 05:18:22 - DateTime64(3) DateTime64(6) DateTime64(6, \'Europe/Moscow\') -1988-05-16 19:00:01.447 2064-03-18 05:47:09.972361 2104-06-20 09:26:44.845879 -2076-04-17 18:22:00.873 1973-04-20 21:20:34.769886 2052-08-01 07:14:05.921510 -1991-04-07 13:55:25.230 2054-10-20 03:48:21.341514 2013-02-07 18:37:45.437737 -2023-06-24 16:54:15.821 2101-04-03 08:48:59.544378 2039-07-05 08:51:02.770005 -2084-03-05 21:04:37.956 2031-10-18 03:20:14.437888 2076-03-16 14:08:20.993528 -1999-01-02 11:09:55.187 2054-01-01 16:49:22.580109 1997-01-09 20:11:35.889758 -2051-03-20 20:58:44.360 1975-02-11 06:38:15.042546 2015-10-21 23:47:13.191963 -2028-08-29 13:37:25.531 1975-02-14 07:25:38.319928 2103-09-16 20:57:23.033927 -2068-04-17 13:07:59.759 2024-03-06 21:42:43.711891 2045-04-22 19:38:11.140126 -2064-12-25 16:23:42.781 2025-08-18 15:44:56.149625 2093-09-26 16:30:56.744858 +1978-06-07 23:50:57.320 2013-08-28 10:21:54.010758 1991-08-25 16:23:26.140215 +1978-08-25 17:07:25.427 2034-05-02 20:49:42.148578 2015-08-26 15:26:31.783160 +2037-04-04 10:50:56.898 2055-05-28 11:12:48.819271 2068-12-26 09:58:49.635722 +2041-09-02 07:07:24.891 2051-08-01 14:15:40.218654 2081-10-19 15:55:40.057084 +1976-07-15 23:59:41.974 2075-01-29 20:34:10.425321 1996-12-31 10:51:28.562331 +1974-11-03 08:09:51.992 2010-04-19 04:09:03.451487 1994-05-15 15:42:53.162162 +2061-10-11 20:14:02.729 1981-07-22 10:13:45.729103 2084-05-27 08:59:37.746021 +1989-12-13 02:01:16.532 1992-10-05 07:07:57.973222 2037-10-24 18:53:50.985504 +1992-12-28 12:26:04.030 1971-07-29 09:20:38.230976 1980-03-26 18:49:55.428516 +2051-12-11 10:09:13.162 1982-01-12 03:25:45.754492 2010-05-17 11:01:28.452864 - Float32 Float64 -2.3424705e38 5.304765772621186e307 -4.5936326e37 1.3693852957827914e308 -4.2396088e37 1.1817811347484115e308 -2.6575997e38 5.065787759860024e307 -5.8727575e37 8.815282962741328e307 -5.3163816e37 1.7729324649694315e308 -1.6688205e38 4.2226828718895e307 -1.3370661e38 1.3625030842560206e307 -1.0302116e38 1.5435548915708008e308 -2.8546838e38 1.4784044970034722e308 +-1.3551149e32 1.2262973812461839e235 +1.6263936e-15 -1.4319274895836525e122 +-8.991488e-33 -3.587091060722666e303 +4.6137895e27 9.96990958623199e-254 +9.749564e-13 -3.014080971435583e-286 +0.01518069 nan +-2.5833165e-24 -2.6774132404843463e217 +240769800000 4.559039863342969e-218 +2.0838264e-33 -6.156499824044965e254 +7.317837e-36 -1.6511853645079817e-21 - Decimal32(4) Decimal64(8) Decimal64(8) --133835.3849 87676267830.44260947 10041303591043480341650.6377217747572943 -57979.8001 -68015271123.73929132 -11658496611537681782723.8256877955807880 -53511.3873 -78637963449.98695195 16686303649199763212696.4854950355256776 --94060.4776 90273888640.14252543 7993046724924589483272.0796323974797493 -74124.6230 20114310313.64207198 -4810540869033768101015.4448286464595642 -67102.1725 -60472921957.85611731 1764715777766465744700.9237855716355053 -210634.7821 -20967919098.37725326 -16938476260073815366594.8118263905360890 -168761.4855 -74544559691.08355371 -9350794626143586522954.2962771754340925 -130030.9956 -54650148153.48939189 -13456138041801265081736.4812607484010998 --69184.9419 38286965773.25360062 11485126437992390872631.7990315807376230 +-18731.5032 81241713112.39967992 -10576027963457111164764.0798899532879521 +65289.5061 -27889310937.24180887 5807515838469365530027.7612329616030438 +-197586.1517 -751754543.85331084 3835903211857734974086.0358362773591932 +183596.0063 8217353434.41964030 13633006218585943284268.9826084812209912 +73041.2674 -88881500366.49430454 -148702703925022894263.3187064158377476 +101454.4494 -27768337.71540858 -634829280961262229789.4961995996929358 +-174012.0101 -13636289325.35403038 -3611949395160064991369.2765012316944096 +138203.8526 13556098030.08819271 134470734594381953531.9736002591779584 +15395.1766 -8047388876.97332962 16804394201271843589306.4234533639925009 +8569.7048 -49271659843.47126295 -14851374957489266092927.8687987539036841 - Tuple(Int32, Int64) -(-1338353849,5443401583997919274) -(579798001,-4395013219466225457) -(535113873,-6320083677072023324) -(-940604776,5198178071978083704) -(741246230,9045663333607591872) -(671021725,-254077702000359992) -(2106347821,4333039311970693040) -(1687614855,1398111012802844853) -(1300309956,-2607799430518358920) -(-691849419,-3276329910820132538) +(-187315032,8124171311239967992) +(652895061,-2788931093724180887) +(-1975861517,-75175454385331084) +(1835960063,821735343441964030) +(730412674,-8888150036649430454) +(1014544494,-2776833771540858) +(-1740120101,-1363628932535403038) +(1382038526,1355609803008819271) +(153951766,-804738887697332962) +(85697048,-4927165984347126295) - Array(Int8) -[27,83,32,84,-29,-27,87] -[-1] -[-35,94,-55] +[-122,110,114,1,124,34,-10,-1,-30,61] +[-56,-18] +[-75,106,19,-19,71,-64,96,-6] +[-18,23,-48,18,30,-123,-37,-51,62,-2] [] -[] -[45,-34,114,-54,-43] -[-6] -[75,64,-66,-115,-30] -[-78,-89,56,66,-109,35] -[-71,72,-9,36,-26,13,41] +[-44,-12,-23,5,-102,121,-92,48] +[-97,-69,95,-87,-59,-60,-117] +[-39] +[-96,-17] +[79,-8,-1,34,-95,79,111,-80,-26,58] - Array(Nullable(Int32)) -[2041372187,915264595,-1583603936,-1732019372,-1830932765,1851158245,2101852759] -[-1193084417] -[468322781,-1297530274,-1407994935] +[24276614,-646532,-288866846,-317494603,-94322617,NULL,-841251554,-187367874,2040137193,-1147195228] +[-993679009,-274671221] +[587200591,-1334882399,2107128550,-1581800064,976027584,733011552,-1898440836,-676878904] +[-827190473,NULL,-168837384,-1185631883,1015254922,1720727300,-1040209412,1686821450,256251035,-18768920] [] -[] -[-1321933267,-488197410,104178034,-1735625782,-1618897195] -[-1272422918] -[-153016757,891437888,1950049214,6580109,-1644079134] -[790042546,161321895,1074319928,161583938,515711891,1709750563] -[-149817671,1755521096,815845879,-51580892,1361921510,-1688868851,-1185529559] +[1163715250,-308788249,-852056475,NULL,2026080544,-138469036,-500590428,-455892108] +[-885069056,1947296644,-571843233,16972592,-274748143,-1080380583,1756489194] +[2121012739] +[NULL,NULL] +[NULL,-1874507055,-886731441,821482880,311112585,-127271920,873843770,NULL,1482086359,1352327168] - Tuple(Int32, Array(Int64)) -(-1338353849,[5443401583997919274,-4395013219466225457,-6320083677072023324,5198178071978083704,9045663333607591872,-254077702000359992,4333039311970693040]) -(579798001,[1398111012802844853,-2607799430518358920,-3276329910820132538,956654340036924402,340055300607421421]) -(535113873,[]) -(-940604776,[-9182366379883086416,-3549936112074464250,-5069075924065328373,-5005976579831091773]) -(741246230,[-7294587049092886539]) -(671021725,[1131449717368086026,6226099517671026657,3337938833953948518,-104956130729581604,515805789944032293]) -(2106347821,[2731028582309582302,-8197314279937271385,7439592879615992239,-8726726222408049230,-4046170041070917399,-8162695179087422573,7147712321550951494,-2473105312361834401,2871941532606538254]) -(1687614855,[7045950974355040215,8128475529675984757,3862453874713979777,8584893221699499395,-4344095019439049735,7221768832555831190,5712009283210486481,8657278465574644253,-4620821897447975309]) -(1300309956,[-3580736586972265629]) -(-691849419,[7980379733974797651,-548434416689229144]) +(-187315032,[-1179703908046100129,-5733276247123822513,-6793779541583578394,3148260644406230976,-2907172753635797124,2079447297870140215,-5092250158453768456,7390467479849635722]) +(652895061,[7244842965196057084,-80611897324989285,-1326235429680389454,-344141642787805595,-594719979102566112,-1958041690570123100,8363575405000452864]) +(-1975861517,[72896731291475295]) +(1835960063,[-4640199267198194415,9109680350160872938]) +(730412674,[3628163485716526423,-8050946496653339179,3528242107232128335,-546628733788015735,-4538379399781299142,5808200961534384087,808655365001887293,-8844185874661444452,643019962728680518,8250477176286620863]) +(1014544494,[-287233184700985380,1749427670542803376,-6435907283441313909,-398230413002921126,7257457516659393153,1409595563647451721,-946856126400551895,-8238390188888204749,8805947767553724527]) +(-1740120101,[4566168555161068712,2303771336793744574,-2858308093688847501,-674013896865039545,597627259745635607,1493493065813843889]) +(1382038526,[8163231169061670909]) +(153951766,[-8934224507121259258,-5682982952279561296,-7665466011309768105,1158449958889177529,-5943902981558064139]) +(85697048,[2092132020040612180,-7829994932198211729,5992776369613298329,-2557419325779681965,-2080080576758631926,1226592045800496749,1791516122484114661,-6512855691472390097,764717459130233392,4903403636828862838]) - Nullable(String) -;\\Sm\'sH -T -@^1 +fSRH40d6sX - -7-f)$ -9 -)&}y3 -w5>+\'@ -+g+N^g$ +=@ep]Vw~ +b\' +6xGwg|(&Q +^ipx|,=a5N +(U]p\'l` +U6 +\'%Y~t9 +RL,{Xs\\tw - Array(String) -['Y9n(%ub','\\\'f%7','','X0O@','D','4^,~q','a(gmt6#{X'] -['@#q4?Q%\'.'] -['h','#B','{'] +['6xGwg|(&Q','^ipx|,=a5N','(U]p\'l`','U6','\'%Y~t9','RL,{Xs\\tw','`xbguF','?/;UTko','k3Igp@',''] +['TlL','UeS}D'] +['0Z3|h','-]','&b!M-e;7','Dj7peUH{T','=D[','_@}a ','_}!','O,9V'] +['r;5qbK&t+','4?a]25n#','_K','4S9,;m','RM nh|E7*','-L[3','','Fm?\'','/D$','.7^Jp5sba$'] [] -[] -['~vYP/4f9.',':,7u.0',';e.<','^O,i','3'] -['!y1/Z\'5D'] -['&- KDN%>[','>-xM./ B','?+//','M,.71QR#_','~N'] -['z9P/%m','7q\'!k','Q%] #.*3','U:&XeP{*',',','s.3'] -['+k.=%','8\'nb=P','-uY ,h8(w','=\'W$','','m<+%l','<~+@ Vw'] +['CB','TaI&szh','Hnc?lApSP','2O"ms26O>','bX?}ix [','UlI+1','U','NQTpY#'] +['8+>','# ;M<:{M','tb:^UG','?AaqQ?$Ee\'','PbE.6x]^'] - UUID -4b8ad8e6-77fe-a02a-c301-c6c33e91d8cf -a84a8d61-8560-a0e4-4823-a36a537a8578 -7d88a6e0-3d74-cfc0-fc79-55ad1aebc5c8 -3c220c4b-fbd0-efb0-1367-168f02acd8b5 -dbcf3c3c-127c-bc78-d288-234e0a942946 -0d46b80d-bebc-93f2-04b8-1e7e84b1abed -8091ae6d-8194-3db0-cebc-17ea18786406 -b9a7064d-de99-e30b-ba87-32d2cd3e2dc3 -9ac46bba-2ba7-4ff5-0fb3-b785f4f0de0a -56678c86-2703-2fe1-2e52-bdaf2fce8366 +26ea6355-f4d5-cca8-6d6e-86ff8a3abef3 +3c78b86e-2b89-3682-5260-3bfe9847dfdb +051ba218-092d-1e16-70be-dac38ce0b4f8 +d94bb9f2-5787-4e69-fef4-ec529188a474 +0b6763f0-ed3d-ebfe-84a6-f2a21322b24a +fff6227c-0172-6e86-ed13-6ab5eec83de2 +12d017ee-fa60-c047-f4d4-fe3ecddb851e +bb9f30a4-799a-05e9-efa0-d98bc4c5a95f +b06f4fa1-22ff-f84f-a1b7-a5807d983ae6 +2bb0de60-3a2c-ffc0-d7a7-a5c88ed8177c - Array(Nullable(UUID)) -['4b8ad8e6-77fe-a02a-c301-c6c33e91d8cf','a84a8d61-8560-a0e4-4823-a36a537a8578','7d88a6e0-3d74-cfc0-fc79-55ad1aebc5c8','3c220c4b-fbd0-efb0-1367-168f02acd8b5','dbcf3c3c-127c-bc78-d288-234e0a942946','0d46b80d-bebc-93f2-04b8-1e7e84b1abed','8091ae6d-8194-3db0-cebc-17ea18786406'] -['b9a7064d-de99-e30b-ba87-32d2cd3e2dc3'] -['9ac46bba-2ba7-4ff5-0fb3-b785f4f0de0a','56678c86-2703-2fe1-2e52-bdaf2fce8366','fe8b1ef4-86dd-23dc-0728-82a212b42c25'] +['fff6227c-0172-6e86-ed13-6ab5eec83de2','12d017ee-fa60-c047-f4d4-fe3ecddb851e','bb9f30a4-799a-05e9-efa0-d98bc4c5a95f','b06f4fa1-22ff-f84f-a1b7-a5807d983ae6','2bb0de60-3a2c-ffc0-d7a7-a5c88ed8177c','1cdbae4c-ceb2-1337-b954-b175f5efbef8','66903704-3c83-8f8a-648a-da4ac1ffa9fc','fee19be8-0f46-149b-ed98-43e7455ce2b2','fb395cff-cd36-a665-f7bf-215478c38920','e4d3a374-e229-98a4-7411-6384cb3eeb00'] +[NULL,'7e6c1603-68b1-e5ea-3259-d377a92d3557'] +['90454ad1-294d-55d5-30f6-d580cb258d4f','f869fc10-128b-3389-c104-6ffb3415cc3a','509ae000-5856-d7d7-0b38-ebc520a6be3d','854323ce-3546-f49c-08ec-773fcded8c46','727f9547-6808-2cbf-fc03-8af0ba318fdc','18473732-8ef1-61b0-a6af-106cb321978b','fa79338c-9e34-2b5a-64b7-ab28e4f8c281',NULL] +['8dab5bc5-a641-5a33-7a35-02d6b3af106f','3f5e4a1b-baef-24a8-1ff8-a3d32d5164be','d8553fee-c90d-6373-f6a5-6bee8faebb47','084b32d3-8d3a-8d17-14b9-f407ce57dfb1','71499f7f-6351-4ffd-8403-42257572ad06','b121fd1d-58c6-abb0-959e-cb46ae0d0257','1013a419-91a7-c1b9-ad83-03bafcd8abf5','1d08befc-d5bd-b954-9356-451821377f6f','532a9e5b-6927-da99-dc82-38ac492edd53','e32211bc-761c-b20a-1105-baf786421e6d'] [] -[] -['25e69006-a800-55de-8e3d-4a17f81a19a7','673ebe4e-af09-61af-86e4-70bca5481db2','c7d91dc9-0123-e8e9-8eb8-47fb80b35b93','6331c67c-7aba-2446-ddad-c3d24c8a985f','27db2f9b-92b4-220e-61c8-3f14833fe7d7'] -['70ce256d-7fca-cf75-359a-3155bba86b81'] -['7723aae6-820e-b583-c3b6-ac9887cab3f9','6438e07d-9a7a-4f96-4f45-222df77ea2d1','7824d4e5-6e37-2a1d-bfdf-8af768445673','ce4eab0b-b346-0363-6ec0-0116104b4d53','f86391cd-c2a3-d2a8-ea85-27ed3508504c'] -['b87d39f3-e56c-7128-7a66-4e516e2ce1c0','af25bac3-d662-673a-3516-022e687643ed','657c9997-3b29-f51c-7193-6a3b9a0c18eb','19bb38b5-6f97-a81c-2d56-57f189119a1a','0c7e416d-c669-dc04-1130-ff950fbbf44b','9705bc44-8d08-c734-6f47-8edcc1608a81'] -['7c20103f-659a-f845-399c-abdc8dc88ba0','bf479e85-1a0f-66c3-66fa-f6029e4ee2a8','f14af1a0-823c-b414-eb8e-e6b05b019868','ce353e45-2a9e-492c-1c54-d50459160ecf','1e232279-77ad-db7e-82f6-b4b3e30cdc2e','991111af-30a3-1ff7-e15a-023dfa0a8a6e','a749ef39-dc02-d05c-e8b2-129a7cccfd24'] +['18dcbe6b-ea0d-04e5-a59d-b0415468382f','0a9cd27a-38ff-3230-440c-63a0db8af976','dff6ee08-7533-ac1f-3f4b-a6450dffffb3','20fc45e0-8b0d-3652-1ef9-6a69cfab4ad8','09f997b7-ff78-73cf-d2e1-014279fc877b','3e96646a-f80c-c4b5-b6ea-ac9de22a1c1d','b9804d77-da87-b21e-33e8-a148791c24c3','4763e2bd-c2ec-9d95-8e8d-78d0fdcd6100'] +['150955fb-d228-6700-f1ee-9c8579cfdf10',NULL,NULL,'706b1793-0b9b-b007-e892-8bb3f13ed5aa','d48ec551-5c15-5f80-09cf-5f3a09b4232e','11f16f65-b07b-8ca6-9037-c400f4cee06f','fd5cb198-44c1-631c-545a-b493d3456ca4'] +['8978d7e4-5b0b-ce2d-c714-66d3728f8226'] +['716309e3-bc54-8fae-d159-077e7f58328a','80413bd9-30b3-e071-8079-0f198e1952bf'] +['a55486af-06c3-4d55-fb6e-54559812b2c0','9f93002e-efb6-c6fd-3928-154e7bbe640b','1f197019-cde5-8fe3-16c0-45d857e32f5d','f2532c84-e28e-d4b8-89a5-335907448f4e','09c22b07-180f-4acb-d898-8d6c84667cf3','d11f8791-0962-9a60-d28c-2f3763793435','9aa152ff-1f38-4172-909e-a75c2f6cd244','7a5e4b17-0178-0a6d-7955-f6e5488d4da2','2bf6aea1-9cb5-1726-8dac-c7144c65e4a9','bfddcefd-68fa-85d8-a550-87605cd540cc'] - FixedString(4) -G -- -5S - T -W^ --r -K -@ -8B -#H +A8CCD5F4 +5563EA26 +F3BE3A8A +FF866E6D +8236892B +6EB8783C +DBDF4798 +FE3B6052 +161E2D09 +18A21B05 - String -;\\Sm\'sH -T -@^1 +fSRH40d6sX - -7-f)$ -9 -)&}y3 -w5>+\'@ -+g+N^g$ +=@ep]Vw~ +b\' +6xGwg|(&Q +^ipx|,=a5N +(U]p\'l` +U6 +\'%Y~t9 +RL,{Xs\\tw - -[27] -119308.4417 ('1998-04-01 17:38:08.539','4b8ad8e6-77fe-a02a-c301-c6c33e91d8cf') -[83] 46832.2781 ('1970-03-18 06:48:29.214','a84a8d61-8560-a0e4-4823-a36a537a8578') -[32] -129753.0274 ('1995-01-14 03:15:46.162','7d88a6e0-3d74-cfc0-fc79-55ad1aebc5c8') -[] -140799.4935 ('2004-01-17 09:12:08.895','3c220c4b-fbd0-efb0-1367-168f02acd8b5') -[] -132193.3267 ('1986-05-06 01:18:11.938','dbcf3c3c-127c-bc78-d288-234e0a942946') -[84] -48819.7410 ('2101-05-10 09:27:05.563','0d46b80d-bebc-93f2-04b8-1e7e84b1abed') -[-29] 10417.8034 ('1995-11-08 18:51:19.096','8091ae6d-8194-3db0-cebc-17ea18786406') -[-27] -173562.5782 ('2013-02-27 03:31:50.404','b9a7064d-de99-e30b-ba87-32d2cd3e2dc3') -[] -161889.7195 ('2068-07-13 23:42:17.445','9ac46bba-2ba7-4ff5-0fb3-b785f4f0de0a') -[87] -127242.2918 ('2033-08-04 15:06:45.865','56678c86-2703-2fe1-2e52-bdaf2fce8366') +[-122] -9432.2617 ('2001-08-23 08:05:41.222','f7bf2154-78c3-8920-e4d3-a374e22998a4') +[110] 31562.7502 ('2045-02-27 11:46:14.976','74116384-cb3e-eb00-0102-fb30ddea5d5f') +[114] -84125.1554 ('2023-06-06 06:55:06.492','bf9ab359-ef9f-ad11-7e6c-160368b1e5ea') +[1] -18736.7874 ('1977-03-10 04:41:16.215','3259d377-a92d-3557-9045-4ad1294d55d5') +[] 204013.7193 ('2026-05-05 05:20:23.160','30f6d580-cb25-8d4f-f869-fc10128b3389') +[124] -114719.5228 ('2010-11-11 22:57:23.722','c1046ffb-3415-cc3a-509a-e0005856d7d7') +[34,-10] -99367.9009 ('2031-05-08 10:00:41.084','0b38ebc5-20a6-be3d-8543-23ce3546f49c') +[] -27467.1221 ('2021-03-08 03:39:14.331','08ec773f-cded-8c46-727f-954768082cbf') +[-1] 58720.0591 ('1976-06-07 23:26:18.162','fc038af0-ba31-8fdc-1847-37328ef161b0') +[-30,61] -133488.2399 ('2048-05-14 09:05:06.021','a6af106c-b321-978b-fa79-338c9e342b5a') - -[] 3608695403 ZL 109414.2847 h 2.2986075276244747e306 ('1985-05-10','2009-10-28 20:06:11','1993-01-03 17:51:52.981','b13ff007-c245-d737-85b2-1fa003e57127') . -[85] 4204173796 ], -199466.5471 h 1.1231803213254798e308 ('2075-04-03','1983-02-12 23:57:05','2060-06-06 20:15:08.751','a2f2cbf4-b11b-6976-7b91-14b6964acbe2') * -[-94,100] 32713522 8D$ 102255.5602 h 1.738807291208415e308 ('2029-07-12','2056-08-07 23:18:32','2081-01-25 13:13:30.589','445a77b5-0a27-3485-8dd8-c7cc35d2692f') -[] 4117557956 0b>+ 65942.4942 w 5.949505844751135e307 ('2048-03-05','2074-01-22 02:32:44','2073-12-04 05:05:06.955','c12095e6-b82c-d81c-4629-acd80e02b080')  -[] 1511604199 Il= -96352.6064 o 1.6472659147355216e308 ('2024-06-01','2024-12-26 00:54:40','2038-04-14 05:21:44.387','ebbbe70a-0321-ff18-89de-2bc9a9e4c454') Q -[-18] 2278197196 ~ 193977.7666 o 1.213689191969361e308 ('2060-10-04','1992-10-24 16:31:53','1983-06-10 08:51:48.294','805b0a62-9ada-a47e-2d5e-63cb5923549c') \t -[] 3761265784 N"(6 -59230.0369 o 1.2102282609858645e308 ('2106-02-07','2060-07-09 20:14:59','2007-03-17 04:51:09.288','429df3a4-ff18-28d5-9ad8-dcdd78e8b1ae') Y| -[] 66606254 6x&+ 130635.2269 o 1.1958868988757417e308 ('2088-10-07','2070-03-01 21:30:45','1978-05-22 14:28:52.523','d63c5cbb-9418-ce59-000c-056f88157bfa') у -[-27,-12] 4089193163 )+.8 -111081.7896 o 1.464035857434812e308 ('2106-02-07','2007-04-27 23:04:36','1987-07-21 04:32:01.821','2bc4860a-7214-300a-851e-b61011c346ef') # -[14,-43] 3638976325 #" 116961.4294 o 9.260305126207595e307 ('2042-06-11','2087-12-28 00:21:16','2071-04-01 21:44:13.058','c00d218a-913f-b657-1ff9-99927741f7ab') Fx +[] 1900051923 { -189530.5846 h -5.6279699579452485e47 ('1984-12-06','2028-08-17 06:05:01','2036-04-02 23:52:28.468','4b3d498c-dd44-95c1-5b75-921504ec5d8d') F743 +[55] 3047524030 li&lF 93462.3661 h 2.8979254388809897e54 ('1976-01-10','1987-07-14 00:25:51','2021-11-19 04:44:08.986','486e5b26-5fe8-fe3e-12ef-09aee40643e0') 9E75 +[-23] 2514120753 (`u, -119659.6174 w 1.3231258347475906e34 ('2106-02-07','2074-08-10 06:25:12','1976-12-04 18:31:55.745','86a9b3c1-4593-4d56-7762-3aa1dd22cbbf') AD43 +[100,-42] 3999367674 -112975.9852 h 2.658098863752086e-160 ('2081-05-13','2071-08-07 13:34:33','1980-11-11 12:00:44.669','9754e8ac-5145-befb-63d9-a12dd1cf1f3a') DF63 +[-71] 775049089 \N -158115.1178 w 4.1323844687113747e-305 ('2106-02-07','2090-07-31 16:45:26','2076-07-10 09:11:06.385','57c69bc6-dddd-0975-e932-a7b5173a1304') EB1D +[48,-120] 3848918261 1 Date: Sun, 8 Mar 2020 01:59:56 +0300 Subject: [PATCH 123/712] Update JEMalloc just in case --- contrib/jemalloc | 2 +- .../internal/jemalloc_internal_defs.h | 6 +++++ .../jemalloc/internal/jemalloc_preamble.h | 22 +++++++++++++++++++ .../jemalloc/jemalloc_macros.h | 18 ++++++++++----- .../internal/jemalloc_internal_defs.h | 6 +++++ .../jemalloc/internal/jemalloc_preamble.h | 22 +++++++++++++++++++ .../jemalloc/jemalloc_macros.h | 17 +++++++++----- 7 files changed, 81 insertions(+), 12 deletions(-) diff --git a/contrib/jemalloc b/contrib/jemalloc index cd2931ad9bb..ea6b3e973b4 160000 --- a/contrib/jemalloc +++ b/contrib/jemalloc @@ -1 +1 @@ -Subproject commit cd2931ad9bbd78208565716ab102e86d858c2fff +Subproject commit ea6b3e973b477b8061e0076bb257dbd7f3faa756 diff --git a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h index 9c46a3a9320..5e598348e72 100644 --- a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h +++ b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h @@ -379,4 +379,10 @@ */ #define JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE +/* + * popcount*() functions to use for bitmapping. + */ +#define JEMALLOC_INTERNAL_POPCOUNTL __builtin_popcountl +#define JEMALLOC_INTERNAL_POPCOUNT __builtin_popcount + #endif /* JEMALLOC_INTERNAL_DEFS_H_ */ diff --git a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_preamble.h b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_preamble.h index c150785fb4a..d79551e1f25 100644 --- a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_preamble.h +++ b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_preamble.h @@ -191,4 +191,26 @@ static const bool have_background_thread = #endif ; +#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS 1 +#define JEMALLOC_GCC_U8_SYNC_ATOMICS 1 + +/* + * Are extra safety checks enabled; things like checking the size of sized + * deallocations, double-frees, etc. + */ +static const bool config_opt_safety_checks = +#ifdef JEMALLOC_OPT_SAFETY_CHECKS + true +#elif defined(JEMALLOC_DEBUG) + /* + * This lets us only guard safety checks by one flag instead of two; fast + * checks can guard solely by config_opt_safety_checks and run in debug mode + * too. + */ + true +#else + false +#endif + ; + #endif /* JEMALLOC_PREAMBLE_H */ diff --git a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/jemalloc_macros.h b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/jemalloc_macros.h index 79b13337fbb..34235894285 100644 --- a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/jemalloc_macros.h +++ b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/jemalloc_macros.h @@ -4,13 +4,13 @@ #include #include -#define JEMALLOC_VERSION "5.1.0-97-gcd2931ad9bbd78208565716ab102e86d858c2fff" +#define JEMALLOC_VERSION "5.2.1-0-gea6b3e973b477b8061e0076bb257dbd7f3faa756" #define JEMALLOC_VERSION_MAJOR 5 -#define JEMALLOC_VERSION_MINOR 1 -#define JEMALLOC_VERSION_BUGFIX 0 -#define JEMALLOC_VERSION_NREV 97 -#define JEMALLOC_VERSION_GID "cd2931ad9bbd78208565716ab102e86d858c2fff" -#define JEMALLOC_VERSION_GID_IDENT cd2931ad9bbd78208565716ab102e86d858c2fff +#define JEMALLOC_VERSION_MINOR 2 +#define JEMALLOC_VERSION_BUGFIX 1 +#define JEMALLOC_VERSION_NREV 0 +#define JEMALLOC_VERSION_GID "ea6b3e973b477b8061e0076bb257dbd7f3faa756" +#define JEMALLOC_VERSION_GID_IDENT ea6b3e973b477b8061e0076bb257dbd7f3faa756 #define MALLOCX_LG_ALIGN(la) ((int)(la)) #if LG_SIZEOF_PTR == 2 @@ -69,6 +69,7 @@ # define JEMALLOC_EXPORT __declspec(dllimport) # endif # endif +# define JEMALLOC_FORMAT_ARG(i) # define JEMALLOC_FORMAT_PRINTF(s, i) # define JEMALLOC_NOINLINE __declspec(noinline) # ifdef __cplusplus @@ -96,6 +97,11 @@ # ifndef JEMALLOC_EXPORT # define JEMALLOC_EXPORT JEMALLOC_ATTR(visibility("default")) # endif +# ifdef JEMALLOC_HAVE_ATTR_FORMAT_ARG +# define JEMALLOC_FORMAT_ARG(i) JEMALLOC_ATTR(__format_arg__(3)) +# else +# define JEMALLOC_FORMAT_ARG(i) +# endif # ifdef JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF # define JEMALLOC_FORMAT_PRINTF(s, i) JEMALLOC_ATTR(format(gnu_printf, s, i)) # elif defined(JEMALLOC_HAVE_ATTR_FORMAT_PRINTF) diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h index 43936e8eba0..7c21fa79397 100644 --- a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h +++ b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h @@ -370,4 +370,10 @@ */ #define JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE +/* + * popcount*() functions to use for bitmapping. + */ +#define JEMALLOC_INTERNAL_POPCOUNTL __builtin_popcountl +#define JEMALLOC_INTERNAL_POPCOUNT __builtin_popcount + #endif /* JEMALLOC_INTERNAL_DEFS_H_ */ diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_preamble.h b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_preamble.h index c150785fb4a..d79551e1f25 100644 --- a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_preamble.h +++ b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_preamble.h @@ -191,4 +191,26 @@ static const bool have_background_thread = #endif ; +#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS 1 +#define JEMALLOC_GCC_U8_SYNC_ATOMICS 1 + +/* + * Are extra safety checks enabled; things like checking the size of sized + * deallocations, double-frees, etc. + */ +static const bool config_opt_safety_checks = +#ifdef JEMALLOC_OPT_SAFETY_CHECKS + true +#elif defined(JEMALLOC_DEBUG) + /* + * This lets us only guard safety checks by one flag instead of two; fast + * checks can guard solely by config_opt_safety_checks and run in debug mode + * too. + */ + true +#else + false +#endif + ; + #endif /* JEMALLOC_PREAMBLE_H */ diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_macros.h b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_macros.h index 7432f1cda53..34235894285 100644 --- a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_macros.h +++ b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_macros.h @@ -4,12 +4,13 @@ #include #include -#define JEMALLOC_VERSION "5.1.0-56-g41b7372eadee941b9164751b8d4963f915d3ceae" +#define JEMALLOC_VERSION "5.2.1-0-gea6b3e973b477b8061e0076bb257dbd7f3faa756" #define JEMALLOC_VERSION_MAJOR 5 -#define JEMALLOC_VERSION_MINOR 1 -#define JEMALLOC_VERSION_BUGFIX 0 -#define JEMALLOC_VERSION_NREV 56 -#define JEMALLOC_VERSION_GID "41b7372eadee941b9164751b8d4963f915d3ceae" +#define JEMALLOC_VERSION_MINOR 2 +#define JEMALLOC_VERSION_BUGFIX 1 +#define JEMALLOC_VERSION_NREV 0 +#define JEMALLOC_VERSION_GID "ea6b3e973b477b8061e0076bb257dbd7f3faa756" +#define JEMALLOC_VERSION_GID_IDENT ea6b3e973b477b8061e0076bb257dbd7f3faa756 #define MALLOCX_LG_ALIGN(la) ((int)(la)) #if LG_SIZEOF_PTR == 2 @@ -68,6 +69,7 @@ # define JEMALLOC_EXPORT __declspec(dllimport) # endif # endif +# define JEMALLOC_FORMAT_ARG(i) # define JEMALLOC_FORMAT_PRINTF(s, i) # define JEMALLOC_NOINLINE __declspec(noinline) # ifdef __cplusplus @@ -95,6 +97,11 @@ # ifndef JEMALLOC_EXPORT # define JEMALLOC_EXPORT JEMALLOC_ATTR(visibility("default")) # endif +# ifdef JEMALLOC_HAVE_ATTR_FORMAT_ARG +# define JEMALLOC_FORMAT_ARG(i) JEMALLOC_ATTR(__format_arg__(3)) +# else +# define JEMALLOC_FORMAT_ARG(i) +# endif # ifdef JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF # define JEMALLOC_FORMAT_PRINTF(s, i) JEMALLOC_ATTR(format(gnu_printf, s, i)) # elif defined(JEMALLOC_HAVE_ATTR_FORMAT_PRINTF) From 20eeb4651ec8ab5837d662785b01e8ca6794dd1f Mon Sep 17 00:00:00 2001 From: Denis Zhuravlev Date: Sat, 7 Mar 2020 19:35:27 -0400 Subject: [PATCH 124/712] Update adopters.md Fixed https://lifestreet.com/ URL --- docs/en/introduction/adopters.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/introduction/adopters.md b/docs/en/introduction/adopters.md index d1c7a35cead..8d103cb89ea 100644 --- a/docs/en/introduction/adopters.md +++ b/docs/en/introduction/adopters.md @@ -36,7 +36,7 @@ | [Integros](https://integros.com) | Platform for video services | Analytics | — | — | [Slides in Russian, May 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup22/strategies.pdf) | | [Kodiak Data](https://www.kodiakdata.com/) | Clouds | Main product | — | — | [Slides in Engish, April 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup13/kodiak_data.pdf) | | [Kontur](https://kontur.ru) | Software Development | Metrics | — | — | [Talk in Russian, November 2018](https://www.youtube.com/watch?v=U4u4Bd0FtrY) | -| [LifeStreet](https://cloudflare.com) | Ad network | Main product | 60 servers in 3 replicas | 2-2.5 PiB | [Blog post in Russian, February 2017](https://habr.com/en/post/322620/) | +| [LifeStreet](https://lifestreet.com/) | Ad network | Main product | 60 servers in 3 replicas | 2-2.5 PiB | [Blog post in Russian, February 2017](https://habr.com/en/post/322620/) | | [Mail.ru Cloud Solutions](https://mcs.mail.ru/) | Cloud services | Main product | — | — | [Running ClickHouse Instance, in Russian](https://mcs.mail.ru/help/db-create/clickhouse#) | | [MessageBird](https://www.messagebird.com) | Telecommunications | Statistics | — | — | [Slides in English, November 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup20/messagebird.pdf) | | [MGID](https://www.mgid.com/) | Ad network | Web-analytics | — | — | [Our experience in implementing analytical DBMS ClickHouse, in Russian](http://gs-studio.com/news-about-it/32777----clickhouse---c) | From 6f8a8bb21e492ad3ff0d1dd119074435d087abc2 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 8 Mar 2020 04:24:13 +0300 Subject: [PATCH 125/712] Updated test --- .../01087_storage_generate.reference | 200 +++++++++--------- .../0_stateless/01087_storage_generate.sql | 2 +- .../01087_table_function_generate.reference | 24 +-- .../01087_table_function_generate.sql | 4 +- 4 files changed, 115 insertions(+), 115 deletions(-) diff --git a/dbms/tests/queries/0_stateless/01087_storage_generate.reference b/dbms/tests/queries/0_stateless/01087_storage_generate.reference index 33e5fdd18ee..3680d8d943d 100644 --- a/dbms/tests/queries/0_stateless/01087_storage_generate.reference +++ b/dbms/tests/queries/0_stateless/01087_storage_generate.reference @@ -1,103 +1,103 @@ 100 - -[] -183162.1041 ('2074-01-15 19:36:42.413','0345f8ad-8936-8cc9-9ff2-394f225fc318') -[3] -39049.0845 ('2045-07-04 15:01:09.380','f79d47d8-4030-9916-54b9-495a5ccc7202') -[-95,-104] -14324.1685 ('2023-10-02 06:05:53.887','ea94157b-737b-a272-acd5-c7ab9c6f07c3') -[] -182420.0194 ('2002-01-08 01:42:50.396','ac579c0f-8523-144f-aa4c-c9587cc27144') -[] 127157.2100 ('2006-11-23 14:25:39.542','d07c5204-ef95-6804-83df-01dedaf32522') -[28] -198600.4267 ('2060-09-25 02:57:41.504','b13ff007-c245-d737-85b2-1fa003e57127') -[] -87232.0739 ('2027-05-12 20:26:59.405','a2f2cbf4-b11b-6976-7b91-14b6964acbe2') -[] -110349.8987 ('2042-11-01 10:51:30.039','445a77b5-0a27-3485-8dd8-c7cc35d2692f') -[32,73] 123253.5669 ('2016-06-21 00:23:02.917','c12095e6-b82c-d81c-4629-acd80e02b080') -[-109,85] 34320.8302 ('2080-12-31 16:49:32.509','ebbbe70a-0321-ff18-89de-2bc9a9e4c454') -[68,76] 160458.5593 ('2030-05-23 03:33:29.681','805b0a62-9ada-a47e-2d5e-63cb5923549c') -[69] -189322.2887 ('2084-06-19 03:31:58.508','429df3a4-ff18-28d5-9ad8-dcdd78e8b1ae') -[] 189158.4731 ('1985-12-21 16:36:51.092','d63c5cbb-9418-ce59-000c-056f88157bfa') -[-120] 114890.5905 ('2089-02-19 22:30:18.216','2bc4860a-7214-300a-851e-b61011c346ef') -[] 54493.1631 ('2062-08-18 04:57:01.348','c00d218a-913f-b657-1ff9-99927741f7ab') -[103] 88891.6006 ('2071-05-23 21:46:45.644','036d2746-f7aa-b5a4-b716-b8e8a5e041da') -[28] -41173.9863 ('2044-03-11 10:06:47.659','6bf54ef0-9bad-54d4-5ca3-02d79883b697') -[] -64809.0652 ('2010-11-02 23:46:46.150','ae5cafb4-fe3c-71a5-9a76-0314c44180de') -[125,74] 28139.7661 ('1989-07-19 22:10:13.477','ee9b8173-4426-8615-97eb-a904266847e5') -[-101] -29032.2527 ('2052-08-13 08:55:15.045','ab8ee358-ff53-de7e-f012-cd0eed754ff2') -[82] 170334.6151 ('2034-02-23 18:50:46.847','f1402791-1d23-e56d-25f0-5a51a3cb245a') -[27,56] 168648.5067 ('2004-05-31 19:47:19.448','5019365e-f74d-b31e-aacb-63d8189e3e3e') -[-74] 89577.4738 ('1997-01-15 09:05:57.691','d144325c-24a9-411e-cc10-78b0637f75a7') -[100] 178532.5772 ('2055-04-22 16:20:11.498','41439c27-fba1-1b66-13a7-cf79fded4d9a') -[32] 123101.7871 ('2103-02-18 15:53:42.748','25b804d3-a73e-ed14-e6e1-eafb0d9473cd') -[-115,-85] -208371.1662 ('2039-10-22 18:06:50.235','41fba85a-5080-48bb-e18d-a8af04a890aa') -[-12,101] -7791.5577 ('2073-07-27 11:35:45.239','c00be55f-59ac-762c-af0a-9e33cf30a1f4') -[-127] -18602.1918 ('2024-02-05 19:54:00.798','ea85cbed-66f2-197b-4e63-dfbdcd306cce') -[-78,125] -112158.3556 ('2016-08-12 06:46:17.173','15809e25-b003-010b-c63c-9e880568736a') -[] 151191.1081 ('1982-11-12 17:59:10.171','d6bbbe2c-fca0-53c8-22a6-de9e0715d3cc') -[46] 58689.9611 ('2085-04-13 09:13:07.230','fe4be382-eb78-4cf9-fa57-c6eccf955419') -[-4,42] -88292.1046 ('1980-03-06 08:29:12.503','2633522e-ff9c-b837-1b9b-6559875c13b0') -[-2] 64983.6649 ('2034-07-07 11:20:23.903','d19f5c4d-e444-2e5b-a55d-5280d1760b94') -[-31,-110] -25638.6649 ('2025-05-17 21:45:25.519','3654a15e-bfa3-6075-b5b8-07e25310de1f') -[25] -140469.2476 ('2083-12-13 23:55:25.450','940f7441-ae40-d810-f6c3-e2fff468050c') -[-99] 128186.7318 ('1995-09-15 04:26:33.803','4cfd264f-ff00-4190-929c-b675826607d3') -[] -52961.0340 ('2046-03-19 14:15:50.245','314de821-308b-c61c-e256-9f6afed5d4f3') -[] -155852.9334 ('2052-01-29 06:31:08.957','5be2ccd2-b5bb-921d-5b5e-4a0e22385de7') -[-74,81] 29366.0091 ('1978-03-05 19:24:49.193','ab9b6a39-89ac-9280-c76f-60d598ce65c6') -[9] 56134.8951 ('2104-04-03 10:27:33.053','339a7f6d-0e0b-e039-78c0-2d045457d821') -[-61] 68841.1188 ('2059-07-26 12:14:33.197','c817bbb3-d091-b73c-1b9a-53f8a03bffb6') -[89] 168034.0459 ('2028-05-29 08:02:02.393','bc217a73-e802-1772-80b5-d8c827295799') -[124] 11648.6762 ('2084-12-25 12:10:35.676','77390177-1dd6-a5c0-dd35-4f85e38bcb2c') -[-47,-125] -120893.6705 ('2012-10-18 22:52:57.524','472814b2-4033-c5a5-7d86-fb36079e88fb') -[35] 153250.6252 ('2006-11-06 00:05:25.456','de0d6ed9-eca6-e01e-eb1c-c46c8ad6e33e') -[-43,70] -141086.3184 ('2013-02-03 23:07:11.759','65d48b24-cdc0-f7db-cb16-d0ad03279bcc') -[120,-57] -93351.1404 ('2000-02-03 14:39:00.466','6991722b-90dc-e9dd-c5e7-f28bd1d4f0d8') -[34,43] 187780.4567 ('2014-02-10 05:22:19.250','3db77bc5-d877-b22e-6667-955bf36d2e08') -[73] -90148.5697 ('2014-10-05 18:34:31.419','5a0f919e-38c9-0a68-e805-977db04d0acb') -[] -179121.0029 ('2077-01-23 07:57:55.365','fcf79336-a6dc-44fd-8c78-7e74e07b60fa') -[-69,120] 119321.8003 ('1989-07-01 13:11:35.185','92f6a362-250c-cfcd-acd7-99399cbf88ad') -[] 208864.8324 ('1991-02-17 03:04:00.682','b0dc8e88-ea6f-c2da-c116-3e4873dc8d54') -[22,-14] -127735.4391 ('2036-08-10 08:33:03.806','5ab1ab2b-913d-ff8a-6f8f-86387e77ed5c') -[83,-70] -142476.9847 ('2074-11-22 19:27:13.085','51b9d30a-3b10-265c-4086-1ac35b634ec7') -[] -128052.2443 ('2088-01-02 10:58:36.999','745e8226-d906-7fb3-33f4-9a079037bdcd') -[12,-116] -88390.1399 ('2074-02-18 17:46:45.208','fb5f827e-1809-6cab-2855-d45df20ecd92') -[] -84110.2097 ('2039-03-24 17:08:15.660','88e18c93-6276-d176-dad1-7db72e340ca7') -[] 202866.8175 ('2104-01-25 13:42:41.758','10faa33e-d383-c6b3-399d-44c06ebb00f5') -[-21] 151775.1601 ('1995-10-20 15:44:53.296','7ccaf135-787d-2ac0-09c0-7545c798ee14') -[-19] -15498.5738 ('2097-08-02 18:34:16.406','cf97f268-02c0-24fc-bbf3-c7b272632c14') -[116] -72670.9713 ('2020-08-31 18:10:41.904','f9cdd931-e2ed-0584-d4b9-67a6df717a4c') -[] 124014.7040 ('1975-07-23 11:17:25.176','ccf33ba5-8fd8-c8b5-ccc4-a9cb892d4b55') -[-56] -204745.8115 ('2037-11-13 01:03:12.923','6dc83c7b-7782-57b4-a293-18ca8aba331d') -[] -28535.2534 ('2105-04-07 20:51:09.990','0d9f3a2f-d4f2-a330-7b6e-001ea3aacbde') -[-124,-128] -31519.7583 ('1993-02-14 23:06:10.338','a073dafb-6f1f-273e-acf9-88200f82af6d') -[46] -154950.9257 ('2032-06-04 23:16:16.051','e6aa3b80-9f53-6c10-0cc8-622622f964b4') -[] 206914.3454 ('2003-10-05 10:44:30.786','137ed3be-2d40-d1c1-7aff-b32f7e21c0da') -[-47] 91521.1349 ('2006-09-01 04:06:32.496','52e4ef43-9379-4864-8f63-8e205875a096') -[121] 161456.7813 ('2027-11-03 10:20:30.670','f1abbd17-f399-657c-1a47-1dd627578b53') -[99] -127959.4741 ('2084-08-18 06:04:41.942','2a3b92c3-75ed-bd20-5a77-b77cbe1ce479') -[-97] 82020.4570 ('2061-10-25 06:16:50.814','8625d479-6e81-318f-5077-a9deb13c50e0') -[71] -121599.1388 ('2010-04-02 11:05:18.877','0ec279cf-c9b2-dc65-40c0-2d0f390b1102') -[] 98975.6469 ('2049-03-06 08:56:25.010','845340d7-a1df-9ddf-b737-9eb90ca6344c') -[92,81] 135864.7854 ('2040-12-30 21:17:28.184','ea224755-198e-c9ae-c59b-0517a7459d7c') -[81] -154620.5037 ('1984-06-07 02:36:28.734','52d3b727-043f-1d43-6f48-51e8abdc2127') -[38] 33379.3375 ('2057-10-19 17:03:44.317','e709bfc2-0915-9e4e-4d01-c10b24795e30') -[] 7491.1071 ('1971-04-29 09:30:25.245','26bcd2ab-6d0b-fc20-27eb-084c4248af7d') -[-122] -135635.3813 ('2010-03-04 23:05:25.982','66ed96eb-fc6e-653e-0353-ac4477ea60a6') -[] -174748.4115 ('2020-10-28 07:39:33.461','e17fa9ba-2595-c0f9-2f85-d6bbdc2f6f6a') -[72,106] 25749.2190 ('2008-06-15 04:03:39.682','0e47b616-da80-091e-664d-2a35bc57a480') -[-84,97] 109277.9244 ('1998-10-27 10:40:00.442','9488bce4-46d7-8249-78aa-540b8be43937') -[-120,-107] -64113.5210 ('2091-12-03 06:46:11.903','325fcb1c-8552-b434-b349-732d62be19f1') -[] -66141.6000 ('2085-10-05 08:08:11.830','4c66022b-75b9-b0a8-3897-b9de8ea851f1') -[-34,-102] -142314.4437 ('2038-01-27 12:04:29.739','91e9eb11-5679-02ef-6ea6-2c9fdcb12ed9') -[103] 96187.7213 ('1978-10-07 13:57:43.616','7c02e8e3-9e98-5043-8029-34e32ad1af61') -[] -21344.8423 ('2085-01-13 00:10:52.538','52cb36f8-987a-f414-7e0f-93ddccc5c377') -[16] -95098.4107 ('2074-02-19 18:56:00.878','821e4b10-f70a-4bee-ef0c-ac12eab994f3') -[21,86] 27954.7748 ('2033-10-18 03:15:38.815','bfe4d932-c5ed-45c0-9f50-72a6394d49af') -[] 149788.2085 ('2073-09-10 20:42:48.693','5e7d825e-5c88-7c89-4235-0e7934739a12') -[33,116] -148302.8732 ('2044-08-10 22:05:18.943','a53d4b07-5529-7472-3cca-3770f52b3648') -[] -98384.4505 ('2070-01-28 05:17:35.804','4833b839-51a3-87b8-7709-30676f697aa4') -[] -75597.1523 ('2075-02-04 19:24:01.477','d64becff-5c08-b0a0-e7f1-b86eaf5f1913') -[] 179005.6113 ('2100-05-27 21:54:12.965','d87ce81c-c471-b6b3-93b7-05225cb577be') -[] -134366.9213 ('2054-11-16 18:19:00.801','c348fced-6700-f0f6-cda0-14aef7ea6948') -[10] 82182.0343 ('2017-03-04 09:41:21.249','e19f0022-49ab-2d41-872d-be35669a79bc') -[-28] 90333.8564 ('2032-11-19 01:23:37.107','e2586be2-e968-21d0-d1b1-b438c55a59a3') -[-73] 185647.6735 ('2001-01-23 16:20:26.442','24b04f39-f272-24ff-538d-41e636a1a37a') -[-79,7] -87628.8007 ('2005-03-25 04:17:49.969','38a10e9d-7086-f358-8e50-c72b278bec42') -[119,-55] -208591.8591 ('1976-11-14 15:17:57.569','d0935dc7-7f56-71db-67f2-1b4e52770ba9') -[-108,-124] 181408.0349 ('2056-10-27 05:07:32.393','29d655c1-c35a-1245-25e2-65b4f233cb9c') +[] -54259.6828 ('2088-03-01 16:26:24.094','d3c2a216-a98c-d56c-7bf7-62de9f264cf4') +[88] 34528.4014 ('2031-12-09 00:40:39.898','9ef777c8-de0e-d25e-e16c-5b624f88523c') +[-1] 121968.7945 ('2060-02-05 09:18:12.011','7655e515-d2ca-2f06-0950-e4f44f69aca7') +[-103,75] -135033.4349 ('2038-12-19 20:38:58.695','86b57d15-292d-2517-9acf-47cd053e7a3a') +[110] -202668.6900 ('2009-06-18 01:53:29.808','bc630f78-7d58-0c46-dd4b-27fc35625e96') +[-22,2] 168636.9728 ('2074-09-03 09:20:20.936','7624ce27-9bff-4e9d-3f18-6851a97dd0ca') +[-22,-62] -75192.4989 ('2085-10-11 21:51:12.855','a4c4d0ed-f448-244e-1723-ca1bba816f2b') +[-2,-90] 133592.5064 ('2010-10-28 21:18:04.633','8ba9103b-f90c-b49b-38c1-223ae5f42bf7') +[-94,80] 197330.6359 ('2024-03-30 22:08:45.772','83442013-3677-5097-065d-72dfbe8a3506') +[23] 167557.6237 ('2078-07-25 21:54:42.480','be14d98e-5b24-54ee-c959-d24fa9a58fdd') +[46,-10,-63] 185107.1979 ('2040-10-07 06:06:53.504','5ed1fe6a-9313-41d7-4bf9-3948e961509f') +[-107,68] -163781.3045 ('2021-12-21 19:18:58.933','7b634f19-0863-829e-484b-be288aab54a1') +[-35,-116,73] -203577.5379 ('2093-08-01 20:21:09.407','d371bad4-b098-ffdd-f84c-6a02390c2939') +[61] 152284.9386 ('2089-12-20 19:21:33.149','9e8426c1-278a-4d9c-4076-364a95b065e3') +[75] 170968.4171 ('2020-07-17 15:45:31.975','47397a81-bda7-8bd9-59f7-d60e2204fe99') +[-115,93] -173740.5652 ('2098-04-25 22:10:33.327','117e31dd-102e-ee6c-0dbd-0a4203c18ca5') +[-20,4,21] 63834.8685 ('2000-07-08 18:09:40.271','10b0fa48-55a3-755a-4a44-36315ae04c1c') +[-110,117,91] -160640.1506 ('1998-04-18 10:58:04.479','6dfa3a8e-6e65-543c-5f50-1ff45835aa5a') +[62] 63817.7977 ('2043-01-24 02:07:18.972','98b8ef31-4f65-2f8b-1ea7-b1473900099e') +[-2] -175477.0173 ('2007-01-16 07:46:14.781','ec92f616-6e1f-003a-54c6-c5f9118d2f1b') +[] 197663.3035 ('2046-06-30 17:04:56.788','fb3244a4-8af2-104f-2a6f-25a7b7b9a112') +[-24] -174299.4691 ('2058-02-23 14:50:58.839','d63ee868-fa93-bf8b-0264-8ebbceb13e3b') +[95,38] -65083.7371 ('2015-03-10 13:33:16.429','47bd199c-f99e-51ea-84e9-b65cce9d167c') +[91,110,72] 130908.9643 ('2036-03-16 15:17:53.679','0dd4ca31-1e09-d7e0-f3df-60cad3cfa805') +[] 208972.3779 ('2034-03-05 22:29:21.994','1069d77c-dfd2-912e-60b8-3c5b964f7e11') +[-32] 167938.5050 ('2093-09-10 20:39:39.050','9d1025b6-2d0c-1d84-dafd-02668eb29270') +[] 153744.6987 ('2088-10-02 11:02:11.024','a88e6cb7-2210-5ce5-6bcf-24afc0eca5b6') +[67] -74220.6650 ('2074-12-30 18:43:40.817','68096065-18c8-8aca-fd21-15330ead669d') +[6] 66759.8938 ('2091-09-01 19:07:18.219','bb14f4cc-0b54-9a8c-e835-71333b28c03b') +[-28,-82,9] 168625.3131 ('2002-03-20 21:02:30.321','405bb877-6e28-8b91-cb62-bd82a3fa797c') +[] -19760.1670 ('2044-11-08 07:52:03.325','13769348-9e58-0e75-3972-8bbadc150715') +[] 160663.7797 ('2025-04-12 13:17:53.501','e6370321-94f5-97e6-0348-a84e72ff5b42') +[-17,18] 99105.9856 ('1972-05-01 12:23:11.688','02618b9e-97cd-4698-d2e8-3f52f4c5a09a') +[86,77] -116990.3914 ('1981-12-31 05:06:54.198','3ac42bb4-8652-b1a8-10bb-98f0337261f8') +[-109,69,-63] -151527.3587 ('2001-01-17 11:19:56.504','77fe7ee2-f279-2855-bfd2-a7d7cee678cc') +[] -57762.3928 ('1978-08-16 18:47:37.660','ab9a110a-fd8d-3c4c-5a49-34c2005536ce') +[-77] 107274.6407 ('2017-01-12 12:03:02.657','c1ad4f17-cc54-45f3-9410-9c1011653f6d') +[] 107133.6410 ('2050-10-05 06:29:27.154','36e576aa-c77f-994e-1925-4a4c40da3a0f') +[] 46672.2176 ('2094-01-21 20:25:39.144','e9ba850d-604e-bc7d-417c-1078e89d4615') +[-87,-122,-65] -86258.4663 ('2081-06-17 03:37:45.498','64795221-9719-7937-b4d2-be5f30065ece') +[-53] -48672.1424 ('1992-06-27 17:27:23.602','7c67bc31-c7bb-6197-fdca-f73329b976f2') +[34] -108954.7820 ('2096-07-03 23:06:30.632','9c1b37d7-4ced-9428-a0ae-34c5436b14c4') +[] -168124.2364 ('1987-06-03 06:47:12.945','d1c39af4-f920-5095-b8e2-0f878950167b') +[] -112431.4799 ('2021-07-26 07:04:58.527','da07a72d-7e1f-8890-4c4b-326835d11b39') +[-35,-95,58] -181254.9139 ('2086-11-12 17:17:14.473','22f74d0b-dfc0-3f7a-33f4-8055d8fa7846') +[98,119] 11468.5238 ('2092-02-25 11:07:07.695','a1fb97bf-1885-6715-c233-b88a6cd111e4') +[] 82333.8963 ('1989-11-23 01:38:57.012','a2b82b5b-8331-555c-579b-de4b0eeb7e81') +[-5,-66,69] 32055.8376 ('2040-12-17 16:49:08.704','4537d25e-a2db-ea9a-8e24-a16ed7e0c6e4') +[81,-84,-24] -210815.2512 ('2047-06-09 13:30:06.922','ac3c5b5f-f977-2830-c398-d10a6076a498') +[84,-105] -175413.7733 ('1998-11-03 04:30:21.191','c535feac-1943-c0a1-23f0-645d5406db24') +[58,31] -335.8512 ('1973-07-09 12:21:10.444','24a7dd3d-2565-1de3-05d9-e45fd8ba7729') +[-49,-47] 177399.2836 ('2049-03-15 15:33:00.190','e4432b9b-61e9-d451-dc87-ae3b9da6fd35') +[] 211525.2349 ('2106-01-11 10:44:18.918','23315435-7132-05b5-5a9b-c2c738433a87') +[45,-95,-39] -15314.9732 ('2055-10-29 13:51:12.182','833b2efa-8c72-f5f6-3040-cb4831e8ceb9') +[] 213384.5774 ('2067-02-10 22:02:42.113','0cd7f438-caa7-0d21-867c-1fdb6d67d797') +[99] -147316.5599 ('2000-05-09 21:37:34.776','a3ea6796-38d5-72ff-910d-8b4300831916') +[] 8828.2471 ('1993-11-30 16:53:22.503','7209213f-38bb-cfed-1955-f1fad5a9577a') +[117,9,-35] -134812.6269 ('2065-09-04 23:47:26.589','d33d0d6f-b9c0-2850-4593-cfc9f1e20a4d') +[-35,-58,-101] -9101.5369 ('2023-08-24 20:56:11.695','87fbe3f9-b1f0-c030-a4c0-8662045923b4') +[-58,87] 122510.9099 ('2019-08-09 17:40:29.849','c1d3a2cc-878f-c2c3-4a0b-10e98cda8b4a') +[4,19,58] -13496.8672 ('2027-05-01 09:11:48.659','8996ae31-d670-cbfe-b735-b16b7c3b3476') +[23,-75,-89] -51218.2860 ('2010-06-02 02:49:03.396','d32b8b61-cc3e-31fa-2a2a-abefa60bfcee') +[50] -45297.4315 ('2087-04-15 06:46:08.247','04fe9603-97fc-07a4-6248-0f21e408c884') +[-23,17,63] 89185.9462 ('2065-10-26 08:27:12.817','a5fbf764-70b4-8b65-4a8f-7550abca3859') +[-6] -129925.3690 ('2013-11-05 07:44:45.233','11db26b3-e2b5-b9fa-6b0e-79c43a2e67ab') +[-72,-108] 203171.5475 ('2000-01-28 09:34:58.032','14d5399e-7949-20c7-0e47-85e2fce5836c') +[-73,34,-27] 2676.7265 ('2057-10-25 14:37:10.049','00049a92-4350-badb-3764-dd7f019b9b31') +[65,-7] -153472.9461 ('1973-04-12 02:34:41.245','e0a0324d-1552-d11e-f3a5-fbd822d206c5') +[] 81837.7838 ('2041-09-20 20:56:39.712','f7923f2c-e526-1706-79b9-58045d9deaa7') +[-113,8] 173192.6905 ('2066-04-02 09:59:59.356','e3013e5c-92e3-c03c-b57a-e1939e00a1a7') +[107] 9694.1102 ('1984-11-02 13:11:34.034','e973db18-07b7-2117-f3ba-e7002adfa939') +[] -76460.9664 ('2051-02-10 09:54:42.143','b8344c22-9e8a-7052-c644-9c3e5989cdf1') +[59,59,0] 27041.7606 ('2083-02-17 18:21:22.547','4d6b137b-a3e1-f36d-2c0c-c8d718dda388') +[-114] 133673.9630 ('2005-10-02 20:34:27.452','04785b75-30e5-af8b-547e-d15bcb7f49fb') +[43] -169861.2000 ('2006-12-13 09:26:13.923','cb865d38-d961-d7f9-acbb-583b9f31252f') +[] 197115.2174 ('2060-04-08 04:17:00.488','0f26c4b4-b24c-1fd5-c619-31bcf71a4831') +[-25] -200081.9506 ('2055-12-25 02:30:16.276','0b32ad69-2c84-4269-9718-e3171482878a') +[14,110] -40196.4463 ('2084-08-13 19:37:07.588','ed882071-acba-b3ab-5d77-d79a9544a834') +[-62,-71,-82] -154958.9747 ('2100-07-08 02:32:53.741','7711c7c1-0d22-e302-fc86-61ef5e68db96') +[96,-114,-101] 78910.3320 ('2100-07-19 15:02:27.109','756bfd26-c4b3-94b8-e991-c7ab7a833b76') +[49] 80117.2267 ('1970-07-04 03:50:56.748','aebac019-9054-4a77-2ccd-8801fc4a7496') +[] 102078.4801 ('2055-01-07 01:22:33.624','21f2e59a-a1ca-5df3-27fd-aa95456cfbe5') +[-106] -108728.4237 ('2020-05-27 11:56:18.121','6b7b6674-9342-2360-4cc0-f7ef8a2404de') +[] 173213.5631 ('2034-01-18 19:04:16.059','2dc0038d-67c1-f0ee-280b-f3f0f536b01a') +[42] 139872.2503 ('2001-07-16 11:09:28.754','d6487da6-1077-1053-f314-9a1079f5df15') +[] 1107.5244 ('2031-02-26 15:06:00.846','b32bee8f-85b7-3c71-bb24-9a0093e6a08c') +[] 85892.8913 ('2088-04-13 14:54:18.514','84f3b59b-8d23-78a6-3032-91392344584f') +[43] -109644.2714 ('1974-07-04 14:45:43.139','cf722ca8-15f5-6fe2-997c-0cf88e95e902') +[] 212557.3762 ('2069-03-03 07:21:08.439','9e676cac-36e6-2962-f7b1-578214f0dfbd') +[-128,55] 80471.0777 ('1970-04-01 18:54:40.257','ca358854-416b-9c95-0b9b-c7fed7bb7cb5') +[-30,-54] -132205.4512 ('2017-12-15 22:54:15.750','3558faa4-2d2f-c533-437f-1e03d3600f1d') +[-116,-72] -91499.6670 ('2105-09-23 21:06:17.755','07bb6e47-3234-c268-40d7-332388dc06f8') +[] -201636.5228 ('2085-01-27 07:54:42.717','86c3bdc3-ff0f-1723-07c2-845aa3c02370') +[-103,-39] 44330.7722 ('2064-07-02 11:08:28.068','0869c79d-6bdd-5d2d-a3d1-ffe13f6aa810') +[99] -31035.5391 ('2093-07-26 01:50:23.026','aeb59338-254f-dc09-fbd7-263da415e211') +[101] 157961.4729 ('2036-05-04 02:35:07.845','8b6221a9-8dad-4655-7460-6b3031b06893') +[111] 84732.4403 ('1997-04-06 16:10:18.624','08806a79-59f4-c833-eedc-a200bb851767') +[9,-48] -190491.5590 ('2031-11-03 19:47:03.757','914e6166-c96e-e0e4-101a-0bb516cf5a2f') +[-41] -132501.8311 ('2089-11-21 21:38:28.848','6de6cc8d-3c49-641e-fb12-87ed5ecb97b0') +[77] 64903.6579 ('1985-04-17 17:08:03.998','26484b8a-f3f1-587f-7777-bc7a57a689c3') - diff --git a/dbms/tests/queries/0_stateless/01087_storage_generate.sql b/dbms/tests/queries/0_stateless/01087_storage_generate.sql index 54ecd3007a9..bc69e8abbac 100644 --- a/dbms/tests/queries/0_stateless/01087_storage_generate.sql +++ b/dbms/tests/queries/0_stateless/01087_storage_generate.sql @@ -7,7 +7,7 @@ DROP TABLE IF EXISTS test_table; SELECT '-'; DROP TABLE IF EXISTS test_table_2; -CREATE TABLE test_table_2(a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3), UUID)) ENGINE=GenerateRandom(3, 5, 10); +CREATE TABLE test_table_2(a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3), UUID)) ENGINE=GenerateRandom(10, 5, 3); SELECT * FROM test_table_2 LIMIT 100; diff --git a/dbms/tests/queries/0_stateless/01087_table_function_generate.reference b/dbms/tests/queries/0_stateless/01087_table_function_generate.reference index 291e5b1689a..96c73c97411 100644 --- a/dbms/tests/queries/0_stateless/01087_table_function_generate.reference +++ b/dbms/tests/queries/0_stateless/01087_table_function_generate.reference @@ -214,25 +214,25 @@ U6 \'%Y~t9 RL,{Xs\\tw - +[] -27467.1221 ('2021-03-08 03:39:14.331','08ec773f-cded-8c46-727f-954768082cbf') +[] 204013.7193 ('2026-05-05 05:20:23.160','30f6d580-cb25-8d4f-f869-fc10128b3389') [-122] -9432.2617 ('2001-08-23 08:05:41.222','f7bf2154-78c3-8920-e4d3-a374e22998a4') +[-30,61] -133488.2399 ('2048-05-14 09:05:06.021','a6af106c-b321-978b-fa79-338c9e342b5a') +[-1] 58720.0591 ('1976-06-07 23:26:18.162','fc038af0-ba31-8fdc-1847-37328ef161b0') +[1] -18736.7874 ('1977-03-10 04:41:16.215','3259d377-a92d-3557-9045-4ad1294d55d5') +[34,-10] -99367.9009 ('2031-05-08 10:00:41.084','0b38ebc5-20a6-be3d-8543-23ce3546f49c') [110] 31562.7502 ('2045-02-27 11:46:14.976','74116384-cb3e-eb00-0102-fb30ddea5d5f') [114] -84125.1554 ('2023-06-06 06:55:06.492','bf9ab359-ef9f-ad11-7e6c-160368b1e5ea') -[1] -18736.7874 ('1977-03-10 04:41:16.215','3259d377-a92d-3557-9045-4ad1294d55d5') -[] 204013.7193 ('2026-05-05 05:20:23.160','30f6d580-cb25-8d4f-f869-fc10128b3389') [124] -114719.5228 ('2010-11-11 22:57:23.722','c1046ffb-3415-cc3a-509a-e0005856d7d7') -[34,-10] -99367.9009 ('2031-05-08 10:00:41.084','0b38ebc5-20a6-be3d-8543-23ce3546f49c') -[] -27467.1221 ('2021-03-08 03:39:14.331','08ec773f-cded-8c46-727f-954768082cbf') -[-1] 58720.0591 ('1976-06-07 23:26:18.162','fc038af0-ba31-8fdc-1847-37328ef161b0') -[-30,61] -133488.2399 ('2048-05-14 09:05:06.021','a6af106c-b321-978b-fa79-338c9e342b5a') - [] 1900051923 { -189530.5846 h -5.6279699579452485e47 ('1984-12-06','2028-08-17 06:05:01','2036-04-02 23:52:28.468','4b3d498c-dd44-95c1-5b75-921504ec5d8d') F743 -[55] 3047524030 li&lF 93462.3661 h 2.8979254388809897e54 ('1976-01-10','1987-07-14 00:25:51','2021-11-19 04:44:08.986','486e5b26-5fe8-fe3e-12ef-09aee40643e0') 9E75 -[-23] 2514120753 (`u, -119659.6174 w 1.3231258347475906e34 ('2106-02-07','2074-08-10 06:25:12','1976-12-04 18:31:55.745','86a9b3c1-4593-4d56-7762-3aa1dd22cbbf') AD43 -[100,-42] 3999367674 -112975.9852 h 2.658098863752086e-160 ('2081-05-13','2071-08-07 13:34:33','1980-11-11 12:00:44.669','9754e8ac-5145-befb-63d9-a12dd1cf1f3a') DF63 -[-71] 775049089 \N -158115.1178 w 4.1323844687113747e-305 ('2106-02-07','2090-07-31 16:45:26','2076-07-10 09:11:06.385','57c69bc6-dddd-0975-e932-a7b5173a1304') EB1D -[48,-120] 3848918261 1 Date: Sun, 8 Mar 2020 12:32:22 +0300 Subject: [PATCH 126/712] Fix stateless tests with msan (#9531) * try run tests * try fix build * try enable other libraries * suppress some msan warnings * Update msan_suppressions.txt * Update msan_suppressions.txt * use function names in suppressions list * update submodule --- cmake/sanitize.cmake | 21 --------------------- contrib/openssl | 2 +- dbms/tests/msan_suppressions.txt | 9 +++++++++ 3 files changed, 10 insertions(+), 22 deletions(-) diff --git a/cmake/sanitize.cmake b/cmake/sanitize.cmake index 13947425f7b..3d192f1fe76 100644 --- a/cmake/sanitize.cmake +++ b/cmake/sanitize.cmake @@ -35,27 +35,6 @@ if (SANITIZE) set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libmsan") endif () - # Temporarily disable many external libraries that don't work under - # MemorySanitizer yet. - set (ENABLE_HDFS 0 CACHE BOOL "") - set (ENABLE_CAPNP 0 CACHE BOOL "") - set (ENABLE_RDKAFKA 0 CACHE BOOL "") - set (ENABLE_POCO_MONGODB 0 CACHE BOOL "") - set (ENABLE_POCO_NETSSL 0 CACHE BOOL "") - set (ENABLE_POCO_ODBC 0 CACHE BOOL "") - set (ENABLE_ODBC 0 CACHE BOOL "") - set (ENABLE_MYSQL 0 CACHE BOOL "") - set (ENABLE_EMBEDDED_COMPILER 0 CACHE BOOL "") - set (USE_INTERNAL_CAPNP_LIBRARY 0 CACHE BOOL "") - set (USE_SIMDJSON 0 CACHE BOOL "") - set (ENABLE_ORC 0 CACHE BOOL "") - set (ENABLE_PARQUET 0 CACHE BOOL "") - set (USE_CAPNP 0 CACHE BOOL "") - set (USE_INTERNAL_ORC_LIBRARY 0 CACHE BOOL "") - set (USE_ORC 0 CACHE BOOL "") - set (USE_AVRO 0 CACHE BOOL "") - set (ENABLE_SSL 0 CACHE BOOL "") - elseif (SANITIZE STREQUAL "thread") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} -fsanitize=thread") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} -fsanitize=thread") diff --git a/contrib/openssl b/contrib/openssl index debbae80cb4..07e96230645 160000 --- a/contrib/openssl +++ b/contrib/openssl @@ -1 +1 @@ -Subproject commit debbae80cb44de55fd8040fdfbe4b506601ff2a6 +Subproject commit 07e9623064508d15dd61367f960ebe7fc9aecd77 diff --git a/dbms/tests/msan_suppressions.txt b/dbms/tests/msan_suppressions.txt index b3db90b9123..0ceef2d834c 100644 --- a/dbms/tests/msan_suppressions.txt +++ b/dbms/tests/msan_suppressions.txt @@ -4,3 +4,12 @@ fun:__gxx_personality_* # We apply std::tolower to uninitialized padding, but don't use the result, so # it is OK. Reproduce with "select ngramDistanceCaseInsensitive(materialize(''), '')" fun:tolower + +# May be it's not OK, but suppress it to run other tests +# Some functions in OpenSSL: +fun:probable_prime +fun:BN_bin2bn +fun:BN_add_word +fun:bn_div_fixed_top +fun:bn_mul_words +fun:BN_cmp From 3ed270dcb78d3b876dddd1ad11822666a694f6c7 Mon Sep 17 00:00:00 2001 From: Artem Zuikov Date: Sun, 8 Mar 2020 14:07:05 +0300 Subject: [PATCH 127/712] Rewrite CROSS/COMMA to INNER JOIN using table's columns knowledge (#9512) use column names in CrossToInnerJoinVisitor --- .../Interpreters/CrossToInnerJoinVisitor.cpp | 160 ++-- .../Interpreters/CrossToInnerJoinVisitor.h | 5 +- .../Interpreters/DatabaseAndTableWithAlias.h | 22 + .../ExtractExpressionInfoVisitor.cpp | 5 +- dbms/src/Interpreters/IdentifierSemantic.cpp | 61 +- dbms/src/Interpreters/IdentifierSemantic.h | 14 +- .../Interpreters/InterpreterSelectQuery.cpp | 84 +- .../JoinToSubqueryTransformVisitor.cpp | 10 +- dbms/src/Interpreters/JoinedTables.cpp | 53 +- dbms/src/Interpreters/JoinedTables.h | 22 +- dbms/src/Interpreters/QueryAliasesVisitor.cpp | 7 +- dbms/src/Interpreters/QueryAliasesVisitor.h | 8 +- .../TranslateQualifiedNamesVisitor.cpp | 4 +- .../01095_tpch_like_smoke.reference | 14 + .../0_stateless/01095_tpch_like_smoke.sql | 802 ++++++++++++++++++ 15 files changed, 1089 insertions(+), 182 deletions(-) create mode 100644 dbms/tests/queries/0_stateless/01095_tpch_like_smoke.reference create mode 100644 dbms/tests/queries/0_stateless/01095_tpch_like_smoke.sql diff --git a/dbms/src/Interpreters/CrossToInnerJoinVisitor.cpp b/dbms/src/Interpreters/CrossToInnerJoinVisitor.cpp index 740fe35e936..21cd688cd61 100644 --- a/dbms/src/Interpreters/CrossToInnerJoinVisitor.cpp +++ b/dbms/src/Interpreters/CrossToInnerJoinVisitor.cpp @@ -4,8 +4,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -27,41 +29,26 @@ namespace ErrorCodes namespace { -struct JoinedTable +struct JoinedElement { - DatabaseAndTableWithAlias table; - ASTTablesInSelectQueryElement * element = nullptr; - ASTTableJoin * join = nullptr; - ASTPtr array_join = nullptr; - bool has_using = false; - - JoinedTable(ASTPtr table_element) + JoinedElement(const ASTTablesInSelectQueryElement & table_element) + : element(table_element) { - element = table_element->as(); - if (!element) - throw Exception("Logical error: TablesInSelectQueryElement expected", ErrorCodes::LOGICAL_ERROR); + if (element.table_join) + join = element.table_join->as(); + } - if (element->table_join) - { - join = element->table_join->as(); - if (join->kind == ASTTableJoin::Kind::Cross || - join->kind == ASTTableJoin::Kind::Comma) - { - if (!join->children.empty()) - throw Exception("Logical error: CROSS JOIN has expressions", ErrorCodes::LOGICAL_ERROR); - } + void checkTableName(const DatabaseAndTableWithAlias & table, const String & current_database) const + { + if (!element.table_expression) + throw Exception("Not a table expression in JOIN (ARRAY JOIN?)", ErrorCodes::LOGICAL_ERROR); - if (join->using_expression_list) - has_using = true; - } + ASTTableExpression * table_expression = element.table_expression->as(); + if (!table_expression) + throw Exception("Wrong table expression in JOIN", ErrorCodes::LOGICAL_ERROR); - if (element->table_expression) - { - const auto & expr = element->table_expression->as(); - table = DatabaseAndTableWithAlias(expr); - } - - array_join = element->array_join; + if (!table.same(DatabaseAndTableWithAlias(*table_expression, current_database))) + throw Exception("Inconsistent table names", ErrorCodes::LOGICAL_ERROR); } void rewriteCommaToCross() @@ -70,7 +57,24 @@ struct JoinedTable join->kind = ASTTableJoin::Kind::Cross; } + void rewriteCrossToInner(ASTPtr on_expression) + { + join->kind = ASTTableJoin::Kind::Inner; + join->strictness = ASTTableJoin::Strictness::All; + + join->on_expression = on_expression; + join->children.push_back(join->on_expression); + } + + ASTPtr arrayJoin() const { return element.array_join; } + const ASTTableJoin * tableJoin() const { return join; } + bool canAttachOnExpression() const { return join && !join->on_expression; } + bool hasUsing() const { return join && join->using_expression_list; } + +private: + const ASTTablesInSelectQueryElement & element; + ASTTableJoin * join = nullptr; }; bool isComparison(const String & name) @@ -89,13 +93,14 @@ class CheckExpressionVisitorData public: using TypeToVisit = const ASTFunction; - CheckExpressionVisitorData(const std::vector & tables_) + CheckExpressionVisitorData(const std::vector & tables_, + const std::vector & tables_with_columns, + Aliases && aliases_) : joined_tables(tables_) + , tables(tables_with_columns) + , aliases(aliases_) , ands_only(true) - { - for (auto & joined : joined_tables) - tables.push_back(joined.table); - } + {} void visit(const ASTFunction & node, const ASTPtr & ast) { @@ -160,9 +165,10 @@ public: } private: - const std::vector & joined_tables; - std::vector tables; + const std::vector & joined_tables; + const std::vector & tables; std::map> asts_to_join_on; + Aliases aliases; bool ands_only; size_t canMoveEqualsToJoinOn(const ASTFunction & node) @@ -177,6 +183,12 @@ private: if (!left || !right) return false; + /// Moving expressions that use column aliases is not supported. + if (left->isShort() && aliases.count(left->shortName())) + return false; + if (right->isShort() && aliases.count(right->shortName())) + return false; + return checkIdentifiers(*left, *right); } @@ -185,15 +197,17 @@ private: /// @return table position to attach expression to or 0. size_t checkIdentifiers(const ASTIdentifier & left, const ASTIdentifier & right) { - size_t left_table_pos = 0; - bool left_match = IdentifierSemantic::chooseTable(left, tables, left_table_pos); + std::optional left_table_pos = IdentifierSemantic::getMembership(left); + if (!left_table_pos) + left_table_pos = IdentifierSemantic::chooseTable(left, tables); - size_t right_table_pos = 0; - bool right_match = IdentifierSemantic::chooseTable(right, tables, right_table_pos); + std::optional right_table_pos = IdentifierSemantic::getMembership(right); + if (!right_table_pos) + right_table_pos = IdentifierSemantic::chooseTable(right, tables); - if (left_match && right_match && (left_table_pos != right_table_pos)) + if (left_table_pos && right_table_pos && (*left_table_pos != *right_table_pos)) { - size_t table_pos = std::max(left_table_pos, right_table_pos); + size_t table_pos = std::max(*left_table_pos, *right_table_pos); if (joined_tables[table_pos].canAttachOnExpression()) return table_pos; } @@ -205,7 +219,7 @@ using CheckExpressionMatcher = ConstOneTypeMatcher; -bool getTables(ASTSelectQuery & select, std::vector & joined_tables, size_t & num_comma) +bool getTables(ASTSelectQuery & select, std::vector & joined_tables, size_t & num_comma) { if (!select.tables()) return false; @@ -224,23 +238,37 @@ bool getTables(ASTSelectQuery & select, std::vector & joined_tables for (auto & child : tables->children) { - joined_tables.emplace_back(JoinedTable(child)); - JoinedTable & t = joined_tables.back(); - if (t.array_join) + auto table_element = child->as(); + if (!table_element) + throw Exception("Logical error: TablesInSelectQueryElement expected", ErrorCodes::LOGICAL_ERROR); + + joined_tables.emplace_back(JoinedElement(*table_element)); + JoinedElement & t = joined_tables.back(); + + if (t.arrayJoin()) { ++num_array_join; continue; } - if (t.has_using) + if (t.hasUsing()) { ++num_using; continue; } - if (auto * join = t.join) + if (auto * join = t.tableJoin()) + { + if (join->kind == ASTTableJoin::Kind::Cross || + join->kind == ASTTableJoin::Kind::Comma) + { + if (!join->children.empty()) + throw Exception("Logical error: CROSS JOIN has expressions", ErrorCodes::LOGICAL_ERROR); + } + if (join->kind == ASTTableJoin::Kind::Comma) ++num_comma; + } } if (num_using && (num_tables - num_array_join) > 2) @@ -251,12 +279,20 @@ bool getTables(ASTSelectQuery & select, std::vector & joined_tables if (num_array_join || num_using) return false; + return true; } } +bool CrossToInnerJoinMatcher::needChildVisit(ASTPtr & node, const ASTPtr &) +{ + if (node->as()) + return false; + return true; +} + void CrossToInnerJoinMatcher::visit(ASTPtr & ast, Data & data) { if (auto * t = ast->as()) @@ -266,10 +302,19 @@ void CrossToInnerJoinMatcher::visit(ASTPtr & ast, Data & data) void CrossToInnerJoinMatcher::visit(ASTSelectQuery & select, ASTPtr &, Data & data) { size_t num_comma = 0; - std::vector joined_tables; + std::vector joined_tables; if (!getTables(select, joined_tables, num_comma)) return; + /// Check if joined_tables are consistent with known tables_with_columns + { + if (joined_tables.size() != data.tables_with_columns.size()) + throw Exception("Logical error: inconsistent number of tables", ErrorCodes::LOGICAL_ERROR); + + for (size_t i = 0; i < joined_tables.size(); ++i) + joined_tables[i].checkTableName(data.tables_with_columns[i].table, data.current_database); + } + /// COMMA to CROSS if (num_comma) @@ -283,7 +328,13 @@ void CrossToInnerJoinMatcher::visit(ASTSelectQuery & select, ASTPtr &, Data & da if (!select.where()) return; - CheckExpressionVisitor::Data visitor_data{joined_tables}; + Aliases aliases; + QueryAliasesVisitor::Data query_aliases_data{aliases}; + if (ASTPtr with = select.with()) + QueryAliasesVisitor(query_aliases_data).visit(with); + QueryAliasesVisitor(query_aliases_data).visit(select.select()); + + CheckExpressionVisitor::Data visitor_data{joined_tables, data.tables_with_columns, std::move(aliases)}; CheckExpressionVisitor(visitor_data).visit(select.where()); if (visitor_data.complex()) @@ -293,12 +344,7 @@ void CrossToInnerJoinMatcher::visit(ASTSelectQuery & select, ASTPtr &, Data & da { if (visitor_data.matchAny(i)) { - ASTTableJoin & join = *joined_tables[i].join; - join.kind = ASTTableJoin::Kind::Inner; - join.strictness = ASTTableJoin::Strictness::All; - - join.on_expression = visitor_data.makeOnExpression(i); - join.children.push_back(join.on_expression); + joined_tables[i].rewriteCrossToInner(visitor_data.makeOnExpression(i)); data.done = true; } } diff --git a/dbms/src/Interpreters/CrossToInnerJoinVisitor.h b/dbms/src/Interpreters/CrossToInnerJoinVisitor.h index 522d368e3fa..4c5ae97bc34 100644 --- a/dbms/src/Interpreters/CrossToInnerJoinVisitor.h +++ b/dbms/src/Interpreters/CrossToInnerJoinVisitor.h @@ -6,6 +6,7 @@ namespace DB { class ASTSelectQuery; +struct TableWithColumnNamesAndTypes; /// AST transformer. It replaces cross joins with equivalented inner join if possible. class CrossToInnerJoinMatcher @@ -13,10 +14,12 @@ class CrossToInnerJoinMatcher public: struct Data { + const std::vector & tables_with_columns; + const String current_database; bool done = false; }; - static bool needChildVisit(ASTPtr &, const ASTPtr &) { return true; } + static bool needChildVisit(ASTPtr &, const ASTPtr &); static void visit(ASTPtr & ast, Data & data); private: diff --git a/dbms/src/Interpreters/DatabaseAndTableWithAlias.h b/dbms/src/Interpreters/DatabaseAndTableWithAlias.h index 2510d6e0679..5b98669d83e 100644 --- a/dbms/src/Interpreters/DatabaseAndTableWithAlias.h +++ b/dbms/src/Interpreters/DatabaseAndTableWithAlias.h @@ -35,6 +35,12 @@ struct DatabaseAndTableWithAlias /// Check if it satisfies another db_table name. @note opterion is not symmetric. bool satisfies(const DatabaseAndTableWithAlias & table, bool table_may_be_an_alias); + + /// Exactly the same table name + bool same(const DatabaseAndTableWithAlias & db_table) const + { + return database == db_table.database && table == db_table.table && alias == db_table.alias; + } }; struct TableWithColumnNames @@ -80,6 +86,19 @@ struct TableWithColumnNamesAndTypes , columns(columns_) {} + bool hasColumn(const String & name) const + { + if (names.empty()) + { + for (auto & col : columns) + names.insert(col.name); + for (auto & col : hidden_columns) + names.insert(col.name); + } + + return names.count(name); + } + void addHiddenColumns(const NamesAndTypesList & addition) { hidden_columns.insert(hidden_columns.end(), addition.begin(), addition.end()); @@ -99,6 +118,9 @@ struct TableWithColumnNamesAndTypes return TableWithColumnNames(table, std::move(out_columns), std::move(out_hidden_columns)); } + +private: + mutable NameSet names; }; std::vector getDatabaseAndTables(const ASTSelectQuery & select_query, const String & current_database); diff --git a/dbms/src/Interpreters/ExtractExpressionInfoVisitor.cpp b/dbms/src/Interpreters/ExtractExpressionInfoVisitor.cpp index 1240b6a09d6..f0ca33b6b8b 100644 --- a/dbms/src/Interpreters/ExtractExpressionInfoVisitor.cpp +++ b/dbms/src/Interpreters/ExtractExpressionInfoVisitor.cpp @@ -50,9 +50,8 @@ void ExpressionInfoMatcher::visit(const ASTIdentifier & identifier, const ASTPtr } else { - size_t best_table_pos = 0; - if (IdentifierSemantic::chooseTable(identifier, data.tables, best_table_pos)) - data.unique_reference_tables_pos.emplace(best_table_pos); + if (auto best_table_pos = IdentifierSemantic::chooseTable(identifier, data.tables)) + data.unique_reference_tables_pos.emplace(*best_table_pos); } } diff --git a/dbms/src/Interpreters/IdentifierSemantic.cpp b/dbms/src/Interpreters/IdentifierSemantic.cpp index ed882c1118d..1cc67107d05 100644 --- a/dbms/src/Interpreters/IdentifierSemantic.cpp +++ b/dbms/src/Interpreters/IdentifierSemantic.cpp @@ -14,29 +14,18 @@ namespace ErrorCodes namespace { -const DatabaseAndTableWithAlias & extractTable(const DatabaseAndTableWithAlias & table) -{ - return table; -} - -const DatabaseAndTableWithAlias & extractTable(const TableWithColumnNames & table) -{ - return table.table; -} - template -IdentifierSemantic::ColumnMatch tryChooseTable(const ASTIdentifier & identifier, const std::vector & tables, - size_t & best_table_pos, bool allow_ambiguous) +std::optional tryChooseTable(const ASTIdentifier & identifier, const std::vector & tables, bool allow_ambiguous) { using ColumnMatch = IdentifierSemantic::ColumnMatch; - best_table_pos = 0; + size_t best_table_pos = 0; auto best_match = ColumnMatch::NoMatch; size_t same_match = 0; for (size_t i = 0; i < tables.size(); ++i) { - auto match = IdentifierSemantic::canReferColumnToTable(identifier, extractTable(tables[i])); + auto match = IdentifierSemantic::canReferColumnToTable(identifier, tables[i]); if (match != ColumnMatch::NoMatch) { if (match > best_match) @@ -54,9 +43,13 @@ IdentifierSemantic::ColumnMatch tryChooseTable(const ASTIdentifier & identifier, { if (!allow_ambiguous) throw Exception("Ambiguous column '" + identifier.name + "'", ErrorCodes::AMBIGUOUS_COLUMN_NAME); - return ColumnMatch::Ambiguous; + best_match = ColumnMatch::Ambiguous; + return {}; } - return best_match; + + if (best_match != ColumnMatch::NoMatch) + return best_table_pos; + return {}; } } @@ -125,18 +118,22 @@ std::optional IdentifierSemantic::getMembership(const ASTIdentifier & id return identifier.semantic->membership; } -bool IdentifierSemantic::chooseTable(const ASTIdentifier & identifier, const std::vector & tables, - size_t & best_table_pos, bool ambiguous) +std::optional IdentifierSemantic::chooseTable(const ASTIdentifier & identifier, const std::vector & tables, + bool ambiguous) { - static constexpr auto no_match = IdentifierSemantic::ColumnMatch::NoMatch; - return tryChooseTable(identifier, tables, best_table_pos, ambiguous) != no_match; + return tryChooseTable(identifier, tables, ambiguous); } -bool IdentifierSemantic::chooseTable(const ASTIdentifier & identifier, const std::vector & tables, - size_t & best_table_pos, bool ambiguous) +std::optional IdentifierSemantic::chooseTable(const ASTIdentifier & identifier, const std::vector & tables, + bool ambiguous) { - static constexpr auto no_match = IdentifierSemantic::ColumnMatch::NoMatch; - return tryChooseTable(identifier, tables, best_table_pos, ambiguous) != no_match; + return tryChooseTable(identifier, tables, ambiguous); +} + +std::optional IdentifierSemantic::chooseTable(const ASTIdentifier & identifier, const std::vector & tables, + bool ambiguous) +{ + return tryChooseTable(identifier, tables, ambiguous); } std::pair IdentifierSemantic::extractDatabaseAndTable(const ASTIdentifier & identifier) @@ -198,6 +195,22 @@ IdentifierSemantic::ColumnMatch IdentifierSemantic::canReferColumnToTable(const return ColumnMatch::NoMatch; } +IdentifierSemantic::ColumnMatch IdentifierSemantic::canReferColumnToTable(const ASTIdentifier & identifier, + const TableWithColumnNames & db_and_table) +{ + /// TODO: ColumnName match logic is disabled cause caller's code is not ready for it + return canReferColumnToTable(identifier, db_and_table.table); +} + +IdentifierSemantic::ColumnMatch IdentifierSemantic::canReferColumnToTable(const ASTIdentifier & identifier, + const TableWithColumnNamesAndTypes & db_and_table) +{ + ColumnMatch match = canReferColumnToTable(identifier, db_and_table.table); + if (match == ColumnMatch::NoMatch && identifier.isShort() && db_and_table.hasColumn(identifier.shortName())) + match = ColumnMatch::ColumnName; + return match; +} + /// Strip qualificators from left side of column name. /// Example: 'database.table.name' -> 'name'. void IdentifierSemantic::setColumnShortName(ASTIdentifier & identifier, const DatabaseAndTableWithAlias & db_and_table) diff --git a/dbms/src/Interpreters/IdentifierSemantic.h b/dbms/src/Interpreters/IdentifierSemantic.h index e3b69abc61e..0b92b7b7716 100644 --- a/dbms/src/Interpreters/IdentifierSemantic.h +++ b/dbms/src/Interpreters/IdentifierSemantic.h @@ -22,6 +22,7 @@ struct IdentifierSemantic enum class ColumnMatch { NoMatch, + ColumnName, /// column qualified with column names list AliasedTableName, /// column qualified with table name (but table has an alias so its priority is lower than TableName) TableName, /// column qualified with table name DbAndTable, /// column qualified with database and table name @@ -40,6 +41,9 @@ struct IdentifierSemantic static std::optional extractNestedName(const ASTIdentifier & identifier, const String & table_name); static ColumnMatch canReferColumnToTable(const ASTIdentifier & identifier, const DatabaseAndTableWithAlias & db_and_table); + static ColumnMatch canReferColumnToTable(const ASTIdentifier & identifier, const TableWithColumnNames & db_and_table); + static ColumnMatch canReferColumnToTable(const ASTIdentifier & identifier, const TableWithColumnNamesAndTypes & db_and_table); + static void setColumnShortName(ASTIdentifier & identifier, const DatabaseAndTableWithAlias & db_and_table); static void setColumnLongName(ASTIdentifier & identifier, const DatabaseAndTableWithAlias & db_and_table); static bool canBeAlias(const ASTIdentifier & identifier); @@ -47,10 +51,12 @@ struct IdentifierSemantic static void coverName(ASTIdentifier &, const String & alias); static std::optional uncover(const ASTIdentifier & identifier); static std::optional getMembership(const ASTIdentifier & identifier); - static bool chooseTable(const ASTIdentifier &, const std::vector & tables, size_t & best_table_pos, - bool ambiguous = false); - static bool chooseTable(const ASTIdentifier &, const std::vector & tables, size_t & best_table_pos, - bool ambiguous = false); + static std::optional chooseTable(const ASTIdentifier &, const std::vector & tables, + bool allow_ambiguous = false); + static std::optional chooseTable(const ASTIdentifier &, const std::vector & tables, + bool allow_ambiguous = false); + static std::optional chooseTable(const ASTIdentifier &, const std::vector & tables, + bool allow_ambiguous = false); private: static bool doesIdentifierBelongTo(const ASTIdentifier & identifier, const String & database, const String & table); diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index 1c5fba86c98..8285978c7bd 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -235,23 +235,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( throw Exception("Too deep subqueries. Maximum: " + settings.max_subquery_depth.toString(), ErrorCodes::TOO_DEEP_SUBQUERIES); - JoinedTables joined_tables(getSelectQuery()); - if (joined_tables.hasJoins()) - { - CrossToInnerJoinVisitor::Data cross_to_inner; - CrossToInnerJoinVisitor(cross_to_inner).visit(query_ptr); - - JoinToSubqueryTransformVisitor::Data join_to_subs_data{*context}; - JoinToSubqueryTransformVisitor(join_to_subs_data).visit(query_ptr); - - joined_tables.reset(getSelectQuery()); - } - - max_streams = settings.max_threads; - ASTSelectQuery & query = getSelectQuery(); - - const ASTPtr & left_table_expression = joined_tables.leftTableExpression(); - + bool has_input = input || input_pipe; if (input) { /// Read from prepared input. @@ -262,35 +246,51 @@ InterpreterSelectQuery::InterpreterSelectQuery( /// Read from prepared input. source_header = input_pipe->getHeader(); } - else if (joined_tables.isLeftTableSubquery()) - { - /// Read from subquery. - interpreter_subquery = std::make_unique( - left_table_expression, getSubqueryContext(*context), options.subquery()); - source_header = interpreter_subquery->getSampleBlock(); - } - else if (!storage) - { - if (joined_tables.isLeftTableFunction()) - { - /// Read from table function. propagate all settings from initSettings(), - /// alternative is to call on current `context`, but that can potentially pollute it. - storage = getSubqueryContext(*context).executeTableFunction(left_table_expression); - } - else - storage = joined_tables.getLeftTableStorage(*context); - } + JoinedTables joined_tables(getSubqueryContext(*context), getSelectQuery()); + + if (!has_input && !storage) + storage = joined_tables.getLeftTableStorage(); if (storage) { table_lock = storage->lockStructureForShare(false, context->getInitialQueryId()); table_id = storage->getStorageID(); - - joined_tables.resolveTables(getSubqueryContext(*context), storage); } - else - joined_tables.resolveTables(getSubqueryContext(*context), source_header.getNamesAndTypesList()); + + if (has_input || !joined_tables.resolveTables()) + joined_tables.makeFakeTable(storage, source_header); + + /// Rewrite JOINs + if (!has_input && joined_tables.tablesCount() > 1) + { + CrossToInnerJoinVisitor::Data cross_to_inner{joined_tables.tablesWithColumns(), context->getCurrentDatabase()}; + CrossToInnerJoinVisitor(cross_to_inner).visit(query_ptr); + + JoinToSubqueryTransformVisitor::Data join_to_subs_data{*context}; + JoinToSubqueryTransformVisitor(join_to_subs_data).visit(query_ptr); + + joined_tables.reset(getSelectQuery()); + joined_tables.resolveTables(); + + if (storage && joined_tables.isLeftTableSubquery()) + { + /// Rewritten with subquery. Free storage here locks here. + storage = {}; + table_lock.release(); + table_id = StorageID::createEmpty(); + } + } + + if (!has_input) + { + interpreter_subquery = joined_tables.makeLeftTableSubquery(options.subquery()); + if (interpreter_subquery) + source_header = interpreter_subquery->getSampleBlock(); + } + + max_streams = settings.max_threads; + ASTSelectQuery & query = getSelectQuery(); auto analyze = [&] (bool try_move_to_prewhere = true) { @@ -330,11 +330,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( if (syntax_analyzer_result->rewrite_subqueries) { /// remake interpreter_subquery when PredicateOptimizer rewrites subqueries and main table is subquery - if (joined_tables.isLeftTableSubquery()) - interpreter_subquery = std::make_unique( - left_table_expression, - getSubqueryContext(*context), - options.subquery()); + interpreter_subquery = joined_tables.makeLeftTableSubquery(options.subquery()); } } diff --git a/dbms/src/Interpreters/JoinToSubqueryTransformVisitor.cpp b/dbms/src/Interpreters/JoinToSubqueryTransformVisitor.cpp index 6d543643436..83066996cff 100644 --- a/dbms/src/Interpreters/JoinToSubqueryTransformVisitor.cpp +++ b/dbms/src/Interpreters/JoinToSubqueryTransformVisitor.cpp @@ -147,9 +147,8 @@ struct ColumnAliasesMatcher { bool last_table = false; { - size_t best_table_pos = 0; - if (IdentifierSemantic::chooseTable(*identifier, tables, best_table_pos)) - last_table = (best_table_pos + 1 == tables.size()); + if (auto best_table_pos = IdentifierSemantic::chooseTable(*identifier, tables)) + last_table = (*best_table_pos + 1 == tables.size()); } if (!last_table) @@ -207,10 +206,9 @@ struct ColumnAliasesMatcher bool last_table = false; String long_name; - size_t table_pos = 0; - if (IdentifierSemantic::chooseTable(node, data.tables, table_pos)) + if (auto table_pos = IdentifierSemantic::chooseTable(node, data.tables)) { - auto & table = data.tables[table_pos]; + auto & table = data.tables[*table_pos]; IdentifierSemantic::setColumnLongName(node, table); /// table_name.column_name -> table_alias.column_name long_name = node.name; if (&table == &data.tables.back()) diff --git a/dbms/src/Interpreters/JoinedTables.cpp b/dbms/src/Interpreters/JoinedTables.cpp index d97e78a1669..5b085b70863 100644 --- a/dbms/src/Interpreters/JoinedTables.cpp +++ b/dbms/src/Interpreters/JoinedTables.cpp @@ -6,6 +6,7 @@ #include #include #include +#include namespace DB { @@ -33,8 +34,9 @@ void checkTablesWithColumns(const std::vector & tables_with_columns, const Co } -JoinedTables::JoinedTables(const ASTSelectQuery & select_query) - : table_expressions(getTableExpressions(select_query)) +JoinedTables::JoinedTables(Context && context_, const ASTSelectQuery & select_query) + : context(context_) + , table_expressions(getTableExpressions(select_query)) , left_table_expression(extractTableExpression(select_query, 0)) , left_db_and_table(getDatabaseAndTable(select_query, 0)) {} @@ -49,9 +51,20 @@ bool JoinedTables::isLeftTableFunction() const return left_table_expression && left_table_expression->as(); } -StoragePtr JoinedTables::getLeftTableStorage(Context & context) +std::unique_ptr JoinedTables::makeLeftTableSubquery(const SelectQueryOptions & select_options) { - StoragePtr storage; + if (!isLeftTableSubquery()) + return {}; + return std::make_unique(left_table_expression, context, select_options); +} + +StoragePtr JoinedTables::getLeftTableStorage() +{ + if (isLeftTableSubquery()) + return {}; + + if (isLeftTableFunction()) + return context.executeTableFunction(left_table_expression); if (left_db_and_table) { @@ -75,42 +88,36 @@ StoragePtr JoinedTables::getLeftTableStorage(Context & context) if (tmp_table_id.database_name == database_name && tmp_table_id.table_name == table_name) { /// Read from view source. - storage = context.getViewSource(); + return context.getViewSource(); } } - if (!storage) - { - /// Read from table. Even without table expression (implicit SELECT ... FROM system.one). - storage = context.getTable(database_name, table_name); - } - - return storage; + /// Read from table. Even without table expression (implicit SELECT ... FROM system.one). + return context.getTable(database_name, table_name); } -void JoinedTables::resolveTables(const Context & context, StoragePtr storage) +bool JoinedTables::resolveTables() { tables_with_columns = getDatabaseAndTablesWithColumns(table_expressions, context); checkTablesWithColumns(tables_with_columns, context); - if (tables_with_columns.empty()) + return !tables_with_columns.empty(); +} + +void JoinedTables::makeFakeTable(StoragePtr storage, const Block & source_header) +{ + if (storage) { const ColumnsDescription & storage_columns = storage->getColumns(); tables_with_columns.emplace_back(DatabaseAndTableWithAlias{}, storage_columns.getOrdinary()); + auto & table = tables_with_columns.back(); table.addHiddenColumns(storage_columns.getMaterialized()); table.addHiddenColumns(storage_columns.getAliases()); table.addHiddenColumns(storage_columns.getVirtuals()); } -} - -void JoinedTables::resolveTables(const Context & context, const NamesAndTypesList & source_columns) -{ - tables_with_columns = getDatabaseAndTablesWithColumns(table_expressions, context); - checkTablesWithColumns(tables_with_columns, context); - - if (tables_with_columns.empty()) - tables_with_columns.emplace_back(DatabaseAndTableWithAlias{}, source_columns); + else + tables_with_columns.emplace_back(DatabaseAndTableWithAlias{}, source_header.getNamesAndTypesList()); } } diff --git a/dbms/src/Interpreters/JoinedTables.h b/dbms/src/Interpreters/JoinedTables.h index 225d024581a..c9d24562a26 100644 --- a/dbms/src/Interpreters/JoinedTables.h +++ b/dbms/src/Interpreters/JoinedTables.h @@ -2,6 +2,7 @@ #include #include +#include #include namespace DB @@ -9,6 +10,7 @@ namespace DB class ASTSelectQuery; class Context; +struct SelectQueryOptions; /// Joined tables' columns resolver. /// We want to get each table structure at most once per table occurance. Or even better once per table. @@ -16,32 +18,30 @@ class Context; class JoinedTables { public: - JoinedTables() = default; - JoinedTables(const ASTSelectQuery & select_query); + JoinedTables(Context && contex, const ASTSelectQuery & select_query); void reset(const ASTSelectQuery & select_query) { - *this = JoinedTables(select_query); + *this = JoinedTables(std::move(context), select_query); } - StoragePtr getLeftTableStorage(Context & context); - - /// Resolve columns or get from storage. It assumes storage is not nullptr. - void resolveTables(const Context & context, StoragePtr storage); - /// Resolve columns or get from source list. - void resolveTables(const Context & context, const NamesAndTypesList & source_columns); + StoragePtr getLeftTableStorage(); + bool resolveTables(); + void makeFakeTable(StoragePtr storage, const Block & source_header); const std::vector & tablesWithColumns() const { return tables_with_columns; } bool isLeftTableSubquery() const; bool isLeftTableFunction() const; - bool hasJoins() const { return table_expressions.size() > 1; } + size_t tablesCount() const { return table_expressions.size(); } - const ASTPtr & leftTableExpression() const { return left_table_expression; } const String & leftTableDatabase() const { return database_name; } const String & leftTableName() const { return table_name; } + std::unique_ptr makeLeftTableSubquery(const SelectQueryOptions & select_options); + private: + Context context; std::vector table_expressions; std::vector tables_with_columns; diff --git a/dbms/src/Interpreters/QueryAliasesVisitor.cpp b/dbms/src/Interpreters/QueryAliasesVisitor.cpp index 6de0ece8b59..6d8f0266670 100644 --- a/dbms/src/Interpreters/QueryAliasesVisitor.cpp +++ b/dbms/src/Interpreters/QueryAliasesVisitor.cpp @@ -30,7 +30,7 @@ static String wrongAliasMessage(const ASTPtr & ast, const ASTPtr & prev_ast, con } -bool QueryAliasesMatcher::needChildVisit(ASTPtr & node, const ASTPtr &) +bool QueryAliasesMatcher::needChildVisit(const ASTPtr & node, const ASTPtr &) { /// Don't descent into table functions and subqueries and special case for ArrayJoin. if (node->as() || node->as() || node->as()) @@ -38,7 +38,7 @@ bool QueryAliasesMatcher::needChildVisit(ASTPtr & node, const ASTPtr &) return true; } -void QueryAliasesMatcher::visit(ASTPtr & ast, Data & data) +void QueryAliasesMatcher::visit(const ASTPtr & ast, Data & data) { if (auto * s = ast->as()) visit(*s, ast, data); @@ -81,8 +81,9 @@ void QueryAliasesMatcher::visit(const ASTArrayJoin &, const ASTPtr & ast, Data & /// set unique aliases for all subqueries. this is needed, because: /// 1) content of subqueries could change after recursive analysis, and auto-generated column names could become incorrect /// 2) result of different scalar subqueries can be cached inside expressions compilation cache and must have different names -void QueryAliasesMatcher::visit(ASTSubquery & subquery, const ASTPtr & ast, Data & data) +void QueryAliasesMatcher::visit(const ASTSubquery & const_subquery, const ASTPtr & ast, Data & data) { + ASTSubquery & subquery = const_cast(const_subquery); Aliases & aliases = data.aliases; static std::atomic_uint64_t subquery_index = 0; diff --git a/dbms/src/Interpreters/QueryAliasesVisitor.h b/dbms/src/Interpreters/QueryAliasesVisitor.h index c4e297965c3..62242b500f7 100644 --- a/dbms/src/Interpreters/QueryAliasesVisitor.h +++ b/dbms/src/Interpreters/QueryAliasesVisitor.h @@ -15,19 +15,19 @@ struct ASTArrayJoin; class QueryAliasesMatcher { public: - using Visitor = InDepthNodeVisitor; + using Visitor = ConstInDepthNodeVisitor; struct Data { Aliases & aliases; }; - static void visit(ASTPtr & ast, Data & data); - static bool needChildVisit(ASTPtr & node, const ASTPtr & child); + static void visit(const ASTPtr & ast, Data & data); + static bool needChildVisit(const ASTPtr & node, const ASTPtr & child); private: static void visit(const ASTSelectQuery & select, const ASTPtr & ast, Data & data); - static void visit(ASTSubquery & subquery, const ASTPtr & ast, Data & data); + static void visit(const ASTSubquery & subquery, const ASTPtr & ast, Data & data); static void visit(const ASTArrayJoin &, const ASTPtr & ast, Data & data); static void visitOther(const ASTPtr & ast, Data & data); }; diff --git a/dbms/src/Interpreters/TranslateQualifiedNamesVisitor.cpp b/dbms/src/Interpreters/TranslateQualifiedNamesVisitor.cpp index 04f59b81f0f..27542b97691 100644 --- a/dbms/src/Interpreters/TranslateQualifiedNamesVisitor.cpp +++ b/dbms/src/Interpreters/TranslateQualifiedNamesVisitor.cpp @@ -93,10 +93,10 @@ void TranslateQualifiedNamesMatcher::visit(ASTIdentifier & identifier, ASTPtr &, if (IdentifierSemantic::getColumnName(identifier)) { String short_name = identifier.shortName(); - size_t table_pos = 0; bool allow_ambiguous = data.join_using_columns.count(short_name); - if (IdentifierSemantic::chooseTable(identifier, data.tables, table_pos, allow_ambiguous)) + if (auto best_pos = IdentifierSemantic::chooseTable(identifier, data.tables, allow_ambiguous)) { + size_t table_pos = *best_pos; if (data.unknownColumn(table_pos, identifier)) { String table_name = data.tables[table_pos].table.getQualifiedNamePrefix(false); diff --git a/dbms/tests/queries/0_stateless/01095_tpch_like_smoke.reference b/dbms/tests/queries/0_stateless/01095_tpch_like_smoke.reference new file mode 100644 index 00000000000..20caccfed83 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01095_tpch_like_smoke.reference @@ -0,0 +1,14 @@ +1 +3 +5 +6 +0.0000 +9 +10 +12 +14 +0.00000000 +16 +18 +19 +0.0000 diff --git a/dbms/tests/queries/0_stateless/01095_tpch_like_smoke.sql b/dbms/tests/queries/0_stateless/01095_tpch_like_smoke.sql new file mode 100644 index 00000000000..ae43a9cfc28 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01095_tpch_like_smoke.sql @@ -0,0 +1,802 @@ +CREATE DATABASE IF NOT EXISTS tpch; +USE tpch; + +DROP TABLE IF EXISTS part; +DROP TABLE IF EXISTS supplier; +DROP TABLE IF EXISTS partsupp; +DROP TABLE IF EXISTS customer; +DROP TABLE IF EXISTS orders; +DROP TABLE IF EXISTS lineitem; +DROP TABLE IF EXISTS nation; +DROP TABLE IF EXISTS region; + +CREATE TABLE part +( + p_partkey Int32, -- PK + p_name String, -- variable text, size 55 + p_mfgr FixedString(25), + p_brand FixedString(10), + p_type String, -- variable text, size 25 + p_size Int32, -- integer + p_container FixedString(10), + p_retailprice Decimal(18,2), + p_comment String, -- variable text, size 23 + CONSTRAINT pk CHECK p_partkey >= 0, + CONSTRAINT positive CHECK (p_size >= 0 AND p_retailprice >= 0) +) engine = MergeTree ORDER BY (p_partkey); + +CREATE TABLE supplier +( + s_suppkey Int32, -- PK + s_name FixedString(25), + s_address String, -- variable text, size 40 + s_nationkey Int32, -- FK n_nationkey + s_phone FixedString(15), + s_acctbal Decimal(18,2), + s_comment String, -- variable text, size 101 + CONSTRAINT pk CHECK s_suppkey >= 0 +) engine = MergeTree ORDER BY (s_suppkey); + +CREATE TABLE partsupp +( + ps_partkey Int32, -- PK(1), FK p_partkey + ps_suppkey Int32, -- PK(2), FK s_suppkey + ps_availqty Int32, -- integer + ps_supplycost Decimal(18,2), + ps_comment String, -- variable text, size 199 + CONSTRAINT pk CHECK ps_partkey >= 0, + CONSTRAINT c1 CHECK (ps_availqty >= 0 AND ps_supplycost >= 0) +) engine = MergeTree ORDER BY (ps_partkey, ps_suppkey); + +CREATE TABLE customer +( + c_custkey Int32, -- PK + c_name String, -- variable text, size 25 + c_address String, -- variable text, size 40 + c_nationkey Int32, -- FK n_nationkey + c_phone FixedString(15), + c_acctbal Decimal(18,2), + c_mktsegment FixedString(10), + c_comment String, -- variable text, size 117 + CONSTRAINT pk CHECK c_custkey >= 0 +) engine = MergeTree ORDER BY (c_custkey); + +CREATE TABLE orders +( + o_orderkey Int32, -- PK + o_custkey Int32, -- FK c_custkey + o_orderstatus FixedString(1), + o_totalprice Decimal(18,2), + o_orderdate Date, + o_orderpriority FixedString(15), + o_clerk FixedString(15), + o_shippriority Int32, -- integer + o_comment String, -- variable text, size 79 + CONSTRAINT c1 CHECK o_totalprice >= 0 +) engine = MergeTree ORDER BY (o_orderdate, o_orderkey); + +CREATE TABLE lineitem +( + l_orderkey Int32, -- PK(1), FK o_orderkey + l_partkey Int32, -- FK ps_partkey + l_suppkey Int32, -- FK ps_suppkey + l_linenumber Int32, -- PK(2) + l_quantity Decimal(18,2), + l_extendedprice Decimal(18,2), + l_discount Decimal(18,2), + l_tax Decimal(18,2), + l_returnflag FixedString(1), + l_linestatus FixedString(1), + l_shipdate Date, + l_commitdate Date, + l_receiptdate Date, + l_shipinstruct FixedString(25), + l_shipmode FixedString(10), + l_comment String, -- variable text size 44 + CONSTRAINT c1 CHECK (l_quantity >= 0 AND l_extendedprice >= 0 AND l_tax >= 0 AND l_shipdate <= l_receiptdate) +-- CONSTRAINT c2 CHECK (l_discount >= 0 AND l_discount <= 1) +) engine = MergeTree ORDER BY (l_shipdate, l_receiptdate, l_orderkey, l_linenumber); + +CREATE TABLE nation +( + n_nationkey Int32, -- PK + n_name FixedString(25), + n_regionkey Int32, -- FK r_regionkey + n_comment String, -- variable text, size 152 + CONSTRAINT pk CHECK n_nationkey >= 0 +) Engine = MergeTree ORDER BY (n_nationkey); + +CREATE TABLE region +( + r_regionkey Int32, -- PK + r_name FixedString(25), + r_comment String, -- variable text, size 152 + CONSTRAINT pk CHECK r_regionkey >= 0 +) engine = MergeTree ORDER BY (r_regionkey); + +select 1; +select + l_returnflag, + l_linestatus, + sum(l_quantity) as sum_qty, + sum(l_extendedprice) as sum_base_price, + sum(l_extendedprice * (1 - l_discount)) as sum_disc_price, + sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) as sum_charge, + avg(l_quantity) as avg_qty, + avg(l_extendedprice) as avg_price, + avg(l_discount) as avg_disc, + count(*) as count_order +from + lineitem +where + l_shipdate <= toDate('1998-12-01') - interval 90 day +group by + l_returnflag, + l_linestatus +order by + l_returnflag, + l_linestatus; + +-- select 2; -- rewrite fail +-- select +-- s_acctbal, +-- s_name, +-- n_name, +-- p_partkey, +-- p_mfgr, +-- s_address, +-- s_phone, +-- s_comment +-- from +-- part, +-- supplier, +-- partsupp, +-- nation, +-- region +-- where +-- p_partkey = ps_partkey +-- and s_suppkey = ps_suppkey +-- and p_size = 15 +-- and p_type like '%BRASS' +-- and s_nationkey = n_nationkey +-- and n_regionkey = r_regionkey +-- and r_name = 'EUROPE' +-- and ps_supplycost = ( +-- select +-- min(ps_supplycost) +-- from +-- partsupp, +-- supplier, +-- nation, +-- region +-- where +-- p_partkey = ps_partkey +-- and s_suppkey = ps_suppkey +-- and s_nationkey = n_nationkey +-- and n_regionkey = r_regionkey +-- and r_name = 'EUROPE' +-- ) +-- order by +-- s_acctbal desc, +-- n_name, +-- s_name, +-- p_partkey +-- limit 100; + +select 3; +select + l_orderkey, + sum(l_extendedprice * (1 - l_discount)) as revenue, + o_orderdate, + o_shippriority +from + customer, + orders, + lineitem +where + c_mktsegment = 'BUILDING' + and c_custkey = o_custkey + and l_orderkey = o_orderkey + and o_orderdate < toDate('1995-03-15') + and l_shipdate > toDate('1995-03-15') +group by + l_orderkey, + o_orderdate, + o_shippriority +order by + revenue desc, + o_orderdate +limit 10; + +-- select 4; +-- select +-- o_orderpriority, +-- count(*) as order_count +-- from +-- orders +-- where +-- o_orderdate >= toDate('1993-07-01') +-- and o_orderdate < toDate('1993-07-01') + interval '3' month +-- and exists ( +-- select +-- * +-- from +-- lineitem +-- where +-- l_orderkey = o_orderkey +-- and l_commitdate < l_receiptdate +-- ) +-- group by +-- o_orderpriority +-- order by +-- o_orderpriority; + +select 5; +select + n_name, + sum(l_extendedprice * (1 - l_discount)) as revenue +from + customer, + orders, + lineitem, + supplier, + nation, + region +where + c_custkey = o_custkey + and l_orderkey = o_orderkey + and l_suppkey = s_suppkey + and c_nationkey = s_nationkey + and s_nationkey = n_nationkey + and n_regionkey = r_regionkey + and r_name = 'ASIA' + and o_orderdate >= toDate('1994-01-01') + and o_orderdate < toDate('1994-01-01') + interval '1' year +group by + n_name +order by + revenue desc; + +select 6; +select + sum(l_extendedprice * l_discount) as revenue +from + lineitem +where + l_shipdate >= toDate('1994-01-01') + and l_shipdate < toDate('1994-01-01') + interval '1' year + and l_discount between toDecimal32(0.06, 2) - toDecimal32(0.01, 2) + and toDecimal32(0.06, 2) + toDecimal32(0.01, 2) + and l_quantity < 24; + +-- select 7; +-- select +-- supp_nation, +-- cust_nation, +-- l_year, +-- sum(volume) as revenue +-- from +-- ( +-- select +-- n1.n_name as supp_nation, +-- n2.n_name as cust_nation, +-- extract(year from l_shipdate) as l_year, +-- l_extendedprice * (1 - l_discount) as volume +-- from +-- supplier, +-- lineitem, +-- orders, +-- customer, +-- nation n1, +-- nation n2 +-- where +-- s_suppkey = l_suppkey +-- and o_orderkey = l_orderkey +-- and c_custkey = o_custkey +-- and s_nationkey = n1.n_nationkey +-- and c_nationkey = n2.n_nationkey +-- and ( +-- (n1.n_name = 'FRANCE' and n2.n_name = 'GERMANY') +-- or (n1.n_name = 'GERMANY' and n2.n_name = 'FRANCE') +-- ) +-- and l_shipdate between toDate('1995-01-01') and toDate('1996-12-31') +-- ) as shipping +-- group by +-- supp_nation, +-- cust_nation, +-- l_year +-- order by +-- supp_nation, +-- cust_nation, +-- l_year; + +-- select 8; +-- select +-- o_year, +-- sum(case +-- when nation = 'BRAZIL' then volume +-- else 0 +-- end) / sum(volume) as mkt_share +-- from +-- ( +-- select +-- extract(year from o_orderdate) as o_year, +-- l_extendedprice * (1 - l_discount) as volume, +-- n2.n_name as nation +-- from +-- part, +-- supplier, +-- lineitem, +-- orders, +-- customer, +-- nation n1, +-- nation n2, +-- region +-- where +-- p_partkey = l_partkey +-- and s_suppkey = l_suppkey +-- and l_orderkey = o_orderkey +-- and o_custkey = c_custkey +-- and c_nationkey = n1.n_nationkey +-- and n1.n_regionkey = r_regionkey +-- and r_name = 'AMERICA' +-- and s_nationkey = n2.n_nationkey +-- and o_orderdate between toDate('1995-01-01') and toDate('1996-12-31') +-- and p_type = 'ECONOMY ANODIZED STEEL' +-- ) as all_nations +-- group by +-- o_year +-- order by +-- o_year; + +select 9; +select + nation, + o_year, + sum(amount) as sum_profit +from + ( + select + n_name as nation, + extract(year from o_orderdate) as o_year, + l_extendedprice * (1 - l_discount) - ps_supplycost * l_quantity as amount + from + part, + supplier, + lineitem, + partsupp, + orders, + nation + where + s_suppkey = l_suppkey + and ps_suppkey = l_suppkey + and ps_partkey = l_partkey + and p_partkey = l_partkey + and o_orderkey = l_orderkey + and s_nationkey = n_nationkey + and p_name like '%green%' + ) as profit +group by + nation, + o_year +order by + nation, + o_year desc; + +select 10; +select + c_custkey, + c_name, + sum(l_extendedprice * (1 - l_discount)) as revenue, + c_acctbal, + n_name, + c_address, + c_phone, + c_comment +from + customer, + orders, + lineitem, + nation +where + c_custkey = o_custkey + and l_orderkey = o_orderkey + and o_orderdate >= toDate('1993-10-01') + and o_orderdate < toDate('1993-10-01') + interval '3' month + and l_returnflag = 'R' + and c_nationkey = n_nationkey +group by + c_custkey, + c_name, + c_acctbal, + c_phone, + n_name, + c_address, + c_comment +order by + revenue desc +limit 20; + +-- select 11; -- rewrite fail +-- select +-- ps_partkey, +-- sum(ps_supplycost * ps_availqty) as value +-- from +-- partsupp, +-- supplier, +-- nation +-- where +-- ps_suppkey = s_suppkey +-- and s_nationkey = n_nationkey +-- and n_name = 'GERMANY' +-- group by +-- ps_partkey having +-- sum(ps_supplycost * ps_availqty) > ( +-- select +-- sum(ps_supplycost * ps_availqty) * 0.0100000000 +-- -- ^^^^^^^^^^^^ +-- -- The above constant needs to be adjusted according +-- -- to the scale factor (SF): constant = 0.0001 / SF. +-- from +-- partsupp, +-- supplier, +-- nation +-- where +-- ps_suppkey = s_suppkey +-- and s_nationkey = n_nationkey +-- and n_name = 'GERMANY' +-- ) +-- order by +-- value desc; + +select 12; +select + l_shipmode, + sum(case + when o_orderpriority = '1-URGENT' + or o_orderpriority = '2-HIGH' + then 1 + else 0 + end) as high_line_count, + sum(case + when o_orderpriority <> '1-URGENT' + and o_orderpriority <> '2-HIGH' + then 1 + else 0 + end) as low_line_count +from + orders, + lineitem +where + o_orderkey = l_orderkey + and l_shipmode in ('MAIL', 'SHIP') + and l_commitdate < l_receiptdate + and l_shipdate < l_commitdate + and l_receiptdate >= toDate('1994-01-01') + and l_receiptdate < toDate('1994-01-01') + interval '1' year +group by + l_shipmode +order by + l_shipmode; + +-- select 13; -- rewrite fail +-- select +-- c_count, +-- count(*) as custdist +-- from +-- ( +-- select +-- c_custkey, +-- count(o_orderkey) +-- from +-- customer left outer join orders on +-- c_custkey = o_custkey +-- and o_comment not like '%special%requests%' +-- group by +-- c_custkey +-- ) as c_orders +-- group by +-- c_count +-- order by +-- custdist desc, +-- c_count desc; + +select 14; +select + toDecimal32(100.00, 2) * sum(case + when p_type like 'PROMO%' + then l_extendedprice * (1 - l_discount) + else 0 + end) / (1 + sum(l_extendedprice * (1 - l_discount))) as promo_revenue +from + lineitem, + part +where + l_partkey = p_partkey + and l_shipdate >= toDate('1995-09-01') + and l_shipdate < toDate('1995-09-01') + interval '1' month; + +-- select 15; +-- create view revenue0 as +-- select +-- l_suppkey, +-- sum(l_extendedprice * (1 - l_discount)) +-- from +-- lineitem +-- where +-- l_shipdate >= toDate('1996-01-01') +-- and l_shipdate < toDate('1996-01-01') + interval '3' month +-- group by +-- l_suppkey; +-- select +-- s_suppkey, +-- s_name, +-- s_address, +-- s_phone, +-- total_revenue +-- from +-- supplier, +-- revenue0 +-- where +-- s_suppkey = supplier_no +-- and total_revenue = ( +-- select +-- max(total_revenue) +-- from +-- revenue0 +-- ) +-- order by +-- s_suppkey; +-- drop view revenue0; + +select 16; +select + p_brand, + p_type, + p_size, + count(distinct ps_suppkey) as supplier_cnt +from + partsupp, + part +where + p_partkey = ps_partkey + and p_brand <> 'Brand#45' + and p_type not like 'MEDIUM POLISHED%' + and p_size in (49, 14, 23, 45, 19, 3, 36, 9) + and ps_suppkey not in ( + select + s_suppkey + from + supplier + where + s_comment like '%Customer%Complaints%' + ) +group by + p_brand, + p_type, + p_size +order by + supplier_cnt desc, + p_brand, + p_type, + p_size; + +-- select 17; +-- select +-- sum(l_extendedprice) / 7.0 as avg_yearly +-- from +-- lineitem, +-- part +-- where +-- p_partkey = l_partkey +-- and p_brand = 'Brand#23' +-- and p_container = 'MED BOX' +-- and l_quantity < ( +-- select +-- 0.2 * avg(l_quantity) +-- from +-- lineitem +-- where +-- l_partkey = p_partkey +-- ); + +select 18; +select + c_name, + c_custkey, + o_orderkey, + o_orderdate, + o_totalprice, + sum(l_quantity) +from + customer, + orders, + lineitem +where + o_orderkey in ( + select + l_orderkey + from + lineitem + group by + l_orderkey having + sum(l_quantity) > 300 + ) + and c_custkey = o_custkey + and o_orderkey = l_orderkey +group by + c_name, + c_custkey, + o_orderkey, + o_orderdate, + o_totalprice +order by + o_totalprice desc, + o_orderdate +limit 100; + +select 19; +select + sum(l_extendedprice* (1 - l_discount)) as revenue +from + lineitem, + part +where + ( + p_partkey = l_partkey + and p_brand = 'Brand#12' + and p_container in ('SM CASE', 'SM BOX', 'SM PACK', 'SM PKG') + and l_quantity >= 1 and l_quantity <= 1 + 10 + and p_size between 1 and 5 + and l_shipmode in ('AIR', 'AIR REG') + and l_shipinstruct = 'DELIVER IN PERSON' + ) + or + ( + p_partkey = l_partkey + and p_brand = 'Brand#23' + and p_container in ('MED BAG', 'MED BOX', 'MED PKG', 'MED PACK') + and l_quantity >= 10 and l_quantity <= 10 + 10 + and p_size between 1 and 10 + and l_shipmode in ('AIR', 'AIR REG') + and l_shipinstruct = 'DELIVER IN PERSON' + ) + or + ( + p_partkey = l_partkey + and p_brand = 'Brand#34' + and p_container in ('LG CASE', 'LG BOX', 'LG PACK', 'LG PKG') + and l_quantity >= 20 and l_quantity <= 20 + 10 + and p_size between 1 and 15 + and l_shipmode in ('AIR', 'AIR REG') + and l_shipinstruct = 'DELIVER IN PERSON' + ); + +-- select 20; +-- select +-- s_name, +-- s_address +-- from +-- supplier, +-- nation +-- where +-- s_suppkey in ( +-- select +-- ps_suppkey +-- from +-- partsupp +-- where +-- ps_partkey in ( +-- select +-- p_partkey +-- from +-- part +-- where +-- p_name like 'forest%' +-- ) +-- and ps_availqty > ( +-- select +-- 0.5 * sum(l_quantity) +-- from +-- lineitem +-- where +-- l_partkey = ps_partkey +-- and l_suppkey = ps_suppkey +-- and l_shipdate >= toDate('1994-01-01') +-- and l_shipdate < toDate('1994-01-01') + interval '1' year +-- ) +-- ) +-- and s_nationkey = n_nationkey +-- and n_name = 'CANADA' +-- order by +-- s_name; + +-- select 21; +-- select +-- s_name, +-- count(*) as numwait +-- from +-- supplier, +-- lineitem l1, +-- orders, +-- nation +-- where +-- s_suppkey = l1.l_suppkey +-- and o_orderkey = l1.l_orderkey +-- and o_orderstatus = 'F' +-- and l1.l_receiptdate > l1.l_commitdate +-- and exists ( +-- select +-- * +-- from +-- lineitem l2 +-- where +-- l2.l_orderkey = l1.l_orderkey +-- and l2.l_suppkey <> l1.l_suppkey +-- ) +-- and not exists ( +-- select +-- * +-- from +-- lineitem l3 +-- where +-- l3.l_orderkey = l1.l_orderkey +-- and l3.l_suppkey <> l1.l_suppkey +-- and l3.l_receiptdate > l3.l_commitdate +-- ) +-- and s_nationkey = n_nationkey +-- and n_name = 'SAUDI ARABIA' +-- group by +-- s_name +-- order by +-- numwait desc, +-- s_name +-- limit 100; + +-- select 22; +-- select +-- cntrycode, +-- count(*) as numcust, +-- sum(c_acctbal) as totacctbal +-- from +-- ( +-- select +-- substring(c_phone from 1 for 2) as cntrycode, +-- c_acctbal +-- from +-- customer +-- where +-- substring(c_phone from 1 for 2) in +-- ('13', '31', '23', '29', '30', '18', '17') +-- and c_acctbal > ( +-- select +-- avg(c_acctbal) +-- from +-- customer +-- where +-- c_acctbal > 0.00 +-- and substring(c_phone from 1 for 2) in +-- ('13', '31', '23', '29', '30', '18', '17') +-- ) +-- and not exists ( +-- select +-- * +-- from +-- orders +-- where +-- o_custkey = c_custkey +-- ) +-- ) as custsale +-- group by +-- cntrycode +-- order by +-- cntrycode; + +DROP TABLE part; +DROP TABLE supplier; +DROP TABLE partsupp; +DROP TABLE customer; +DROP TABLE orders; +DROP TABLE lineitem; +DROP TABLE nation; +DROP TABLE region; From 87d909119579efa792e41ab96c3f77c1f92a009e Mon Sep 17 00:00:00 2001 From: "philip.han" Date: Sun, 8 Mar 2020 22:22:20 +0900 Subject: [PATCH 128/712] Set can_be_false to true when it applies to the bloom_filter --- .../MergeTreeIndexConditionBloomFilter.cpp | 2 +- ...78_bloom_filter_operator_not_has.reference | 20 ++++++++++++++++ .../01078_bloom_filter_operator_not_has.sql | 23 +++++++++++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 dbms/tests/queries/0_stateless/01078_bloom_filter_operator_not_has.reference create mode 100644 dbms/tests/queries/0_stateless/01078_bloom_filter_operator_not_has.sql diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp b/dbms/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp index 01f81c3ce1f..acabb7130cc 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp @@ -161,7 +161,7 @@ bool MergeTreeIndexConditionBloomFilter::mayBeTrueOnGranule(const MergeTreeIndex match_rows = maybeTrueOnBloomFilter(&*hash_column, filter, hash_functions); } - rpn_stack.emplace_back(match_rows, !match_rows); + rpn_stack.emplace_back(match_rows, true); if (element.function == RPNElement::FUNCTION_NOT_EQUALS || element.function == RPNElement::FUNCTION_NOT_IN) rpn_stack.back() = !rpn_stack.back(); } diff --git a/dbms/tests/queries/0_stateless/01078_bloom_filter_operator_not_has.reference b/dbms/tests/queries/0_stateless/01078_bloom_filter_operator_not_has.reference new file mode 100644 index 00000000000..7e1f383cb21 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01078_bloom_filter_operator_not_has.reference @@ -0,0 +1,20 @@ +4 +1 +3 +[] 2020-02-27 +['o','a'] 2020-02-27 +2 +[] 2020-02-27 +['e','a','b'] 2020-02-27 +['o','a'] 2020-02-27 +1 +[] 2020-02-27 +['e','a','b'] 2020-02-27 +['o','a'] 2020-02-27 +['o','a','b','c'] 2020-02-27 +0 +[] 2020-02-27 +['e','a','b'] 2020-02-27 +['e','a','b','c','d'] 2020-02-27 +['o','a'] 2020-02-27 +['o','a','b','c'] 2020-02-27 diff --git a/dbms/tests/queries/0_stateless/01078_bloom_filter_operator_not_has.sql b/dbms/tests/queries/0_stateless/01078_bloom_filter_operator_not_has.sql new file mode 100644 index 00000000000..20eabdb081f --- /dev/null +++ b/dbms/tests/queries/0_stateless/01078_bloom_filter_operator_not_has.sql @@ -0,0 +1,23 @@ +DROP TABLE IF EXISTS bloom_filter_not_has; + +CREATE TABLE bloom_filter_not_has (ary Array(LowCardinality(Nullable(String))), d Date, INDEX idx_ary ary TYPE bloom_filter(0.01) GRANULARITY 1024) ENGINE = MergeTree() PARTITION BY d ORDER BY d; + +INSERT INTO bloom_filter_not_has VALUES ([], '2020-02-27') (['o','a'], '2020-02-27') (['e','a','b'], '2020-02-27'); +INSERT INTO bloom_filter_not_has VALUES (['o','a','b','c'], '2020-02-27') (['e','a','b','c','d'], '2020-02-27'); + +SELECT count() FROM bloom_filter_not_has WHERE has(ary, 'a'); +SELECT count() FROM bloom_filter_not_has WHERE NOT has(ary, 'a'); + +SELECT count() FROM bloom_filter_not_has WHERE has(ary, 'b'); +SELECT * FROM bloom_filter_not_has WHERE NOT has(ary, 'b') ORDER BY ary; + +SELECT count() FROM bloom_filter_not_has WHERE has(ary, 'c'); +SELECT * FROM bloom_filter_not_has WHERE NOT has(ary, 'c') ORDER BY ary; + +SELECT count() FROM bloom_filter_not_has WHERE has(ary, 'd'); +SELECT * FROM bloom_filter_not_has WHERE NOT has(ary, 'd') ORDER BY ary; + +SELECT count() FROM bloom_filter_not_has WHERE has(ary, 'f'); +SELECT * FROM bloom_filter_not_has WHERE NOT has(ary, 'f') ORDER BY ary; + +DROP TABLE IF EXISTS bloom_filter_not_has; From bce00a6bf2cc092a3b973a395f42df637843c333 Mon Sep 17 00:00:00 2001 From: alesapin Date: Sun, 8 Mar 2020 17:57:31 +0300 Subject: [PATCH 129/712] Fix parallel tests --- .../01079_parallel_alter_add_drop_column_zookeeper.sh | 2 +- .../0_stateless/01079_parallel_alter_detach_table_zookeeper.sh | 2 +- .../0_stateless/01079_parallel_alter_modify_zookeeper.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dbms/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.sh b/dbms/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.sh index 7a71df32c8e..1fbccf47c72 100755 --- a/dbms/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.sh +++ b/dbms/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.sh @@ -106,6 +106,6 @@ for i in `seq $REPLICAS`; do $CLICKHOUSE_CLIENT --query "SELECT COUNT() FROM system.mutations WHERE is_done = 0 and table = 'concurrent_alter_add_drop_$i'" $CLICKHOUSE_CLIENT --query "SELECT * FROM system.mutations WHERE is_done = 0 and table = 'concurrent_alter_add_drop_$i'" $CLICKHOUSE_CLIENT --query "SELECT COUNT() FROM system.replication_queue WHERE table = 'concurrent_alter_add_drop_$i'" - $CLICKHOUSE_CLIENT --query "SELECT * FROM system.replication_queue WHERE table = 'concurrent_alter_add_drop_$i'" + $CLICKHOUSE_CLIENT --query "SELECT * FROM system.replication_queue WHERE table = 'concurrent_alter_add_drop_$i' and (type = 'ALTER_METADATA' or type = 'MUTATE_PART')" $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS concurrent_alter_add_drop_$i" done diff --git a/dbms/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh b/dbms/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh index 319c65cc5a7..0ec6f01137f 100755 --- a/dbms/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh +++ b/dbms/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh @@ -105,6 +105,6 @@ for i in `seq $REPLICAS`; do $CLICKHOUSE_CLIENT --query "SELECT SUM(toUInt64(value1)) > $INITIAL_SUM FROM concurrent_alter_detach_$i" $CLICKHOUSE_CLIENT --query "SELECT COUNT() FROM system.mutations WHERE is_done=0 and table = 'concurrent_alter_detach_$i'" # all mutations have to be done $CLICKHOUSE_CLIENT --query "SELECT * FROM system.mutations WHERE is_done=0 and table = 'concurrent_alter_detach_$i'" # all mutations have to be done - $CLICKHOUSE_CLIENT --query "SELECT * FROM system.replication_queue WHERE table = 'concurrent_alter_detach_$i'" # all mutations have to be done + $CLICKHOUSE_CLIENT --query "SELECT * FROM system.replication_queue WHERE table = 'concurrent_alter_detach_$i' and (type = 'ALTER_METADATA' or type = 'MUTATE_PART')" # all mutations and alters have to be done $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS concurrent_alter_detach_$i" done diff --git a/dbms/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh b/dbms/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh index 31643d21815..5d181727301 100755 --- a/dbms/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh +++ b/dbms/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh @@ -110,6 +110,6 @@ for i in `seq $REPLICAS`; do $CLICKHOUSE_CLIENT --query "SELECT SUM(toUInt64(value1)) > $INITIAL_SUM FROM concurrent_alter_mt_$i" $CLICKHOUSE_CLIENT --query "SELECT COUNT() FROM system.mutations WHERE is_done=0 and table = 'concurrent_alter_mt_$i'" # all mutations have to be done $CLICKHOUSE_CLIENT --query "SELECT * FROM system.mutations WHERE is_done=0 and table = 'concurrent_alter_mt_$i'" - $CLICKHOUSE_CLIENT --query "SELECT * FROM system.replication_queue WHERE table = 'concurrent_alter_mt_$i'" + $CLICKHOUSE_CLIENT --query "SELECT * FROM system.replication_queue WHERE table = 'concurrent_alter_mt_$i' and (type = 'ALTER_METADATA' or type = 'MUTATE_PART')" $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS concurrent_alter_mt_$i" done From 0bc86507e5c8f3587593b8ace0e6f606038ba3b2 Mon Sep 17 00:00:00 2001 From: alesapin Date: Sun, 8 Mar 2020 18:21:48 +0300 Subject: [PATCH 130/712] Fix error when tests are being run on partially downloaded data --- docker/test/stateful_with_coverage/Dockerfile | 2 +- docker/test/stateful_with_coverage/run.sh | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docker/test/stateful_with_coverage/Dockerfile b/docker/test/stateful_with_coverage/Dockerfile index 863e55e6326..e5ddf3d3475 100644 --- a/docker/test/stateful_with_coverage/Dockerfile +++ b/docker/test/stateful_with_coverage/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t yandex/clickhouse-stateful-test . +# docker build -t yandex/clickhouse-stateful-test-with-coverage . FROM yandex/clickhouse-stateless-test RUN echo "deb [trusted=yes] http://apt.llvm.org/bionic/ llvm-toolchain-bionic-9 main" >> /etc/apt/sources.list diff --git a/docker/test/stateful_with_coverage/run.sh b/docker/test/stateful_with_coverage/run.sh index 0c7e3e50cb9..cf5eb0ee598 100755 --- a/docker/test/stateful_with_coverage/run.sh +++ b/docker/test/stateful_with_coverage/run.sh @@ -71,7 +71,11 @@ start_clickhouse sleep 5 -/s3downloader --dataset-names $DATASETS +if ! /s3downloader --dataset-names $DATASETS; then + echo "Cannot download datatsets" + exit 1 +fi + chmod 777 -R /var/lib/clickhouse From 49894c7bf2167029e2be733992abcdfe4d542581 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 8 Mar 2020 20:00:08 +0300 Subject: [PATCH 131/712] Update libunwind for WSL --- contrib/libunwind | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/libunwind b/contrib/libunwind index 68cffcbbd18..ede00622ff8 160000 --- a/contrib/libunwind +++ b/contrib/libunwind @@ -1 +1 @@ -Subproject commit 68cffcbbd1840e14664a5f7f19c5e43f65c525b5 +Subproject commit ede00622ff8ecb1848ed22187eabbfaf8b4e9307 From de67bd78bdccaf872da827aa3c03681e06ebba62 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 8 Mar 2020 23:17:49 +0300 Subject: [PATCH 132/712] Added some clang-tidy checks --- .clang-tidy | 2 ++ CMakeLists.txt | 19 +------------------ base/CMakeLists.txt | 4 ++++ cmake/analysis.cmake | 18 ++++++++++++++++++ dbms/CMakeLists.txt | 2 +- 5 files changed, 26 insertions(+), 19 deletions(-) create mode 100644 .clang-tidy create mode 100644 cmake/analysis.cmake diff --git a/.clang-tidy b/.clang-tidy new file mode 100644 index 00000000000..bbd532c8c9a --- /dev/null +++ b/.clang-tidy @@ -0,0 +1,2 @@ +Checks: '-*,google-readability-avoid-underscore-in-googletest-name,misc-throw-by-value-catch-by-reference,misc-misplaced-const,misc-unconventional-assign-operator,modernize-avoid-bind,modernize-loop-convert,modernize-make-shared,modernize-make-unique,modernize-raw-string-literal,modernize-redundant-void-arg,modernize-replace-auto-ptr,modernize-replace-random-shuffle,modernize-use-bool-literals,modernize-use-nullptr,modernize-use-using,performance-faster-string-find,performance-for-range-copy,readability-avoid-const-params-in-decls,readability-const-return-type,readability-container-size-empty,readability-convert-member-functions-to-static,readability-delete-null-pointer,readability-deleted-default,readability-make-member-function-const,readability-misleading-indentation,readability-misplaced-array-index,readability-non-const-parameter,readability-qualified-auto,readability-redundant-access-specifiers,readability-redundant-control-flow,readability-redundant-function-ptr-dereference,readability-redundant-smartptr-get,readability-redundant-string-cstr,readability-redundant-string-init,readability-static-definition-in-anonymous-namespace,readability-string-compare,readability-uniqueptr-delete-release,modernize-use-equals-default,modernize-use-equals-delete,bugprone-undelegated-constructor,readability-redundant-member-init,readability-simplify-subscript-expr,readability-simplify-boolean-expr,readability-inconsistent-declaration-parameter-name' +WarningsAsErrors: '*' diff --git a/CMakeLists.txt b/CMakeLists.txt index 2948225583c..9513caa8eee 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,6 +17,7 @@ project(ClickHouse) include (cmake/arch.cmake) include (cmake/target.cmake) include (cmake/tools.cmake) +include (cmake/analysis.cmake) # Ignore export() since we don't use it, # but it gets broken with a global targets via link_libraries() @@ -285,24 +286,6 @@ if (USE_INCLUDE_WHAT_YOU_USE) endif() endif () -# Using clang-tidy static analyzer http://mariobadr.com/using-clang-tidy-with-cmake-36.html https://cmake.org/cmake/help/v3.6/prop_tgt/LANG_CLANG_TIDY.html -option (ENABLE_CLANG_TIDY "Use 'clang-tidy' static analyzer" OFF) -if (ENABLE_CLANG_TIDY) - if (${CMAKE_VERSION} VERSION_LESS "3.6.0") - message(FATAL_ERROR "clang-tidy requires CMake version at least 3.6.") - endif() - find_program (CLANG_TIDY_EXE NAMES "clang-tidy" DOC "Path to clang-tidy executable") - if (NOT CLANG_TIDY_EXE) - set (USE_CLANG_TIDY 0) - message (STATUS "clang-tidy not found.") - else () - set (USE_CLANG_TIDY 1) - message (STATUS "clang-tidy found: ${CLANG_TIDY_EXE}") - set (DO_CLANG_TIDY "${CLANG_TIDY_EXE}" "-checks=*,-clang-analyzer-alpha.*") - # You can enable it within a directory by: set (CMAKE_CXX_CLANG_TIDY "${DO_CLANG_TIDY}") - endif () -endif () - if (ENABLE_TESTS) message (STATUS "Tests are enabled") endif () diff --git a/base/CMakeLists.txt b/base/CMakeLists.txt index 65593f70a95..6790c87da1c 100644 --- a/base/CMakeLists.txt +++ b/base/CMakeLists.txt @@ -1,3 +1,7 @@ +if (USE_CLANG_TIDY) + set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}") +endif () + add_subdirectory (common) add_subdirectory (loggers) add_subdirectory (daemon) diff --git a/cmake/analysis.cmake b/cmake/analysis.cmake new file mode 100644 index 00000000000..ecc3ea0bdbf --- /dev/null +++ b/cmake/analysis.cmake @@ -0,0 +1,18 @@ +# This file configures static analysis tools that can be integrated to the build process + +option (ENABLE_CLANG_TIDY "Use 'clang-tidy' static analyzer if present" ON) +if (ENABLE_CLANG_TIDY) + if (${CMAKE_VERSION} VERSION_LESS "3.6.0") + message(FATAL_ERROR "clang-tidy requires CMake version at least 3.6.") + endif() + + find_program (CLANG_TIDY_PATH NAMES "clang-tidy") + if (CLANG_TIDY_PATH) + message(STATUS "Using clang-tidy: ${CLANG_TIDY_PATH}. The checks will be run during build process. See the .clang-tidy file at the root directory to configure the checks.") + set (USE_CLANG_TIDY 1) + # The variable CMAKE_CXX_CLANG_TIDY will be set inside dbms and base directories with non third-party code. + # set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}") + else () + message(STATUS "clang-tidy is not found. This is normal - the tool is used only for static code analysis and not essential for build.") + endif () +endif () diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index 821cf3f6654..aa10b0ed2ca 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -6,7 +6,7 @@ if (USE_INCLUDE_WHAT_YOU_USE) endif () if (USE_CLANG_TIDY) - set (CMAKE_CXX_CLANG_TIDY "${DO_CLANG_TIDY}") + set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}") endif () if(COMPILER_PIPE) From 75754f309eaf1e2b4296390a54acc2538a3ea4d7 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 8 Mar 2020 23:28:17 +0300 Subject: [PATCH 133/712] Normalize running time of some queries in performance tests --- dbms/tests/performance/bitCount.xml | 2 +- dbms/tests/performance/date_time.xml | 11 ++---- .../performance/generate_table_function.xml | 38 +++++++++---------- dbms/tests/performance/joins_in_memory.xml | 10 ++--- 4 files changed, 29 insertions(+), 32 deletions(-) diff --git a/dbms/tests/performance/bitCount.xml b/dbms/tests/performance/bitCount.xml index 34fdb24c10b..4519fdfd431 100644 --- a/dbms/tests/performance/bitCount.xml +++ b/dbms/tests/performance/bitCount.xml @@ -21,5 +21,5 @@ - SELECT bitCount({expr}) FROM numbers(1000000) FORMAT Null + SELECT bitCount({expr}) FROM numbers(100000000) FORMAT Null diff --git a/dbms/tests/performance/date_time.xml b/dbms/tests/performance/date_time.xml index 858b20d5784..56af5b85e19 100644 --- a/dbms/tests/performance/date_time.xml +++ b/dbms/tests/performance/date_time.xml @@ -131,11 +131,8 @@ - SELECT count() FROM numbers(100000) WHERE NOT ignore(toDateTime('2017-01-01 00:00:00') + number % 100000000 + rand() % 100000 AS t, {datetime_transform}(t, '{time_zone}')) - - SELECT count() FROM numbers(100000) WHERE NOT ignore(toDate('2017-01-01') + number % 1000 + rand() % 10 AS t, {date_transform}(t)) - - SELECT count() FROM numbers(100000) WHERE NOT ignore(toDateTime('2017-01-01 00:00:00') + number % 100000000 + rand() % 100000 AS t, {binary_function}(t, 1)) - - SELECT count() FROM numbers(100000) WHERE NOT ignore(toDateTime('2017-01-01 00:00:00') + number % 100000000 + rand() % 100000 AS t, toStartOfInterval(t, INTERVAL 1 month)) + SELECT count() FROM numbers(10000000) WHERE NOT ignore(toDateTime('2017-01-01 00:00:00') + number % 100000000 + rand() % 100000 AS t, {datetime_transform}(t, '{time_zone}')) + SELECT count() FROM numbers(10000000) WHERE NOT ignore(toDate('2017-01-01') + number % 1000 + rand() % 10 AS t, {date_transform}(t)) + SELECT count() FROM numbers(10000000) WHERE NOT ignore(toDateTime('2017-01-01 00:00:00') + number % 100000000 + rand() % 100000 AS t, {binary_function}(t, 1)) + SELECT count() FROM numbers(10000000) WHERE NOT ignore(toDateTime('2017-01-01 00:00:00') + number % 100000000 + rand() % 100000 AS t, toStartOfInterval(t, INTERVAL 1 month)) diff --git a/dbms/tests/performance/generate_table_function.xml b/dbms/tests/performance/generate_table_function.xml index 7e14a55b5c7..c53ec285bf5 100644 --- a/dbms/tests/performance/generate_table_function.xml +++ b/dbms/tests/performance/generate_table_function.xml @@ -6,23 +6,23 @@ - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8') LIMIT 100000); - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8', 10, 10, 1) LIMIT 100000); - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Enum8(\'hello\' = 1, \'world\' = 5)', 10, 10, 1) LIMIT 100000); - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Array(Nullable(Enum8(\'hello\' = 1, \'world\' = 5)))', 10, 10, 1) LIMIT 100000); - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Nullable(Enum16(\'h\' = 1, \'w\' = 5 , \'o\' = -200)))', 10, 10, 1) LIMIT 100000); - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('d Date, dt DateTime, dtm DateTime(\'Europe/Moscow\')', 10, 10, 1) LIMIT 100000); - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('dt64 DateTime64, dts64 DateTime64(6), dtms64 DateTime64(6 ,\'Europe/Moscow\')', 10, 10, 1) LIMIT 100000); - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('f32 Float32, f64 Float64', 10, 10, 1) LIMIT 100000); - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('d32 Decimal32(4), d64 Decimal64(8), d128 Decimal128(16)', 10, 10, 1) LIMIT 100000); - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Tuple(Int32, Int64)', 10, 10, 1) LIMIT 100000); - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Array(Int8)', 10, 10, 1) LIMIT 100000); - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Array(Nullable(Int32))', 10, 10, 1) LIMIT 100000); - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Tuple(Int32, Array(Int64))', 10, 10, 1) LIMIT 100000); - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Nullable(String)', 10, 10, 1) LIMIT 100000); - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Array(String)', 10, 10, 1) LIMIT 100000); - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i UUID', 10, 10, 1) LIMIT 100000); - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Array(Nullable(UUID))', 10, 10, 1) LIMIT 100000); - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i FixedString(4)', 10, 10, 1) LIMIT 100000); - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i String', 10, 10, 1) LIMIT 100000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8') LIMIT 10000000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8', 0, 10, 10) LIMIT 10000000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Enum8(\'hello\' = 1, \'world\' = 5)', 0, 10, 10) LIMIT 10000000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Array(Nullable(Enum8(\'hello\' = 1, \'world\' = 5)))', 0, 10, 10) LIMIT 10000000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Nullable(Enum16(\'h\' = 1, \'w\' = 5 , \'o\' = -200)))', 0, 10, 10) LIMIT 10000000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('d Date, dt DateTime, dtm DateTime(\'Europe/Moscow\')', 0, 10, 10) LIMIT 10000000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('dt64 DateTime64, dts64 DateTime64(6), dtms64 DateTime64(6 ,\'Europe/Moscow\')', 0, 10, 10) LIMIT 10000000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('f32 Float32, f64 Float64', 0, 10, 10) LIMIT 10000000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('d32 Decimal32(4), d64 Decimal64(8), d128 Decimal128(16)', 0, 10, 10) LIMIT 10000000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Tuple(Int32, Int64)', 0, 10, 10) LIMIT 10000000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Array(Int8)', 0, 10, 10) LIMIT 10000000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Array(Nullable(Int32))', 0, 10, 10) LIMIT 10000000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Tuple(Int32, Array(Int64))', 0, 10, 10) LIMIT 10000000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Nullable(String)', 0, 10, 10) LIMIT 10000000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Array(String)', 0, 10, 10) LIMIT 10000000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i UUID', 0, 10, 10) LIMIT 10000000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Array(Nullable(UUID))', 0, 10, 10) LIMIT 10000000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i FixedString(4)', 0, 10, 10) LIMIT 10000000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i String', 0, 10, 10) LIMIT 10000000); diff --git a/dbms/tests/performance/joins_in_memory.xml b/dbms/tests/performance/joins_in_memory.xml index 1d3b14ae962..31e75984003 100644 --- a/dbms/tests/performance/joins_in_memory.xml +++ b/dbms/tests/performance/joins_in_memory.xml @@ -9,11 +9,11 @@ CREATE TABLE ints (i64 Int64, i32 Int32, i16 Int16, i8 Int8) ENGINE = Memory - INSERT INTO ints SELECT number AS i64, i64 AS i32, i64 AS i16, i64 AS i8 FROM numbers(5000) - INSERT INTO ints SELECT 10000 + number % 1000 AS i64, i64 AS i32, i64 AS i16, i64 AS i8 FROM numbers(5000) - INSERT INTO ints SELECT 20000 + number % 100 AS i64, i64 AS i32, i64 AS i16, i64 AS i8 FROM numbers(5000) - INSERT INTO ints SELECT 30000 + number % 10 AS i64, i64 AS i32, i64 AS i16, i64 AS i8 FROM numbers(5000) - INSERT INTO ints SELECT 40000 + number % 1 AS i64, i64 AS i32, i64 AS i16, i64 AS i8 FROM numbers(5000) + INSERT INTO ints SELECT number AS i64, i64 AS i32, i64 AS i16, i64 AS i8 FROM numbers(50000) + INSERT INTO ints SELECT 10000 + number % 1000 AS i64, i64 AS i32, i64 AS i16, i64 AS i8 FROM numbers(50000) + INSERT INTO ints SELECT 20000 + number % 100 AS i64, i64 AS i32, i64 AS i16, i64 AS i8 FROM numbers(50000) + INSERT INTO ints SELECT 30000 + number % 10 AS i64, i64 AS i32, i64 AS i16, i64 AS i8 FROM numbers(50000) + INSERT INTO ints SELECT 40000 + number % 1 AS i64, i64 AS i32, i64 AS i16, i64 AS i8 FROM numbers(50000) SELECT COUNT() FROM ints l ANY LEFT JOIN ints r USING i64 WHERE i32 = 200042 SELECT COUNT() FROM ints l ANY LEFT JOIN ints r USING i64,i32,i16,i8 WHERE i32 = 200042 From a542eae3880cdbc1fd450ada42593816b3cf1cad Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 8 Mar 2020 23:47:25 +0300 Subject: [PATCH 134/712] Added results for Pinebook Pro --- website/benchmark_hardware.html | 52 +++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/website/benchmark_hardware.html b/website/benchmark_hardware.html index c5801e289f2..67de998f41a 100644 --- a/website/benchmark_hardware.html +++ b/website/benchmark_hardware.html @@ -2223,6 +2223,57 @@ var results = [0.005, 0.004, 0.004] ] }, + + { + "system": "Pinebook Pro (AArch64, 4 GiB RAM)", + "time": "2020-03-08 00:00:00", + "result": + [ +[0.021, 0.009, 0.007], +[0.195, 0.135, 0.144], +[0.439, 0.264, 0.273], +[1.266, 0.672, 0.706], +[1.337, 0.795, 0.790], +[2.706, 1.989, 1.947], +[0.246, 0.198, 0.197], +[0.157, 0.142, 0.133], +[4.150, 3.769, 3.617], +[5.223, 4.405, 4.234], +[2.391, 1.815, 1.785], +[2.534, 2.158, 2.042], +[7.895, 6.890, 7.003], +[10.338, 9.311, 9.410], +[8.139, 7.441, 7.312], +[8.532, 8.035, 8.011], +[null, null, null], +[null, null, null], +[null, null, null], +[null, null, null], +[null, null, null], +[null, null, null], +[null, null, null], +[null, null, null], +[null, null, null], +[null, null, null], +[null, null, null], +[null, null, null], +[null, null, null], +[null, null, null], +[null, null, null], +[null, null, null], +[null, null, null], +[null, null, null], +[null, null, null], +[null, null, null], +[null, null, null], +[null, null, null], +[null, null, null], +[null, null, null], +[null, null, null], +[null, null, null], +[null, null, null] + ] + } ]; @@ -2654,6 +2705,7 @@ Results for Huawei Taishan are from Peng Gao in sina.com.
Results for Selectel and AMD EPYC 7402P are from Andrey Dudin.
Results for ProLiant are from Denis Ustinov.
Results for AMD EPYC 7502P are from Kostiantyn Velychkovskyi.
+Results for Pinebook Pro are from Aleksey R. @kITerE.
Xeon Gold 6230 server is using 4 x SAMSUNG datacenter class SSD in RAID-10.
Results for Yandex Managed ClickHouse for "cold cache" are biased and should not be compared, because cache was not flushed for every next query.
From 7a7e45e248613f330e8844db8c1f8016b423d0fd Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 9 Mar 2020 00:04:10 +0300 Subject: [PATCH 135/712] clang-tidy, part 1 --- base/common/DateLUTImpl.cpp | 2 +- base/common/JSON.cpp | 4 ++-- base/consistent-hashing-sumbur/sumbur.cpp | 2 +- base/daemon/src/BaseDaemon.cpp | 8 +++---- base/daemon/src/GraphiteWriter.cpp | 2 +- base/mysqlxx/include/mysqlxx/Pool.h | 2 +- base/mysqlxx/src/PoolFactory.cpp | 2 +- base/mysqlxx/src/PoolWithFailover.cpp | 14 ++++++------ base/mysqlxx/src/Query.cpp | 4 ++-- dbms/src/Common/Dwarf.cpp | 1 - dbms/src/Common/Dwarf.h | 10 ++++----- dbms/src/Common/Exception.cpp | 22 ++++++++----------- dbms/src/Common/Exception.h | 2 +- dbms/src/Common/FileChecker.cpp | 6 ++--- dbms/src/Common/Macros.cpp | 2 -- dbms/src/Common/Macros.h | 2 +- .../src/Common/OptimizedRegularExpression.cpp | 2 +- 17 files changed, 39 insertions(+), 48 deletions(-) diff --git a/base/common/DateLUTImpl.cpp b/base/common/DateLUTImpl.cpp index 51f5ceb759c..080f8fb6395 100644 --- a/base/common/DateLUTImpl.cpp +++ b/base/common/DateLUTImpl.cpp @@ -59,7 +59,7 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) time_t start_of_day = DATE_LUT_MIN; cctz::time_zone cctz_time_zone; - if (!cctz::load_time_zone(time_zone.data(), &cctz_time_zone)) + if (!cctz::load_time_zone(time_zone, &cctz_time_zone)) throw Poco::Exception("Cannot load time zone " + time_zone_); cctz::time_zone::absolute_lookup start_of_epoch_lookup = cctz_time_zone.lookup(std::chrono::system_clock::from_time_t(start_of_day)); diff --git a/base/common/JSON.cpp b/base/common/JSON.cpp index c6c0c843a16..92de0dc2d25 100644 --- a/base/common/JSON.cpp +++ b/base/common/JSON.cpp @@ -341,7 +341,7 @@ JSON::Pos JSON::skipArray() const if (*pos == ']') return ++pos; - while (1) + while (true) { pos = JSON(pos, ptr_end, level + 1).skipElement(); @@ -373,7 +373,7 @@ JSON::Pos JSON::skipObject() const if (*pos == '}') return ++pos; - while (1) + while (true) { pos = JSON(pos, ptr_end, level + 1).skipNameValuePair(); diff --git a/base/consistent-hashing-sumbur/sumbur.cpp b/base/consistent-hashing-sumbur/sumbur.cpp index 3b905f0adc7..78f59ca875f 100644 --- a/base/consistent-hashing-sumbur/sumbur.cpp +++ b/base/consistent-hashing-sumbur/sumbur.cpp @@ -108,6 +108,6 @@ unsigned int sumburConsistentHash(unsigned int hashed_int, unsigned int capacity if (L / i - h < part) return n - 1; } } - } while(0); + } while(false); return n - 1; } diff --git a/base/daemon/src/BaseDaemon.cpp b/base/daemon/src/BaseDaemon.cpp index ad479926e30..71a11964481 100644 --- a/base/daemon/src/BaseDaemon.cpp +++ b/base/daemon/src/BaseDaemon.cpp @@ -593,7 +593,7 @@ void debugIncreaseOOMScore() {} void BaseDaemon::initialize(Application & self) { closeFDs(); - task_manager.reset(new Poco::TaskManager); + task_manager = std::make_unique(); ServerApplication::initialize(self); /// now highest priority (lowest value) is PRIO_APPLICATION = -100, we want higher! @@ -778,7 +778,7 @@ void BaseDaemon::initializeTerminationAndSignalProcessing() signal_pipe.setNonBlocking(); signal_pipe.tryIncreaseSize(1 << 20); - signal_listener.reset(new SignalListener(*this)); + signal_listener = std::make_unique(*this); signal_listener_thread.start(*signal_listener); } @@ -839,9 +839,7 @@ void BaseDaemon::defineOptions(Poco::Util::OptionSet& _options) bool isPidRunning(pid_t pid) { - if (getpgid(pid) >= 0) - return 1; - return 0; + return getpgid(pid) >= 0; } BaseDaemon::PID::PID(const std::string & file_) diff --git a/base/daemon/src/GraphiteWriter.cpp b/base/daemon/src/GraphiteWriter.cpp index 41817ca86a2..eeb6b4c1705 100644 --- a/base/daemon/src/GraphiteWriter.cpp +++ b/base/daemon/src/GraphiteWriter.cpp @@ -30,7 +30,7 @@ GraphiteWriter::GraphiteWriter(const std::string & config_name, const std::strin root_path += hostname_in_path; } - if (sub_path.size()) + if (!sub_path.empty()) { if (!root_path.empty()) root_path += "."; diff --git a/base/mysqlxx/include/mysqlxx/Pool.h b/base/mysqlxx/include/mysqlxx/Pool.h index b2b04f3bdfc..b5e0c71a5cc 100644 --- a/base/mysqlxx/include/mysqlxx/Pool.h +++ b/base/mysqlxx/include/mysqlxx/Pool.h @@ -198,7 +198,7 @@ public: return description; } - void removeConnection(Connection * data); + void removeConnection(Connection * connection); protected: /// Number of MySQL connections which are created at launch. diff --git a/base/mysqlxx/src/PoolFactory.cpp b/base/mysqlxx/src/PoolFactory.cpp index 8f07e22671f..d9c1ec1519d 100644 --- a/base/mysqlxx/src/PoolFactory.cpp +++ b/base/mysqlxx/src/PoolFactory.cpp @@ -38,7 +38,7 @@ static std::string getPoolEntryName(const Poco::Util::AbstractConfiguration & co if (!shared) return ""; - std::string entry_name = ""; + std::string entry_name; std::string host = config.getString(config_name + ".host", ""); std::string port = config.getString(config_name + ".port", ""); std::string user = config.getString(config_name + ".user", ""); diff --git a/base/mysqlxx/src/PoolWithFailover.cpp b/base/mysqlxx/src/PoolWithFailover.cpp index 8306922b0e5..2beb50d3da8 100644 --- a/base/mysqlxx/src/PoolWithFailover.cpp +++ b/base/mysqlxx/src/PoolWithFailover.cpp @@ -10,16 +10,16 @@ static bool startsWith(const std::string & s, const char * prefix) using namespace mysqlxx; -PoolWithFailover::PoolWithFailover(const Poco::Util::AbstractConfiguration & cfg, +PoolWithFailover::PoolWithFailover(const Poco::Util::AbstractConfiguration & config, const std::string & config_name, const unsigned default_connections, const unsigned max_connections, const size_t max_tries) : max_tries(max_tries) { - shareable = cfg.getBool(config_name + ".share_connection", false); - if (cfg.has(config_name + ".replica")) + shareable = config.getBool(config_name + ".share_connection", false); + if (config.has(config_name + ".replica")) { Poco::Util::AbstractConfiguration::Keys replica_keys; - cfg.keys(config_name, replica_keys); + config.keys(config_name, replica_keys); for (const auto & replica_config_key : replica_keys) { /// There could be another elements in the same level in configuration file, like "password", "port"... @@ -27,17 +27,17 @@ PoolWithFailover::PoolWithFailover(const Poco::Util::AbstractConfiguration & cfg { std::string replica_name = config_name + "." + replica_config_key; - int priority = cfg.getInt(replica_name + ".priority", 0); + int priority = config.getInt(replica_name + ".priority", 0); replicas_by_priority[priority].emplace_back( - std::make_shared(cfg, replica_name, default_connections, max_connections, config_name.c_str())); + std::make_shared(config, replica_name, default_connections, max_connections, config_name.c_str())); } } } else { replicas_by_priority[0].emplace_back( - std::make_shared(cfg, config_name, default_connections, max_connections)); + std::make_shared(config, config_name, default_connections, max_connections)); } } diff --git a/base/mysqlxx/src/Query.cpp b/base/mysqlxx/src/Query.cpp index dc5c3274641..a0e7cf1a483 100644 --- a/base/mysqlxx/src/Query.cpp +++ b/base/mysqlxx/src/Query.cpp @@ -11,7 +11,7 @@ namespace mysqlxx { -Query::Query(Connection * conn_, const std::string & query_string) : std::ostream(0), conn(conn_) +Query::Query(Connection * conn_, const std::string & query_string) : std::ostream(nullptr), conn(conn_) { /// Важно в случае, если Query используется не из того же потока, что Connection. mysql_thread_init(); @@ -27,7 +27,7 @@ Query::Query(Connection * conn_, const std::string & query_string) : std::ostrea imbue(std::locale::classic()); } -Query::Query(const Query & other) : std::ostream(0), conn(other.conn) +Query::Query(const Query & other) : std::ostream(nullptr), conn(other.conn) { /// Важно в случае, если Query используется не из того же потока, что Connection. mysql_thread_init(); diff --git a/dbms/src/Common/Dwarf.cpp b/dbms/src/Common/Dwarf.cpp index 38606f8a8a8..1a73d7866c2 100644 --- a/dbms/src/Common/Dwarf.cpp +++ b/dbms/src/Common/Dwarf.cpp @@ -847,7 +847,6 @@ bool Dwarf::LineNumberVM::nextDefineFile(std::string_view & program, FileName & } program.remove_prefix(length); - continue; } return false; diff --git a/dbms/src/Common/Dwarf.h b/dbms/src/Common/Dwarf.h index 9abb526a210..40badc1c5a4 100644 --- a/dbms/src/Common/Dwarf.h +++ b/dbms/src/Common/Dwarf.h @@ -186,7 +186,7 @@ private: public: LineNumberVM(std::string_view data, std::string_view compilationDirectory); - bool findAddress(uintptr_t address, Path & file, uint64_t & line); + bool findAddress(uintptr_t target, Path & file, uint64_t & line); private: void init(); @@ -210,8 +210,8 @@ private: // otherwise, 1-based index in the list of include directories uint64_t directoryIndex; }; - // Read one FileName object, remove_prefix sp - static bool readFileName(std::string_view & sp, FileName & fn); + // Read one FileName object, remove_prefix program + static bool readFileName(std::string_view & program, FileName & fn); // Get file name at given index; may be in the initial table // (fileNames_) or defined using DW_LNE_define_file (and we reexecute @@ -259,8 +259,8 @@ private: uint64_t discriminator_; }; - // Read an abbreviation from a std::string_view, return true if at end; remove_prefix sp - static bool readAbbreviation(std::string_view & sp, DIEAbbreviation & abbr); + // Read an abbreviation from a std::string_view, return true if at end; remove_prefix section + static bool readAbbreviation(std::string_view & section, DIEAbbreviation & abbr); // Get abbreviation corresponding to a code, in the chunk starting at // offset in the .debug_abbrev section diff --git a/dbms/src/Common/Exception.cpp b/dbms/src/Common/Exception.cpp index 560fe6ed4f8..97219379ee8 100644 --- a/dbms/src/Common/Exception.cpp +++ b/dbms/src/Common/Exception.cpp @@ -25,10 +25,6 @@ namespace ErrorCodes } -Exception::Exception() -{ -} - Exception::Exception(const std::string & msg, int code) : Poco::Exception(msg, code) { @@ -82,12 +78,12 @@ std::string Exception::getStackTraceString() const } -std::string errnoToString(int code, int e) +std::string errnoToString(int code, int the_errno) { const size_t buf_size = 128; char buf[buf_size]; #ifndef _GNU_SOURCE - int rc = strerror_r(e, buf, buf_size); + int rc = strerror_r(the_errno, buf, buf_size); #ifdef __APPLE__ if (rc != 0 && rc != EINVAL) #else @@ -100,16 +96,16 @@ std::string errnoToString(int code, int e) strcpy(buf, unknown_message); strcpy(buf + strlen(unknown_message), code_str); } - return "errno: " + toString(e) + ", strerror: " + std::string(buf); + return "errno: " + toString(the_errno) + ", strerror: " + std::string(buf); #else (void)code; - return "errno: " + toString(e) + ", strerror: " + std::string(strerror_r(e, buf, sizeof(buf))); + return "errno: " + toString(the_errno) + ", strerror: " + std::string(strerror_r(the_errno, buf, sizeof(buf))); #endif } -void throwFromErrno(const std::string & s, int code, int e) +void throwFromErrno(const std::string & s, int code, int the_errno) { - throw ErrnoException(s + ", " + errnoToString(code, e), code, e); + throw ErrnoException(s + ", " + errnoToString(code, the_errno), code, the_errno); } void throwFromErrnoWithPath(const std::string & s, const std::string & path, int code, int the_errno) @@ -267,9 +263,9 @@ int getCurrentExceptionCode() void rethrowFirstException(const Exceptions & exceptions) { - for (size_t i = 0, size = exceptions.size(); i < size; ++i) - if (exceptions[i]) - std::rethrow_exception(exceptions[i]); + for (auto & exception : exceptions) + if (exception) + std::rethrow_exception(exception); } diff --git a/dbms/src/Common/Exception.h b/dbms/src/Common/Exception.h index 19a0cbda0f7..5b2023aed06 100644 --- a/dbms/src/Common/Exception.h +++ b/dbms/src/Common/Exception.h @@ -21,7 +21,7 @@ namespace ErrorCodes class Exception : public Poco::Exception { public: - Exception(); + Exception() = default; Exception(const std::string & msg, int code); enum CreateFromPocoTag { CreateFromPoco }; diff --git a/dbms/src/Common/FileChecker.cpp b/dbms/src/Common/FileChecker.cpp index e6fcfef2ad6..687b4dccca7 100644 --- a/dbms/src/Common/FileChecker.cpp +++ b/dbms/src/Common/FileChecker.cpp @@ -103,7 +103,7 @@ void FileChecker::save() const /// `escapeForFileName` is not really needed. But it is left for compatibility with the old code. writeJSONString(escapeForFileName(it->first), *out, settings); - writeString(":{\"size\":\"", *out); + writeString(R"(:{"size":")", *out); writeIntText(it->second, *out); writeString("\"}", *out); } @@ -136,8 +136,8 @@ void FileChecker::load(Map & local_map, const String & path) const JSON json(out.str()); JSON files = json["yandex"]; - for (const JSON name_value : files) - local_map[unescapeForFileName(name_value.getName())] = name_value.getValue()["size"].toUInt(); + for (const JSON file : files) // NOLINT + local_map[unescapeForFileName(file.getName())] = file.getValue()["size"].toUInt(); } } diff --git a/dbms/src/Common/Macros.cpp b/dbms/src/Common/Macros.cpp index a234bdd8be6..cb257cf95a4 100644 --- a/dbms/src/Common/Macros.cpp +++ b/dbms/src/Common/Macros.cpp @@ -12,8 +12,6 @@ namespace ErrorCodes extern const int SYNTAX_ERROR; } -Macros::Macros() {} - Macros::Macros(const Poco::Util::AbstractConfiguration & config, const String & root_key) { Poco::Util::AbstractConfiguration::Keys keys; diff --git a/dbms/src/Common/Macros.h b/dbms/src/Common/Macros.h index 9c2429cda14..3409cf542b8 100644 --- a/dbms/src/Common/Macros.h +++ b/dbms/src/Common/Macros.h @@ -23,7 +23,7 @@ namespace DB class Macros { public: - Macros(); + Macros() = default; Macros(const Poco::Util::AbstractConfiguration & config, const String & key); /** Replace the substring of the form {macro_name} with the value for macro_name, obtained from the config file. diff --git a/dbms/src/Common/OptimizedRegularExpression.cpp b/dbms/src/Common/OptimizedRegularExpression.cpp index ffa180f6d72..ce660c1e2ab 100644 --- a/dbms/src/Common/OptimizedRegularExpression.cpp +++ b/dbms/src/Common/OptimizedRegularExpression.cpp @@ -361,7 +361,7 @@ bool OptimizedRegularExpressionImpl::match(const char * subject, si { match.offset = pos - haystack; match.length = required_substring.size(); - return 1; + return true; } } else From 1657b9fb60bff7466591ecc503a92205abb3454e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 9 Mar 2020 00:18:53 +0300 Subject: [PATCH 136/712] clang-tidy, part 2 --- dbms/src/Common/RemoteHostFilter.cpp | 6 +++--- dbms/src/Common/SensitiveDataMasker.cpp | 2 -- dbms/src/Common/SensitiveDataMasker.h | 1 - dbms/src/Common/SymbolIndex.cpp | 4 +--- dbms/src/Common/TaskStatsInfoGetter.h | 2 +- dbms/src/Common/filesystemHelpers.cpp | 6 +++--- dbms/src/Common/getMultipleKeysFromConfig.cpp | 2 +- dbms/src/Common/parseGlobs.h | 2 +- dbms/src/Common/parseRemoteDescription.cpp | 6 +++--- dbms/src/IO/BrotliReadBuffer.cpp | 4 +--- dbms/src/IO/HDFSCommon.cpp | 2 +- dbms/src/IO/HDFSCommon.h | 2 +- dbms/src/IO/ReadBufferFromFileBase.cpp | 4 +--- dbms/src/IO/ReadBufferFromHDFS.cpp | 4 ---- dbms/src/IO/ReadBufferFromHDFS.h | 2 -- dbms/src/IO/WriteHelpers.h | 2 +- 16 files changed, 18 insertions(+), 33 deletions(-) diff --git a/dbms/src/Common/RemoteHostFilter.cpp b/dbms/src/Common/RemoteHostFilter.cpp index 4c4aa3bca81..fb6fc4e9bc3 100644 --- a/dbms/src/Common/RemoteHostFilter.cpp +++ b/dbms/src/Common/RemoteHostFilter.cpp @@ -35,7 +35,7 @@ void RemoteHostFilter::setValuesFromConfig(const Poco::Util::AbstractConfigurati { std::vector keys; config.keys("remote_url_allow_hosts", keys); - for (auto key : keys) + for (const auto & key : keys) { if (startsWith(key, "host_regexp")) regexp_hosts.push_back(config.getString("remote_url_allow_hosts." + key)); @@ -51,8 +51,8 @@ bool RemoteHostFilter::checkForDirectEntry(const std::string & str) const { if (primary_hosts.find(str) == primary_hosts.end()) { - for (size_t i = 0; i < regexp_hosts.size(); ++i) - if (re2::RE2::FullMatch(str, regexp_hosts[i])) + for (const auto & regexp : regexp_hosts) + if (re2::RE2::FullMatch(str, regexp)) return true; return false; } diff --git a/dbms/src/Common/SensitiveDataMasker.cpp b/dbms/src/Common/SensitiveDataMasker.cpp index 23014e8956e..a57ed152c6d 100644 --- a/dbms/src/Common/SensitiveDataMasker.cpp +++ b/dbms/src/Common/SensitiveDataMasker.cpp @@ -153,8 +153,6 @@ SensitiveDataMasker::SensitiveDataMasker(const Poco::Util::AbstractConfiguration } } -SensitiveDataMasker::~SensitiveDataMasker() {} - void SensitiveDataMasker::addMaskingRule( const std::string & name, const std::string & regexp_string, const std::string & replacement_string) { diff --git a/dbms/src/Common/SensitiveDataMasker.h b/dbms/src/Common/SensitiveDataMasker.h index 99a5c8c72b4..ebf7a587fdf 100644 --- a/dbms/src/Common/SensitiveDataMasker.h +++ b/dbms/src/Common/SensitiveDataMasker.h @@ -49,7 +49,6 @@ private: public: SensitiveDataMasker(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix); - ~SensitiveDataMasker(); /// Returns the number of matched rules. size_t wipeSensitiveData(std::string & data) const; diff --git a/dbms/src/Common/SymbolIndex.cpp b/dbms/src/Common/SymbolIndex.cpp index ec8fd2ffa9d..a9cdc1fa867 100644 --- a/dbms/src/Common/SymbolIndex.cpp +++ b/dbms/src/Common/SymbolIndex.cpp @@ -249,9 +249,7 @@ bool searchAndCollectSymbolsFromELFSymbolTable( else if (section.header.sh_type == SHT_STRTAB && 0 == strcmp(section.name(), string_table_name)) string_table.emplace(section); - if (symbol_table && string_table) - return true; - return false; + return (symbol_table && string_table); })) { return false; diff --git a/dbms/src/Common/TaskStatsInfoGetter.h b/dbms/src/Common/TaskStatsInfoGetter.h index 1caa2a0fd92..abf946c216f 100644 --- a/dbms/src/Common/TaskStatsInfoGetter.h +++ b/dbms/src/Common/TaskStatsInfoGetter.h @@ -16,7 +16,7 @@ public: TaskStatsInfoGetter(); ~TaskStatsInfoGetter(); - void getStat(::taskstats & stat, pid_t tid); + void getStat(::taskstats & out_stats, pid_t tid); /// Whether the current process has permissions (sudo or cap_net_admin capabilties) to get taskstats info static bool checkPermissions(); diff --git a/dbms/src/Common/filesystemHelpers.cpp b/dbms/src/Common/filesystemHelpers.cpp index 4fca3bd1d4e..bbd0b592ee5 100644 --- a/dbms/src/Common/filesystemHelpers.cpp +++ b/dbms/src/Common/filesystemHelpers.cpp @@ -22,10 +22,10 @@ bool enoughSpaceInDirectory(const std::string & path [[maybe_unused]], size_t da { #if POCO_VERSION >= 0x01090000 auto free_space = Poco::File(path).freeSpace(); - if (data_size > free_space) - return false; -#endif + return data_size <= free_space; +#else return true; +#endif } std::unique_ptr createTemporaryFile(const std::string & path) diff --git a/dbms/src/Common/getMultipleKeysFromConfig.cpp b/dbms/src/Common/getMultipleKeysFromConfig.cpp index ecad5cc45eb..7cf49fcc34d 100644 --- a/dbms/src/Common/getMultipleKeysFromConfig.cpp +++ b/dbms/src/Common/getMultipleKeysFromConfig.cpp @@ -12,7 +12,7 @@ std::vector getMultipleKeysFromConfig(const Poco::Util::AbstractCon config.keys(root, config_keys); for (const auto & key : config_keys) { - if (key != name && !(startsWith(key.data(), name + "[") && endsWith(key.data(), "]"))) + if (key != name && !(startsWith(key, name + "[") && endsWith(key, "]"))) continue; values.emplace_back(key); } diff --git a/dbms/src/Common/parseGlobs.h b/dbms/src/Common/parseGlobs.h index 24fff0c3d0a..043a87884cf 100644 --- a/dbms/src/Common/parseGlobs.h +++ b/dbms/src/Common/parseGlobs.h @@ -6,5 +6,5 @@ namespace DB { /* Parse globs in string and make a regexp for it. */ -std::string makeRegexpPatternFromGlobs(const std::string & path); +std::string makeRegexpPatternFromGlobs(const std::string & initial_str_with_globs); } diff --git a/dbms/src/Common/parseRemoteDescription.cpp b/dbms/src/Common/parseRemoteDescription.cpp index f9d56ac6715..cc89af26d99 100644 --- a/dbms/src/Common/parseRemoteDescription.cpp +++ b/dbms/src/Common/parseRemoteDescription.cpp @@ -26,9 +26,9 @@ static void append(std::vector & to, const std::vector & what, s throw Exception("Table function 'remote': first argument generates too many result addresses", ErrorCodes::BAD_ARGUMENTS); std::vector res; - for (size_t i = 0; i < to.size(); ++i) - for (size_t j = 0; j < what.size(); ++j) - res.push_back(to[i] + what[j]); + for (const auto & elem_to : to) + for (const auto & elem_what : what) + res.push_back(elem_to + elem_what); to.swap(res); } diff --git a/dbms/src/IO/BrotliReadBuffer.cpp b/dbms/src/IO/BrotliReadBuffer.cpp index 9a781ea6f0b..f68424cd54e 100644 --- a/dbms/src/IO/BrotliReadBuffer.cpp +++ b/dbms/src/IO/BrotliReadBuffer.cpp @@ -44,9 +44,7 @@ BrotliReadBuffer::BrotliReadBuffer(std::unique_ptr in_, size_t buf_s { } -BrotliReadBuffer::~BrotliReadBuffer() -{ -} +BrotliReadBuffer::~BrotliReadBuffer() = default; bool BrotliReadBuffer::nextImpl() { diff --git a/dbms/src/IO/HDFSCommon.cpp b/dbms/src/IO/HDFSCommon.cpp index 2bd420131b0..d952d81ac89 100644 --- a/dbms/src/IO/HDFSCommon.cpp +++ b/dbms/src/IO/HDFSCommon.cpp @@ -33,7 +33,7 @@ HDFSBuilderPtr createHDFSBuilder(const std::string & uri_str) if (!user_info.empty() && user_info.front() != ':') { std::string user; - size_t delim_pos = user_info.find(":"); + size_t delim_pos = user_info.find(':'); if (delim_pos != std::string::npos) user = user_info.substr(0, delim_pos); else diff --git a/dbms/src/IO/HDFSCommon.h b/dbms/src/IO/HDFSCommon.h index c84990dfea1..d50f6acf2b2 100644 --- a/dbms/src/IO/HDFSCommon.h +++ b/dbms/src/IO/HDFSCommon.h @@ -51,7 +51,7 @@ using HDFSFSPtr = std::unique_ptr, detail::HDFSFsD // set read/connect timeout, default value in libhdfs3 is about 1 hour, and too large /// TODO Allow to tune from query Settings. -HDFSBuilderPtr createHDFSBuilder(const std::string & hdfs_uri); +HDFSBuilderPtr createHDFSBuilder(const std::string & uri_str); HDFSFSPtr createHDFSFS(hdfsBuilder * builder); } #endif diff --git a/dbms/src/IO/ReadBufferFromFileBase.cpp b/dbms/src/IO/ReadBufferFromFileBase.cpp index feaba0abdd3..b598501a608 100644 --- a/dbms/src/IO/ReadBufferFromFileBase.cpp +++ b/dbms/src/IO/ReadBufferFromFileBase.cpp @@ -12,8 +12,6 @@ ReadBufferFromFileBase::ReadBufferFromFileBase(size_t buf_size, char * existing_ { } -ReadBufferFromFileBase::~ReadBufferFromFileBase() -{ -} +ReadBufferFromFileBase::~ReadBufferFromFileBase() = default; } diff --git a/dbms/src/IO/ReadBufferFromHDFS.cpp b/dbms/src/IO/ReadBufferFromHDFS.cpp index 48409683799..5d9fafac86e 100644 --- a/dbms/src/IO/ReadBufferFromHDFS.cpp +++ b/dbms/src/IO/ReadBufferFromHDFS.cpp @@ -67,10 +67,6 @@ bool ReadBufferFromHDFS::nextImpl() return true; } -ReadBufferFromHDFS::~ReadBufferFromHDFS() -{ -} - } #endif diff --git a/dbms/src/IO/ReadBufferFromHDFS.h b/dbms/src/IO/ReadBufferFromHDFS.h index 6d00c8b2310..f9b684718b7 100644 --- a/dbms/src/IO/ReadBufferFromHDFS.h +++ b/dbms/src/IO/ReadBufferFromHDFS.h @@ -22,8 +22,6 @@ public: ReadBufferFromHDFS(ReadBufferFromHDFS &&) = default; bool nextImpl() override; - - ~ReadBufferFromHDFS() override; }; } #endif diff --git a/dbms/src/IO/WriteHelpers.h b/dbms/src/IO/WriteHelpers.h index aaba21a008a..09713e9c2ba 100644 --- a/dbms/src/IO/WriteHelpers.h +++ b/dbms/src/IO/WriteHelpers.h @@ -605,7 +605,7 @@ inline void writeXMLString(const StringRef & s, WriteBuffer & buf) template void formatHex(IteratorSrc src, IteratorDst dst, const size_t num_bytes); void formatUUID(const UInt8 * src16, UInt8 * dst36); -void formatUUID(std::reverse_iterator dst16, UInt8 * dst36); +void formatUUID(std::reverse_iterator src16, UInt8 * dst36); inline void writeUUIDText(const UUID & uuid, WriteBuffer & buf) { From b53ec30978ab4de80916e5d1f2c9bf40a236b650 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 9 Mar 2020 00:29:00 +0300 Subject: [PATCH 137/712] clang-tidy, part 3 --- dbms/programs/server/TCPHandler.cpp | 2 +- .../AggregateFunctionState.cpp | 2 +- dbms/src/Common/Config/ConfigProcessor.h | 2 +- dbms/src/Common/ZooKeeper/IKeeper.cpp | 5 +---- dbms/src/Common/ZooKeeper/Lock.cpp | 10 +++++----- dbms/src/Common/ZooKeeper/TestKeeper.cpp | 16 ++++++++-------- dbms/src/Common/ZooKeeper/ZooKeeper.cpp | 12 +++++------- dbms/src/Common/tests/pod_array.cpp | 2 +- dbms/src/Compression/LZ4_decompress_faster.cpp | 4 ++-- .../src/DataStreams/DistinctBlockInputStream.cpp | 2 +- dbms/src/DataStreams/FilterBlockInputStream.cpp | 2 +- .../DataStreams/TotalsHavingBlockInputStream.cpp | 2 +- dbms/src/Functions/IFunction.cpp | 16 ++++++++-------- dbms/src/IO/parseDateTimeBestEffort.cpp | 2 +- dbms/src/IO/tests/hashing_buffer.h | 2 +- dbms/src/Interpreters/AsynchronousMetrics.cpp | 2 +- dbms/src/Parsers/ASTTablesInSelectQuery.cpp | 2 +- dbms/src/Parsers/ExpressionListParsers.cpp | 2 +- .../Storages/Kafka/KafkaBlockOutputStream.cpp | 4 ---- dbms/src/Storages/Kafka/KafkaBlockOutputStream.h | 1 - .../Kafka/ReadBufferFromKafkaConsumer.cpp | 2 +- dbms/src/Storages/LiveView/StorageLiveView.cpp | 2 +- 22 files changed, 43 insertions(+), 53 deletions(-) diff --git a/dbms/programs/server/TCPHandler.cpp b/dbms/programs/server/TCPHandler.cpp index dc6f0094b6b..c92b73aabbe 100644 --- a/dbms/programs/server/TCPHandler.cpp +++ b/dbms/programs/server/TCPHandler.cpp @@ -127,7 +127,7 @@ void TCPHandler::runImpl() connection_context.setProgressCallback([this] (const Progress & value) { return this->updateProgress(value); }); - while (1) + while (true) { /// We are waiting for a packet from the client. Thus, every `poll_interval` seconds check whether we need to shut down. { diff --git a/dbms/src/AggregateFunctions/AggregateFunctionState.cpp b/dbms/src/AggregateFunctions/AggregateFunctionState.cpp index 02f023fc8cf..fd92953d114 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionState.cpp +++ b/dbms/src/AggregateFunctions/AggregateFunctionState.cpp @@ -52,7 +52,7 @@ DataTypePtr AggregateFunctionState::getReturnType() const return arguments[0]; } - if (arguments.size() > 0) + if (!arguments.empty()) { DataTypePtr argument_type_ptr = arguments[0]; WhichDataType which(*argument_type_ptr); diff --git a/dbms/src/Common/Config/ConfigProcessor.h b/dbms/src/Common/Config/ConfigProcessor.h index d4dc48d8035..ae88234f077 100644 --- a/dbms/src/Common/Config/ConfigProcessor.h +++ b/dbms/src/Common/Config/ConfigProcessor.h @@ -127,7 +127,7 @@ private: private: using NodePtr = Poco::AutoPtr; - void mergeRecursive(XMLDocumentPtr config, Poco::XML::Node * config_node, const Poco::XML::Node * with_node); + void mergeRecursive(XMLDocumentPtr config, Poco::XML::Node * config_root, const Poco::XML::Node * with_root); void merge(XMLDocumentPtr config, XMLDocumentPtr with); diff --git a/dbms/src/Common/ZooKeeper/IKeeper.cpp b/dbms/src/Common/ZooKeeper/IKeeper.cpp index 8156da92228..5c27971038f 100644 --- a/dbms/src/Common/ZooKeeper/IKeeper.cpp +++ b/dbms/src/Common/ZooKeeper/IKeeper.cpp @@ -49,10 +49,7 @@ Exception::Exception(const int32_t code_, const std::string & path) { } -Exception::Exception(const Exception & exc) - : DB::Exception(exc), code(exc.code) -{ -} +Exception::Exception(const Exception & exc) = default; using namespace DB; diff --git a/dbms/src/Common/ZooKeeper/Lock.cpp b/dbms/src/Common/ZooKeeper/Lock.cpp index 423beec26df..c434ec3e172 100644 --- a/dbms/src/Common/ZooKeeper/Lock.cpp +++ b/dbms/src/Common/ZooKeeper/Lock.cpp @@ -11,7 +11,7 @@ bool Lock::tryLock() { /// проверим, что нода создана и я ее владелец if (tryCheck() != Status::LOCKED_BY_ME) - locked.reset(nullptr); + locked.reset(); } else { @@ -20,11 +20,11 @@ bool Lock::tryLock() if (code == Coordination::ZNODEEXISTS) { - locked.reset(nullptr); + locked.reset(); } else if (code == Coordination::ZOK) { - locked.reset(new ZooKeeperHandler(zookeeper)); + locked = std::make_unique(zookeeper); } else { @@ -41,7 +41,7 @@ void Lock::unlock() auto zookeeper = zookeeper_holder->getZooKeeper(); if (tryCheck() == Status::LOCKED_BY_ME) zookeeper->remove(lock_path, -1); - locked.reset(nullptr); + locked.reset(); } } @@ -71,6 +71,6 @@ Lock::Status Lock::tryCheck() const void Lock::unlockAssumeLockNodeRemovedManually() { - locked.reset(nullptr); + locked.reset(); } diff --git a/dbms/src/Common/ZooKeeper/TestKeeper.cpp b/dbms/src/Common/ZooKeeper/TestKeeper.cpp index 7c7d9bba016..cb53ae52cc3 100644 --- a/dbms/src/Common/ZooKeeper/TestKeeper.cpp +++ b/dbms/src/Common/ZooKeeper/TestKeeper.cpp @@ -66,7 +66,7 @@ static void processWatchesImpl(const String & path, TestKeeper::Watches & watche struct TestKeeperCreateRequest final : CreateRequest, TestKeeperRequest { - TestKeeperCreateRequest() {} + TestKeeperCreateRequest() = default; TestKeeperCreateRequest(const CreateRequest & base) : CreateRequest(base) {} ResponsePtr createResponse() const override; ResponsePtr process(TestKeeper::Container & container, int64_t zxid) const override; @@ -79,7 +79,7 @@ struct TestKeeperCreateRequest final : CreateRequest, TestKeeperRequest struct TestKeeperRemoveRequest final : RemoveRequest, TestKeeperRequest { - TestKeeperRemoveRequest() {} + TestKeeperRemoveRequest() = default; TestKeeperRemoveRequest(const RemoveRequest & base) : RemoveRequest(base) {} bool isMutable() const override { return true; } ResponsePtr createResponse() const override; @@ -99,14 +99,14 @@ struct TestKeeperExistsRequest final : ExistsRequest, TestKeeperRequest struct TestKeeperGetRequest final : GetRequest, TestKeeperRequest { - TestKeeperGetRequest() {} + TestKeeperGetRequest() = default; ResponsePtr createResponse() const override; ResponsePtr process(TestKeeper::Container & container, int64_t zxid) const override; }; struct TestKeeperSetRequest final : SetRequest, TestKeeperRequest { - TestKeeperSetRequest() {} + TestKeeperSetRequest() = default; TestKeeperSetRequest(const SetRequest & base) : SetRequest(base) {} bool isMutable() const override { return true; } ResponsePtr createResponse() const override; @@ -126,7 +126,7 @@ struct TestKeeperListRequest final : ListRequest, TestKeeperRequest struct TestKeeperCheckRequest final : CheckRequest, TestKeeperRequest { - TestKeeperCheckRequest() {} + TestKeeperCheckRequest() = default; TestKeeperCheckRequest(const CheckRequest & base) : CheckRequest(base) {} ResponsePtr createResponse() const override; ResponsePtr process(TestKeeper::Container & container, int64_t zxid) const override; @@ -574,11 +574,11 @@ void TestKeeper::finalize() } } -void TestKeeper::pushRequest(RequestInfo && info) +void TestKeeper::pushRequest(RequestInfo && request) { try { - info.time = clock::now(); + request.time = clock::now(); /// We must serialize 'pushRequest' and 'finalize' (from processingThread) calls /// to avoid forgotten operations in the queue when session is expired. @@ -589,7 +589,7 @@ void TestKeeper::pushRequest(RequestInfo && info) if (expired) throw Exception("Session expired", ZSESSIONEXPIRED); - if (!requests_queue.tryPush(std::move(info), operation_timeout.totalMilliseconds())) + if (!requests_queue.tryPush(std::move(request), operation_timeout.totalMilliseconds())) throw Exception("Cannot push request to queue within operation timeout", ZOPERATIONTIMEOUT); } catch (...) diff --git a/dbms/src/Common/ZooKeeper/ZooKeeper.cpp b/dbms/src/Common/ZooKeeper/ZooKeeper.cpp index ae5b548f7e5..f5a630d6a5b 100644 --- a/dbms/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/dbms/src/Common/ZooKeeper/ZooKeeper.cpp @@ -164,8 +164,8 @@ struct ZooKeeperArgs for (auto & host : hosts_strings) { - if (hosts.size()) - hosts += ","; + if (!hosts.empty()) + hosts += ','; hosts += host; } @@ -281,10 +281,10 @@ int32_t ZooKeeper::createImpl(const std::string & path, const std::string & data return code; } -std::string ZooKeeper::create(const std::string & path, const std::string & data, int32_t type) +std::string ZooKeeper::create(const std::string & path, const std::string & data, int32_t mode) { std::string path_created; - check(tryCreate(path, data, type, path_created), path); + check(tryCreate(path, data, mode, path_created), path); return path_created; } @@ -393,9 +393,7 @@ bool ZooKeeper::existsWatch(const std::string & path, Coordination::Stat * stat, if (!(code == Coordination::ZOK || code == Coordination::ZNONODE)) throw KeeperException(code, path); - if (code == Coordination::ZNONODE) - return false; - return true; + return code != Coordination::ZNONODE; } int32_t ZooKeeper::getImpl(const std::string & path, std::string & res, Coordination::Stat * stat, Coordination::WatchCallback watch_callback) diff --git a/dbms/src/Common/tests/pod_array.cpp b/dbms/src/Common/tests/pod_array.cpp index f9d24d439f7..90b1125afd6 100644 --- a/dbms/src/Common/tests/pod_array.cpp +++ b/dbms/src/Common/tests/pod_array.cpp @@ -12,7 +12,7 @@ do \ if ((res)) { (res) = false; } \ } \ } \ -while (0) +while (false) static void test1() { diff --git a/dbms/src/Compression/LZ4_decompress_faster.cpp b/dbms/src/Compression/LZ4_decompress_faster.cpp index 77f9e226de4..cdc5148b199 100644 --- a/dbms/src/Compression/LZ4_decompress_faster.cpp +++ b/dbms/src/Compression/LZ4_decompress_faster.cpp @@ -428,7 +428,7 @@ void NO_INLINE decompressImpl( #if defined(__clang__) #pragma nounroll #endif - while (1) + while (true) { size_t length; @@ -605,7 +605,7 @@ void statistics( const UInt8 * ip = reinterpret_cast(source); UInt8 * op = reinterpret_cast(dest); UInt8 * const output_end = op + dest_size; - while (1) + while (true) { size_t length; diff --git a/dbms/src/DataStreams/DistinctBlockInputStream.cpp b/dbms/src/DataStreams/DistinctBlockInputStream.cpp index 54d7134d0cd..eaebebd4a46 100644 --- a/dbms/src/DataStreams/DistinctBlockInputStream.cpp +++ b/dbms/src/DataStreams/DistinctBlockInputStream.cpp @@ -20,7 +20,7 @@ Block DistinctBlockInputStream::readImpl() { /// Execute until end of stream or until /// a block with some new records will be gotten. - while (1) + while (true) { if (no_more_rows) return Block(); diff --git a/dbms/src/DataStreams/FilterBlockInputStream.cpp b/dbms/src/DataStreams/FilterBlockInputStream.cpp index 43ee0faa09e..3b3cea11412 100644 --- a/dbms/src/DataStreams/FilterBlockInputStream.cpp +++ b/dbms/src/DataStreams/FilterBlockInputStream.cpp @@ -77,7 +77,7 @@ Block FilterBlockInputStream::readImpl() return {}; /// Until non-empty block after filtering or end of stream. - while (1) + while (true) { res = children.back()->read(); if (!res) diff --git a/dbms/src/DataStreams/TotalsHavingBlockInputStream.cpp b/dbms/src/DataStreams/TotalsHavingBlockInputStream.cpp index 058381ad6bd..f8a7742c0d0 100644 --- a/dbms/src/DataStreams/TotalsHavingBlockInputStream.cpp +++ b/dbms/src/DataStreams/TotalsHavingBlockInputStream.cpp @@ -81,7 +81,7 @@ Block TotalsHavingBlockInputStream::readImpl() Block finalized; Block block; - while (1) + while (true) { block = children[0]->read(); diff --git a/dbms/src/Functions/IFunction.cpp b/dbms/src/Functions/IFunction.cpp index 0f785cfcd9e..7355b5df9ad 100644 --- a/dbms/src/Functions/IFunction.cpp +++ b/dbms/src/Functions/IFunction.cpp @@ -413,19 +413,19 @@ static void convertLowCardinalityColumnsToFull(Block & block, const ColumnNumber } } -void ExecutableFunctionAdaptor::execute(Block & block, const ColumnNumbers & args, size_t result, size_t input_rows_count, bool dry_run) +void ExecutableFunctionAdaptor::execute(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count, bool dry_run) { if (impl->useDefaultImplementationForLowCardinalityColumns()) { auto & res = block.safeGetByPosition(result); Block block_without_low_cardinality = block.cloneWithoutColumns(); - for (auto arg : args) + for (auto arg : arguments) block_without_low_cardinality.safeGetByPosition(arg).column = block.safeGetByPosition(arg).column; if (auto * res_low_cardinality_type = typeid_cast(res.type.get())) { - const auto * low_cardinality_column = findLowCardinalityArgument(block, args); + const auto * low_cardinality_column = findLowCardinalityArgument(block, arguments); bool can_be_executed_on_default_arguments = impl->canBeExecutedOnDefaultArguments(); bool use_cache = low_cardinality_result_cache && can_be_executed_on_default_arguments && low_cardinality_column && low_cardinality_column->isSharedDictionary(); @@ -447,9 +447,9 @@ void ExecutableFunctionAdaptor::execute(Block & block, const ColumnNumbers & arg block_without_low_cardinality.safeGetByPosition(result).type = res_low_cardinality_type->getDictionaryType(); ColumnPtr indexes = replaceLowCardinalityColumnsByNestedAndGetDictionaryIndexes( - block_without_low_cardinality, args, can_be_executed_on_default_arguments, input_rows_count); + block_without_low_cardinality, arguments, can_be_executed_on_default_arguments, input_rows_count); - executeWithoutLowCardinalityColumns(block_without_low_cardinality, args, result, block_without_low_cardinality.rows(), dry_run); + executeWithoutLowCardinalityColumns(block_without_low_cardinality, arguments, result, block_without_low_cardinality.rows(), dry_run); auto keys = block_without_low_cardinality.safeGetByPosition(result).column->convertToFullColumnIfConst(); @@ -480,13 +480,13 @@ void ExecutableFunctionAdaptor::execute(Block & block, const ColumnNumbers & arg } else { - convertLowCardinalityColumnsToFull(block_without_low_cardinality, args); - executeWithoutLowCardinalityColumns(block_without_low_cardinality, args, result, input_rows_count, dry_run); + convertLowCardinalityColumnsToFull(block_without_low_cardinality, arguments); + executeWithoutLowCardinalityColumns(block_without_low_cardinality, arguments, result, input_rows_count, dry_run); res.column = block_without_low_cardinality.safeGetByPosition(result).column; } } else - executeWithoutLowCardinalityColumns(block, args, result, input_rows_count, dry_run); + executeWithoutLowCardinalityColumns(block, arguments, result, input_rows_count, dry_run); } void FunctionOverloadResolverAdaptor::checkNumberOfArguments(size_t number_of_arguments) const diff --git a/dbms/src/IO/parseDateTimeBestEffort.cpp b/dbms/src/IO/parseDateTimeBestEffort.cpp index 801810a3044..59fb39ed846 100644 --- a/dbms/src/IO/parseDateTimeBestEffort.cpp +++ b/dbms/src/IO/parseDateTimeBestEffort.cpp @@ -118,7 +118,7 @@ ReturnType parseDateTimeBestEffortImpl(time_t & res, ReadBuffer & in, const Date auto read_alpha_month = [&month] (const auto & alpha) { - if (0 == strncasecmp(alpha, "Jan", 3)) month = 1; + if (0 == strncasecmp(alpha, "Jan", 3)) month = 1; else if (0 == strncasecmp(alpha, "Feb", 3)) month = 2; else if (0 == strncasecmp(alpha, "Mar", 3)) month = 3; else if (0 == strncasecmp(alpha, "Apr", 3)) month = 4; diff --git a/dbms/src/IO/tests/hashing_buffer.h b/dbms/src/IO/tests/hashing_buffer.h index f00a42d4a6a..acbfd750d74 100644 --- a/dbms/src/IO/tests/hashing_buffer.h +++ b/dbms/src/IO/tests/hashing_buffer.h @@ -1,7 +1,7 @@ #include #include -#define FAIL(msg) do { std::cout << msg; exit(1); } while (0) +#define FAIL(msg) do { std::cout << msg; exit(1); } while (false) static CityHash_v1_0_2::uint128 referenceHash(const char * data, size_t len) diff --git a/dbms/src/Interpreters/AsynchronousMetrics.cpp b/dbms/src/Interpreters/AsynchronousMetrics.cpp index 661325b22c2..fadd9e30ebc 100644 --- a/dbms/src/Interpreters/AsynchronousMetrics.cpp +++ b/dbms/src/Interpreters/AsynchronousMetrics.cpp @@ -236,7 +236,7 @@ void AsynchronousMetrics::update() size_t size = sizeof(value); \ mallctl("stats." NAME, &value, &size, nullptr, 0); \ set("jemalloc." NAME, value); \ - } while (0); + } while (false); FOR_EACH_METRIC(GET_METRIC) diff --git a/dbms/src/Parsers/ASTTablesInSelectQuery.cpp b/dbms/src/Parsers/ASTTablesInSelectQuery.cpp index 18750d5ccd3..b085f5a28ae 100644 --- a/dbms/src/Parsers/ASTTablesInSelectQuery.cpp +++ b/dbms/src/Parsers/ASTTablesInSelectQuery.cpp @@ -15,7 +15,7 @@ do \ res->children.push_back(res->member); \ } \ } \ -while (0) +while (false) ASTPtr ASTTableExpression::clone() const diff --git a/dbms/src/Parsers/ExpressionListParsers.cpp b/dbms/src/Parsers/ExpressionListParsers.cpp index f8947262a26..51e70577fc0 100644 --- a/dbms/src/Parsers/ExpressionListParsers.cpp +++ b/dbms/src/Parsers/ExpressionListParsers.cpp @@ -140,7 +140,7 @@ bool ParserLeftAssociativeBinaryOperatorList::parseImpl(Pos & pos, ASTPtr & node bool first = true; auto current_depth = pos.depth; - while (1) + while (true) { if (first) { diff --git a/dbms/src/Storages/Kafka/KafkaBlockOutputStream.cpp b/dbms/src/Storages/Kafka/KafkaBlockOutputStream.cpp index f0cfec1f191..33e5102b1dc 100644 --- a/dbms/src/Storages/Kafka/KafkaBlockOutputStream.cpp +++ b/dbms/src/Storages/Kafka/KafkaBlockOutputStream.cpp @@ -15,10 +15,6 @@ KafkaBlockOutputStream::KafkaBlockOutputStream(StorageKafka & storage_, const Co { } -KafkaBlockOutputStream::~KafkaBlockOutputStream() -{ -} - Block KafkaBlockOutputStream::getHeader() const { return storage.getSampleBlockNonMaterialized(); diff --git a/dbms/src/Storages/Kafka/KafkaBlockOutputStream.h b/dbms/src/Storages/Kafka/KafkaBlockOutputStream.h index daa1c7bd98b..f3eb3dae0ba 100644 --- a/dbms/src/Storages/Kafka/KafkaBlockOutputStream.h +++ b/dbms/src/Storages/Kafka/KafkaBlockOutputStream.h @@ -11,7 +11,6 @@ class KafkaBlockOutputStream : public IBlockOutputStream { public: explicit KafkaBlockOutputStream(StorageKafka & storage_, const Context & context_); - ~KafkaBlockOutputStream() override; Block getHeader() const override; diff --git a/dbms/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp b/dbms/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp index 7edb4c0ac4f..608d8ba2213 100644 --- a/dbms/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp +++ b/dbms/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp @@ -241,7 +241,7 @@ bool ReadBufferFromKafkaConsumer::nextImpl() commit(); size_t waited_for_assignment = 0; - while (1) + while (true) { /// Don't drop old messages immediately, since we may need them for virtual columns. auto new_messages = consumer->poll_batch(batch_size, std::chrono::milliseconds(poll_timeout)); diff --git a/dbms/src/Storages/LiveView/StorageLiveView.cpp b/dbms/src/Storages/LiveView/StorageLiveView.cpp index f126c5564a9..84965e58dc1 100644 --- a/dbms/src/Storages/LiveView/StorageLiveView.cpp +++ b/dbms/src/Storages/LiveView/StorageLiveView.cpp @@ -392,7 +392,7 @@ void StorageLiveView::noUsersThread(std::shared_ptr storage, co auto table_id = storage->getStorageID(); { - while (1) + while (true) { std::unique_lock lock(storage->no_users_thread_wakeup_mutex); if (!storage->no_users_thread_condition.wait_for(lock, std::chrono::seconds(timeout), [&] { return storage->no_users_thread_wakeup; })) From 79c6bd5ae7872d33a290ab264a7b71072255488a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 9 Mar 2020 00:40:00 +0300 Subject: [PATCH 138/712] clang-tidy, part 4 --- .clang-tidy | 2 +- .../performance-test/ConfigPreprocessor.cpp | 2 +- .../AggregateFunctionWindowFunnel.h | 2 +- dbms/src/Columns/ColumnArray.cpp | 6 +++--- dbms/src/Columns/ColumnDecimal.cpp | 2 +- dbms/src/Columns/ColumnString.cpp | 2 +- dbms/src/Common/PODArray.h | 2 +- dbms/src/Common/ZooKeeper/ZooKeeperImpl.cpp | 10 +++++----- dbms/src/Common/ZooKeeper/ZooKeeperImpl.h | 2 +- dbms/src/Common/tests/pod_array.cpp | 12 ++++++------ .../Compression/CachedCompressedReadBuffer.cpp | 2 +- .../ParallelParsingBlockInputStream.cpp | 2 +- dbms/src/DataTypes/DataTypeDateTime64.cpp | 2 +- dbms/src/Dictionaries/DictionaryStructure.cpp | 2 +- dbms/src/Functions/FunctionHelpers.cpp | 16 +++++++--------- dbms/src/Functions/FunctionsStringRegex.cpp | 2 +- .../src/Functions/array/arrayEnumerateExtended.h | 2 +- dbms/src/Functions/array/arrayEnumerateRanked.h | 2 +- dbms/src/Functions/array/arrayUniq.cpp | 2 +- .../extractTimeZoneFromFunctionArguments.cpp | 4 ++-- dbms/src/Interpreters/Aggregator.cpp | 2 +- .../PredicateExpressionsOptimizer.cpp | 2 +- dbms/src/Interpreters/ProcessList.cpp | 2 +- dbms/src/Interpreters/executeQuery.cpp | 2 +- dbms/src/Processors/Formats/LazyOutputFormat.cpp | 2 +- dbms/src/Storages/Kafka/StorageKafka.cpp | 6 +++--- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 6 +++--- dbms/src/Storages/VirtualColumnUtils.cpp | 2 +- 28 files changed, 50 insertions(+), 52 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index bbd532c8c9a..a00642e87b7 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -1,2 +1,2 @@ -Checks: '-*,google-readability-avoid-underscore-in-googletest-name,misc-throw-by-value-catch-by-reference,misc-misplaced-const,misc-unconventional-assign-operator,modernize-avoid-bind,modernize-loop-convert,modernize-make-shared,modernize-make-unique,modernize-raw-string-literal,modernize-redundant-void-arg,modernize-replace-auto-ptr,modernize-replace-random-shuffle,modernize-use-bool-literals,modernize-use-nullptr,modernize-use-using,performance-faster-string-find,performance-for-range-copy,readability-avoid-const-params-in-decls,readability-const-return-type,readability-container-size-empty,readability-convert-member-functions-to-static,readability-delete-null-pointer,readability-deleted-default,readability-make-member-function-const,readability-misleading-indentation,readability-misplaced-array-index,readability-non-const-parameter,readability-qualified-auto,readability-redundant-access-specifiers,readability-redundant-control-flow,readability-redundant-function-ptr-dereference,readability-redundant-smartptr-get,readability-redundant-string-cstr,readability-redundant-string-init,readability-static-definition-in-anonymous-namespace,readability-string-compare,readability-uniqueptr-delete-release,modernize-use-equals-default,modernize-use-equals-delete,bugprone-undelegated-constructor,readability-redundant-member-init,readability-simplify-subscript-expr,readability-simplify-boolean-expr,readability-inconsistent-declaration-parameter-name' +Checks: '-*,google-readability-avoid-underscore-in-googletest-name,misc-throw-by-value-catch-by-reference,misc-misplaced-const,misc-unconventional-assign-operator,modernize-avoid-bind,modernize-loop-convert,modernize-make-shared,modernize-make-unique,modernize-raw-string-literal,modernize-redundant-void-arg,modernize-replace-auto-ptr,modernize-replace-random-shuffle,modernize-use-bool-literals,modernize-use-nullptr,modernize-use-using,performance-faster-string-find,performance-for-range-copy,readability-avoid-const-params-in-decls,readability-const-return-type,readability-container-size-empty,readability-convert-member-functions-to-static,readability-delete-null-pointer,readability-deleted-default,readability-make-member-function-const,readability-misplaced-array-index,readability-non-const-parameter,readability-qualified-auto,readability-redundant-access-specifiers,readability-redundant-control-flow,readability-redundant-function-ptr-dereference,readability-redundant-smartptr-get,readability-redundant-string-cstr,readability-redundant-string-init,readability-static-definition-in-anonymous-namespace,readability-string-compare,readability-uniqueptr-delete-release,modernize-use-equals-default,modernize-use-equals-delete,bugprone-undelegated-constructor,readability-redundant-member-init,readability-simplify-subscript-expr,readability-simplify-boolean-expr,readability-inconsistent-declaration-parameter-name' WarningsAsErrors: '*' diff --git a/dbms/programs/performance-test/ConfigPreprocessor.cpp b/dbms/programs/performance-test/ConfigPreprocessor.cpp index 5da9650454a..8dabac0ec3a 100644 --- a/dbms/programs/performance-test/ConfigPreprocessor.cpp +++ b/dbms/programs/performance-test/ConfigPreprocessor.cpp @@ -45,7 +45,7 @@ void ConfigPreprocessor::removeConfigurationsIf( { auto checker = [&filter_type, &values, &leave] (XMLConfigurationPtr & config) { - if (values.size() == 0) + if (values.empty()) return false; bool remove_or_not = false; diff --git a/dbms/src/AggregateFunctions/AggregateFunctionWindowFunnel.h b/dbms/src/AggregateFunctions/AggregateFunctionWindowFunnel.h index 6316f69cbcb..fe89a633dec 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionWindowFunnel.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionWindowFunnel.h @@ -148,7 +148,7 @@ private: // The Algorithm complexity is O(n). UInt8 getEventLevel(const Data & data) const { - if (data.size() == 0) + if (data.empty()) return 0; if (events_size == 1) return 1; diff --git a/dbms/src/Columns/ColumnArray.cpp b/dbms/src/Columns/ColumnArray.cpp index b51340662b6..e0e5caef3bf 100644 --- a/dbms/src/Columns/ColumnArray.cpp +++ b/dbms/src/Columns/ColumnArray.cpp @@ -424,7 +424,7 @@ ColumnPtr ColumnArray::filter(const Filter & filt, ssize_t result_size_hint) con template ColumnPtr ColumnArray::filterNumber(const Filter & filt, ssize_t result_size_hint) const { - if (getOffsets().size() == 0) + if (getOffsets().empty()) return ColumnArray::create(data); auto res = ColumnArray::create(data->cloneEmpty()); @@ -551,7 +551,7 @@ ColumnPtr ColumnArray::filterGeneric(const Filter & filt, ssize_t result_size_hi ColumnPtr ColumnArray::filterNullable(const Filter & filt, ssize_t result_size_hint) const { - if (getOffsets().size() == 0) + if (getOffsets().empty()) return ColumnArray::create(data); const ColumnNullable & nullable_elems = assert_cast(*data); @@ -574,7 +574,7 @@ ColumnPtr ColumnArray::filterNullable(const Filter & filt, ssize_t result_size_h ColumnPtr ColumnArray::filterTuple(const Filter & filt, ssize_t result_size_hint) const { - if (getOffsets().size() == 0) + if (getOffsets().empty()) return ColumnArray::create(data); const ColumnTuple & tuple = assert_cast(*data); diff --git a/dbms/src/Columns/ColumnDecimal.cpp b/dbms/src/Columns/ColumnDecimal.cpp index ab80f7832a8..38c17ccacb4 100644 --- a/dbms/src/Columns/ColumnDecimal.cpp +++ b/dbms/src/Columns/ColumnDecimal.cpp @@ -217,7 +217,7 @@ void ColumnDecimal::gather(ColumnGathererStream & gatherer) template void ColumnDecimal::getExtremes(Field & min, Field & max) const { - if (data.size() == 0) + if (data.empty()) { min = NearestFieldType(0, scale); max = NearestFieldType(0, scale); diff --git a/dbms/src/Columns/ColumnString.cpp b/dbms/src/Columns/ColumnString.cpp index 8d3f3e66587..ec4d445079f 100644 --- a/dbms/src/Columns/ColumnString.cpp +++ b/dbms/src/Columns/ColumnString.cpp @@ -100,7 +100,7 @@ void ColumnString::insertRangeFrom(const IColumn & src, size_t start, size_t len ColumnPtr ColumnString::filter(const Filter & filt, ssize_t result_size_hint) const { - if (offsets.size() == 0) + if (offsets.empty()) return ColumnString::create(); auto res = ColumnString::create(); diff --git a/dbms/src/Common/PODArray.h b/dbms/src/Common/PODArray.h index 25d39719ba9..272cbdc4fe6 100644 --- a/dbms/src/Common/PODArray.h +++ b/dbms/src/Common/PODArray.h @@ -156,7 +156,7 @@ protected: template void reserveForNextSize(TAllocatorParams &&... allocator_params) { - if (size() == 0) + if (empty()) { // The allocated memory should be multiplication of ELEMENT_SIZE to hold the element, otherwise, // memory issue such as corruption could appear in edge case. diff --git a/dbms/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/dbms/src/Common/ZooKeeper/ZooKeeperImpl.cpp index 454ffcb76a9..0525ebd377b 100644 --- a/dbms/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/dbms/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -412,7 +412,7 @@ void ZooKeeperRequest::write(WriteBuffer & out) const struct ZooKeeperResponse : virtual Response { - virtual ~ZooKeeperResponse() {} + virtual ~ZooKeeperResponse() = default; virtual void readImpl(ReadBuffer &) = 0; }; @@ -480,7 +480,7 @@ struct ZooKeeperCloseResponse final : ZooKeeperResponse struct ZooKeeperCreateRequest final : CreateRequest, ZooKeeperRequest { - ZooKeeperCreateRequest() {} + ZooKeeperCreateRequest() = default; ZooKeeperCreateRequest(const CreateRequest & base) : CreateRequest(base) {} ZooKeeper::OpNum getOpNum() const override { return 1; } @@ -512,7 +512,7 @@ struct ZooKeeperCreateResponse final : CreateResponse, ZooKeeperResponse struct ZooKeeperRemoveRequest final : RemoveRequest, ZooKeeperRequest { - ZooKeeperRemoveRequest() {} + ZooKeeperRemoveRequest() = default; ZooKeeperRemoveRequest(const RemoveRequest & base) : RemoveRequest(base) {} ZooKeeper::OpNum getOpNum() const override { return 2; } @@ -570,7 +570,7 @@ struct ZooKeeperGetResponse final : GetResponse, ZooKeeperResponse struct ZooKeeperSetRequest final : SetRequest, ZooKeeperRequest { - ZooKeeperSetRequest() {} + ZooKeeperSetRequest() = default; ZooKeeperSetRequest(const SetRequest & base) : SetRequest(base) {} ZooKeeper::OpNum getOpNum() const override { return 5; } @@ -613,7 +613,7 @@ struct ZooKeeperListResponse final : ListResponse, ZooKeeperResponse struct ZooKeeperCheckRequest final : CheckRequest, ZooKeeperRequest { - ZooKeeperCheckRequest() {} + ZooKeeperCheckRequest() = default; ZooKeeperCheckRequest(const CheckRequest & base) : CheckRequest(base) {} ZooKeeper::OpNum getOpNum() const override { return 13; } diff --git a/dbms/src/Common/ZooKeeper/ZooKeeperImpl.h b/dbms/src/Common/ZooKeeper/ZooKeeperImpl.h index 8899220e005..88e949dbd45 100644 --- a/dbms/src/Common/ZooKeeper/ZooKeeperImpl.h +++ b/dbms/src/Common/ZooKeeper/ZooKeeperImpl.h @@ -196,7 +196,7 @@ private: using RequestsQueue = ConcurrentBoundedQueue; RequestsQueue requests_queue{1}; - void pushRequest(RequestInfo && request); + void pushRequest(RequestInfo && info); using Operations = std::map; diff --git a/dbms/src/Common/tests/pod_array.cpp b/dbms/src/Common/tests/pod_array.cpp index 90b1125afd6..2a3093b3de7 100644 --- a/dbms/src/Common/tests/pod_array.cpp +++ b/dbms/src/Common/tests/pod_array.cpp @@ -168,11 +168,11 @@ static void test2() ASSERT_CHECK((arr[1] == 2), res); ASSERT_CHECK((arr[2] == 3), res); - ASSERT_CHECK((arr2.size() == 0), res); + ASSERT_CHECK((arr2.empty()), res); arr.swap(arr2); - ASSERT_CHECK((arr.size() == 0), res); + ASSERT_CHECK((arr.empty()), res); ASSERT_CHECK((arr2.size() == 3), res); ASSERT_CHECK((arr2[0] == 1), res); @@ -200,11 +200,11 @@ static void test2() ASSERT_CHECK((arr[3] == 4), res); ASSERT_CHECK((arr[4] == 5), res); - ASSERT_CHECK((arr2.size() == 0), res); + ASSERT_CHECK((arr2.empty()), res); arr.swap(arr2); - ASSERT_CHECK((arr.size() == 0), res); + ASSERT_CHECK((arr.empty()), res); ASSERT_CHECK((arr2.size() == 5), res); ASSERT_CHECK((arr2[0] == 1), res); @@ -409,7 +409,7 @@ static void test3() Array arr2{std::move(arr)}; - ASSERT_CHECK((arr.size() == 0), res); + ASSERT_CHECK((arr.empty()), res); ASSERT_CHECK((arr2.size() == 3), res); ASSERT_CHECK((arr2[0] == 1), res); @@ -428,7 +428,7 @@ static void test3() Array arr2{std::move(arr)}; - ASSERT_CHECK((arr.size() == 0), res); + ASSERT_CHECK((arr.empty()), res); ASSERT_CHECK((arr2.size() == 5), res); ASSERT_CHECK((arr2[0] == 1), res); diff --git a/dbms/src/Compression/CachedCompressedReadBuffer.cpp b/dbms/src/Compression/CachedCompressedReadBuffer.cpp index 878c980d53f..f1bbc2188f0 100644 --- a/dbms/src/Compression/CachedCompressedReadBuffer.cpp +++ b/dbms/src/Compression/CachedCompressedReadBuffer.cpp @@ -61,7 +61,7 @@ bool CachedCompressedReadBuffer::nextImpl() cache->set(key, owned_cell); } - if (owned_cell->data.size() == 0) + if (owned_cell->data.empty()) return false; working_buffer = Buffer(owned_cell->data.data(), owned_cell->data.data() + owned_cell->data.size() - owned_cell->additional_bytes); diff --git a/dbms/src/DataStreams/ParallelParsingBlockInputStream.cpp b/dbms/src/DataStreams/ParallelParsingBlockInputStream.cpp index 3669e77080b..e25e31e33b8 100644 --- a/dbms/src/DataStreams/ParallelParsingBlockInputStream.cpp +++ b/dbms/src/DataStreams/ParallelParsingBlockInputStream.cpp @@ -156,7 +156,7 @@ Block ParallelParsingBlockInputStream::readImpl() next_block_in_current_unit = 0; } - if (unit.block_ext.block.size() == 0) + if (unit.block_ext.block.empty()) { /* * Can we get zero blocks for an entire segment, when the format parser diff --git a/dbms/src/DataTypes/DataTypeDateTime64.cpp b/dbms/src/DataTypes/DataTypeDateTime64.cpp index 228cfbd17c8..40909c1000b 100644 --- a/dbms/src/DataTypes/DataTypeDateTime64.cpp +++ b/dbms/src/DataTypes/DataTypeDateTime64.cpp @@ -235,7 +235,7 @@ getArgument(const ASTPtr & arguments, size_t argument_index, const char * argume static DataTypePtr create64(const String & /*type_name*/, const ASTPtr & arguments) { - if (!arguments || arguments->size() == 0) + if (!arguments || arguments->empty()) return std::make_shared(DataTypeDateTime64::default_scale); const auto scale = getArgument(arguments, 0, "scale", "DateType64"); diff --git a/dbms/src/Dictionaries/DictionaryStructure.cpp b/dbms/src/Dictionaries/DictionaryStructure.cpp index 45506f246b4..f8b8fbd6aab 100644 --- a/dbms/src/Dictionaries/DictionaryStructure.cpp +++ b/dbms/src/Dictionaries/DictionaryStructure.cpp @@ -292,7 +292,7 @@ std::vector DictionaryStructure::getAttributes( for (const auto & config_elem : config_elems) { - if (!startsWith(config_elem.data(), "attribute")) + if (!startsWith(config_elem, "attribute")) continue; const auto prefix = config_prefix + '.' + config_elem + '.'; diff --git a/dbms/src/Functions/FunctionHelpers.cpp b/dbms/src/Functions/FunctionHelpers.cpp index 97690af634f..7002879f995 100644 --- a/dbms/src/Functions/FunctionHelpers.cpp +++ b/dbms/src/Functions/FunctionHelpers.cpp @@ -112,7 +112,7 @@ void validateArgumentType(const IFunction & func, const DataTypes & arguments, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); const auto & argument = arguments[argument_index]; - if (validator_func(*argument) == false) + if (!validator_func(*argument)) throw Exception("Illegal type " + argument->getName() + " of " + std::to_string(argument_index) + " argument of function " + func.getName() + @@ -151,10 +151,10 @@ void validateArgumentsImpl(const IFunction & func, int FunctionArgumentDescriptor::isValid(const DataTypePtr & data_type, const ColumnPtr & column) const { - if (type_validator_func && (data_type == nullptr || type_validator_func(*data_type) == false)) + if (type_validator_func && (data_type == nullptr || !type_validator_func(*data_type))) return ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT; - if (column_validator_func && (column == nullptr || column_validator_func(*column) == false)) + if (column_validator_func && (column == nullptr || !column_validator_func(*column))) return ErrorCodes::ILLEGAL_COLUMN; return 0; @@ -194,20 +194,18 @@ void validateFunctionArgumentTypes(const IFunction & func, throw Exception("Incorrect number of arguments for function " + func.getName() + " provided " + std::to_string(arguments.size()) - + (arguments.size() ? " (" + joinArgumentTypes(arguments) + ")" : String{}) + + (!arguments.empty() ? " (" + joinArgumentTypes(arguments) + ")" : String{}) + ", expected " + std::to_string(mandatory_args.size()) - + (optional_args.size() ? " to " + std::to_string(mandatory_args.size() + optional_args.size()) : "") + + (!optional_args.empty() ? " to " + std::to_string(mandatory_args.size() + optional_args.size()) : "") + " (" + joinArgumentTypes(mandatory_args) - + (optional_args.size() ? ", [" + joinArgumentTypes(optional_args) + "]" : "") + + (!optional_args.empty() ? ", [" + joinArgumentTypes(optional_args) + "]" : "") + ")", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); } validateArgumentsImpl(func, arguments, 0, mandatory_args); - if (optional_args.size()) - { + if (!optional_args.empty()) validateArgumentsImpl(func, arguments, mandatory_args.size(), optional_args); - } } std::pair, const ColumnArray::Offset *> diff --git a/dbms/src/Functions/FunctionsStringRegex.cpp b/dbms/src/Functions/FunctionsStringRegex.cpp index c57830264a7..3a4d46431f6 100644 --- a/dbms/src/Functions/FunctionsStringRegex.cpp +++ b/dbms/src/Functions/FunctionsStringRegex.cpp @@ -898,7 +898,7 @@ public: String needle = c1_const->getValue(); String replacement = c2_const->getValue(); - if (needle.size() == 0) + if (needle.empty()) throw Exception("Length of the second argument of function replace must be greater than 0.", ErrorCodes::ARGUMENT_OUT_OF_BOUND); if (const ColumnString * col = checkAndGetColumn(column_src.get())) diff --git a/dbms/src/Functions/array/arrayEnumerateExtended.h b/dbms/src/Functions/array/arrayEnumerateExtended.h index 7f2523feed5..69db59e698e 100644 --- a/dbms/src/Functions/array/arrayEnumerateExtended.h +++ b/dbms/src/Functions/array/arrayEnumerateExtended.h @@ -39,7 +39,7 @@ public: DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { - if (arguments.size() == 0) + if (arguments.empty()) throw Exception("Number of arguments for function " + getName() + " doesn't match: passed " + toString(arguments.size()) + ", should be at least 1.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); diff --git a/dbms/src/Functions/array/arrayEnumerateRanked.h b/dbms/src/Functions/array/arrayEnumerateRanked.h index 3962c927343..133f5d7cb81 100644 --- a/dbms/src/Functions/array/arrayEnumerateRanked.h +++ b/dbms/src/Functions/array/arrayEnumerateRanked.h @@ -98,7 +98,7 @@ public: DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { - if (arguments.size() == 0) + if (arguments.empty()) throw Exception( "Number of arguments for function " + getName() + " doesn't match: passed " + std::to_string(arguments.size()) + ", should be at least 1.", diff --git a/dbms/src/Functions/array/arrayUniq.cpp b/dbms/src/Functions/array/arrayUniq.cpp index 1b66a3da318..d5aedb20883 100644 --- a/dbms/src/Functions/array/arrayUniq.cpp +++ b/dbms/src/Functions/array/arrayUniq.cpp @@ -41,7 +41,7 @@ public: DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { - if (arguments.size() == 0) + if (arguments.empty()) throw Exception("Number of arguments for function " + getName() + " doesn't match: passed " + toString(arguments.size()) + ", should be at least 1.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); diff --git a/dbms/src/Functions/extractTimeZoneFromFunctionArguments.cpp b/dbms/src/Functions/extractTimeZoneFromFunctionArguments.cpp index b49ceeedab3..e90c9646db3 100644 --- a/dbms/src/Functions/extractTimeZoneFromFunctionArguments.cpp +++ b/dbms/src/Functions/extractTimeZoneFromFunctionArguments.cpp @@ -38,7 +38,7 @@ std::string extractTimeZoneNameFromFunctionArguments(const ColumnsWithTypeAndNam } else { - if (!arguments.size()) + if (arguments.empty()) return {}; /// If time zone is attached to an argument of type DateTime. @@ -57,7 +57,7 @@ const DateLUTImpl & extractTimeZoneFromFunctionArguments(Block & block, const Co return DateLUT::instance(extractTimeZoneNameFromColumn(*block.getByPosition(arguments[time_zone_arg_num]).column)); else { - if (!arguments.size()) + if (arguments.empty()) return DateLUT::instance(); /// If time zone is attached to an argument of type DateTime. diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index 83e05604d5e..f4a833879cb 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -2324,7 +2324,7 @@ void Aggregator::destroyWithoutKey(AggregatedDataVariants & result) const void Aggregator::destroyAllAggregateStates(AggregatedDataVariants & result) { - if (result.size() == 0) + if (result.empty()) return; LOG_TRACE(log, "Destroying aggregate states"); diff --git a/dbms/src/Interpreters/PredicateExpressionsOptimizer.cpp b/dbms/src/Interpreters/PredicateExpressionsOptimizer.cpp index 950cc60433a..d2f3ecfa30d 100644 --- a/dbms/src/Interpreters/PredicateExpressionsOptimizer.cpp +++ b/dbms/src/Interpreters/PredicateExpressionsOptimizer.cpp @@ -98,7 +98,7 @@ std::vector PredicateExpressionsOptimizer::extractTablesPredicates(const A { if (expression_info.unique_reference_tables_pos.size() == 1) tables_predicates[*expression_info.unique_reference_tables_pos.begin()].emplace_back(predicate_expression); - else if (expression_info.unique_reference_tables_pos.size() == 0) + else if (expression_info.unique_reference_tables_pos.empty()) { for (size_t index = 0; index < tables_predicates.size(); ++index) tables_predicates[index].emplace_back(predicate_expression); diff --git a/dbms/src/Interpreters/ProcessList.cpp b/dbms/src/Interpreters/ProcessList.cpp index 58e6eaa0f4d..d0d020e2ba2 100644 --- a/dbms/src/Interpreters/ProcessList.cpp +++ b/dbms/src/Interpreters/ProcessList.cpp @@ -281,7 +281,7 @@ ProcessListEntry::~ProcessListEntry() user_process_list.resetTrackers(); /// This removes memory_tracker for all requests. At this time, no other memory_trackers live. - if (parent.processes.size() == 0) + if (parent.processes.empty()) { /// Reset MemoryTracker, similarly (see above). parent.total_memory_tracker.logPeakMemoryUsage(); diff --git a/dbms/src/Interpreters/executeQuery.cpp b/dbms/src/Interpreters/executeQuery.cpp index 2609447de68..ca5146d1bc1 100644 --- a/dbms/src/Interpreters/executeQuery.cpp +++ b/dbms/src/Interpreters/executeQuery.cpp @@ -598,7 +598,7 @@ void executeQuery( const char * end; /// If 'istr' is empty now, fetch next data into buffer. - if (istr.buffer().size() == 0) + if (istr.buffer().empty()) istr.next(); size_t max_query_size = context.getSettingsRef().max_query_size; diff --git a/dbms/src/Processors/Formats/LazyOutputFormat.cpp b/dbms/src/Processors/Formats/LazyOutputFormat.cpp index 129d1a06b9d..e440743b0fb 100644 --- a/dbms/src/Processors/Formats/LazyOutputFormat.cpp +++ b/dbms/src/Processors/Formats/LazyOutputFormat.cpp @@ -11,7 +11,7 @@ Block LazyOutputFormat::getBlock(UInt64 milliseconds) { if (finished_processing) { - if (queue.size() == 0) + if (queue.empty()) return {}; } diff --git a/dbms/src/Storages/Kafka/StorageKafka.cpp b/dbms/src/Storages/Kafka/StorageKafka.cpp index 429c9a17bd2..6aca827208d 100644 --- a/dbms/src/Storages/Kafka/StorageKafka.cpp +++ b/dbms/src/Storages/Kafka/StorageKafka.cpp @@ -292,7 +292,7 @@ bool StorageKafka::checkDependencies(const StorageID & table_id) { // Check if all dependencies are attached auto dependencies = global_context.getDependencies(table_id); - if (dependencies.size() == 0) + if (dependencies.empty()) return true; // Check the dependencies are ready? @@ -324,7 +324,7 @@ void StorageKafka::threadFunc() auto dependencies = global_context.getDependencies(table_id); // Keep streaming as long as there are attached views and streaming is not cancelled - while (!stream_cancelled && num_created_consumers > 0 && dependencies.size() > 0) + while (!stream_cancelled && num_created_consumers > 0 && !dependencies.empty()) { if (!checkDependencies(table_id)) break; @@ -543,7 +543,7 @@ void registerStorageKafka(StorageFactory & factory) { throw Exception("Row delimiter must be a char", ErrorCodes::BAD_ARGUMENTS); } - else if (arg.size() == 0) + else if (arg.empty()) { row_delimiter = '\0'; } diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 3c41f7f4773..12b7f41206d 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -265,7 +265,7 @@ void ReplicatedMergeTreeQueue::removePartFromMutations(const String & part_name) MutationStatus & status = *it->second; status.parts_to_do.removePartAndCoveredParts(part_name); - if (status.parts_to_do.size() == 0) + if (status.parts_to_do.empty()) some_mutations_are_probably_done = true; if (!status.latest_failed_part.empty() && part_info.contains(status.latest_failed_part_info)) @@ -695,7 +695,7 @@ void ReplicatedMergeTreeQueue::updateMutations(zkutil::ZooKeeperPtr zookeeper, C } } - if (mutation.parts_to_do.size() == 0) + if (mutation.parts_to_do.empty()) { some_mutations_are_probably_done = true; } @@ -1377,7 +1377,7 @@ bool ReplicatedMergeTreeQueue::tryFinalizeMutations(zkutil::ZooKeeperPtr zookeep mutation.parts_to_do.clear(); } } - else if (mutation.parts_to_do.size() == 0) + else if (mutation.parts_to_do.empty()) { LOG_TRACE(log, "Will check if mutation " << mutation.entry->znode_name << " is done"); candidates.push_back(mutation.entry); diff --git a/dbms/src/Storages/VirtualColumnUtils.cpp b/dbms/src/Storages/VirtualColumnUtils.cpp index 70042361563..b22b8b02746 100644 --- a/dbms/src/Storages/VirtualColumnUtils.cpp +++ b/dbms/src/Storages/VirtualColumnUtils.cpp @@ -57,7 +57,7 @@ void extractFunctions(const ASTPtr & expression, const NameSet & columns, std::v /// Construct a conjunction from given functions ASTPtr buildWhereExpression(const ASTs & functions) { - if (functions.size() == 0) + if (functions.empty()) return nullptr; if (functions.size() == 1) return functions[0]; From e8b3bac597cffec781a5f239f99f1b848bad6861 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 9 Mar 2020 00:53:03 +0300 Subject: [PATCH 139/712] clang-tidy, part 5 --- .../performance-test/ConfigPreprocessor.cpp | 2 +- dbms/src/Access/AccessControlManager.cpp | 4 +-- dbms/src/Access/AccessRights.h | 32 +++++++++---------- dbms/src/Access/AccessRightsContext.h | 4 +-- dbms/src/Access/DiskAccessStorage.cpp | 2 +- dbms/src/Access/GenericRoleSet.cpp | 10 ++---- dbms/src/Access/MemoryAccessStorage.cpp | 3 -- dbms/src/Access/MemoryAccessStorage.h | 1 - dbms/src/Access/MultipleAccessStorage.cpp | 5 --- dbms/src/Access/MultipleAccessStorage.h | 1 - dbms/src/Access/QuotaContextFactory.cpp | 4 +-- dbms/src/Access/RowPolicyContextFactory.cpp | 2 +- dbms/src/Access/UsersConfigAccessStorage.cpp | 3 -- dbms/src/Access/UsersConfigAccessStorage.h | 1 - .../getDictionaryConfigurationFromAST.cpp | 4 +-- dbms/src/Interpreters/ExpressionActions.cpp | 2 +- dbms/src/Parsers/ASTDictionary.cpp | 4 +-- dbms/src/Parsers/ASTIdentifier.cpp | 4 +-- .../Formats/Impl/AvroRowOutputFormat.cpp | 2 +- .../Storages/MergeTree/MergeTreeIndices.cpp | 9 +++--- 20 files changed, 38 insertions(+), 61 deletions(-) diff --git a/dbms/programs/performance-test/ConfigPreprocessor.cpp b/dbms/programs/performance-test/ConfigPreprocessor.cpp index 8dabac0ec3a..850fd9f14c6 100644 --- a/dbms/programs/performance-test/ConfigPreprocessor.cpp +++ b/dbms/programs/performance-test/ConfigPreprocessor.cpp @@ -21,7 +21,7 @@ std::vector ConfigPreprocessor::processConfig( const auto path = Poco::Path(path_str); test->setString("path", path.absolute().toString()); - if (test->getString("name", "") == "") + if (test->getString("name", "").empty()) test->setString("name", path.getBaseName()); } diff --git a/dbms/src/Access/AccessControlManager.cpp b/dbms/src/Access/AccessControlManager.cpp index 3261e30ec55..b3854e69eec 100644 --- a/dbms/src/Access/AccessControlManager.cpp +++ b/dbms/src/Access/AccessControlManager.cpp @@ -37,9 +37,7 @@ AccessControlManager::AccessControlManager() } -AccessControlManager::~AccessControlManager() -{ -} +AccessControlManager::~AccessControlManager() = default; void AccessControlManager::setLocalDirectory(const String & directory_path) diff --git a/dbms/src/Access/AccessRights.h b/dbms/src/Access/AccessRights.h index 5c5ed382f50..67d205ec6dc 100644 --- a/dbms/src/Access/AccessRights.h +++ b/dbms/src/Access/AccessRights.h @@ -34,8 +34,8 @@ public: void grant(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::string_view & column); void grant(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::vector & columns); void grant(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const Strings & columns); - void grant(const AccessRightsElement & access, std::string_view current_database = {}); - void grant(const AccessRightsElements & access, std::string_view current_database = {}); + void grant(const AccessRightsElement & element, std::string_view current_database = {}); + void grant(const AccessRightsElements & elements, std::string_view current_database = {}); /// Revokes a specified access granted earlier on a specified database/table/column. /// Does nothing if the specified access is not granted. @@ -48,8 +48,8 @@ public: void revoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::string_view & column); void revoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::vector & columns); void revoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const Strings & columns); - void revoke(const AccessRightsElement & access, std::string_view current_database = {}); - void revoke(const AccessRightsElements & access, std::string_view current_database = {}); + void revoke(const AccessRightsElement & element, std::string_view current_database = {}); + void revoke(const AccessRightsElements & elements, std::string_view current_database = {}); /// Revokes a specified access granted earlier on a specified database/table/column or on lower levels. /// The function also restricts access if it's granted on upper level. @@ -61,8 +61,8 @@ public: void partialRevoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::string_view & column); void partialRevoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::vector & columns); void partialRevoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const Strings & columns); - void partialRevoke(const AccessRightsElement & access, std::string_view current_database = {}); - void partialRevoke(const AccessRightsElements & access, std::string_view current_database = {}); + void partialRevoke(const AccessRightsElement & element, std::string_view current_database = {}); + void partialRevoke(const AccessRightsElements & elements, std::string_view current_database = {}); /// Revokes a specified access granted earlier on a specified database/table/column or on lower levels. /// The function also restricts access if it's granted on upper level. @@ -74,8 +74,8 @@ public: void fullRevoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::string_view & column); void fullRevoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::vector & columns); void fullRevoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const Strings & columns); - void fullRevoke(const AccessRightsElement & access, std::string_view current_database = {}); - void fullRevoke(const AccessRightsElements & access, std::string_view current_database = {}); + void fullRevoke(const AccessRightsElement & element, std::string_view current_database = {}); + void fullRevoke(const AccessRightsElements & elements, std::string_view current_database = {}); /// Returns the information about all the access granted. struct Elements @@ -95,8 +95,8 @@ public: bool isGranted(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::string_view & column) const; bool isGranted(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::vector & columns) const; bool isGranted(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const Strings & columns) const; - bool isGranted(const AccessRightsElement & access, std::string_view current_database = {}) const; - bool isGranted(const AccessRightsElements & access, std::string_view current_database = {}) const; + bool isGranted(const AccessRightsElement & element, std::string_view current_database = {}) const; + bool isGranted(const AccessRightsElements & elements, std::string_view current_database = {}) const; friend bool operator ==(const AccessRights & left, const AccessRights & right); friend bool operator !=(const AccessRights & left, const AccessRights & right) { return !(left == right); } @@ -109,23 +109,23 @@ private: template void grantImpl(const AccessFlags & access, const Args &... args); - void grantImpl(const AccessRightsElement & access, std::string_view current_database); - void grantImpl(const AccessRightsElements & access, std::string_view current_database); + void grantImpl(const AccessRightsElement & element, std::string_view current_database); + void grantImpl(const AccessRightsElements & elements, std::string_view current_database); template void revokeImpl(const AccessFlags & access, const Args &... args); template - void revokeImpl(const AccessRightsElement & access, std::string_view current_database); + void revokeImpl(const AccessRightsElement & element, std::string_view current_database); template - void revokeImpl(const AccessRightsElements & access, std::string_view current_database); + void revokeImpl(const AccessRightsElements & elements, std::string_view current_database); template bool isGrantedImpl(const AccessFlags & access, const Args &... args) const; - bool isGrantedImpl(const AccessRightsElement & access, std::string_view current_database) const; - bool isGrantedImpl(const AccessRightsElements & access, std::string_view current_database) const; + bool isGrantedImpl(const AccessRightsElement & element, std::string_view current_database) const; + bool isGrantedImpl(const AccessRightsElements & elements, std::string_view current_database) const; template AccessFlags getAccessImpl(const Args &... args) const; diff --git a/dbms/src/Access/AccessRightsContext.h b/dbms/src/Access/AccessRightsContext.h index be6101e5e9b..8fc5066cfe4 100644 --- a/dbms/src/Access/AccessRightsContext.h +++ b/dbms/src/Access/AccessRightsContext.h @@ -128,10 +128,10 @@ private: bool checkAccessImpl(Poco::Logger * log_, const AccessFlags & access, const Args &... args) const; template - bool checkAccessImpl(Poco::Logger * log_, const AccessRightsElement & access) const; + bool checkAccessImpl(Poco::Logger * log_, const AccessRightsElement & element) const; template - bool checkAccessImpl(Poco::Logger * log_, const AccessRightsElements & access) const; + bool checkAccessImpl(Poco::Logger * log_, const AccessRightsElements & elements) const; boost::shared_ptr calculateResultAccess(bool grant_option) const; boost::shared_ptr calculateResultAccess(bool grant_option, UInt64 readonly_, bool allow_ddl_, bool allow_introspection_) const; diff --git a/dbms/src/Access/DiskAccessStorage.cpp b/dbms/src/Access/DiskAccessStorage.cpp index 1707526b50c..f5f42e1ff80 100644 --- a/dbms/src/Access/DiskAccessStorage.cpp +++ b/dbms/src/Access/DiskAccessStorage.cpp @@ -254,7 +254,7 @@ namespace } - static const std::vector & getAllAccessEntityTypes() + const std::vector & getAllAccessEntityTypes() { static const std::vector res = {typeid(User), typeid(Role), typeid(RowPolicy), typeid(Quota)}; return res; diff --git a/dbms/src/Access/GenericRoleSet.cpp b/dbms/src/Access/GenericRoleSet.cpp index 89029c89331..1e751f995c1 100644 --- a/dbms/src/Access/GenericRoleSet.cpp +++ b/dbms/src/Access/GenericRoleSet.cpp @@ -270,10 +270,7 @@ bool GenericRoleSet::match(const UUID & user_id, const std::vector & enabl bool in_except_list = std::any_of( enabled_roles.begin(), enabled_roles.end(), [this](const UUID & enabled_role) { return except_ids.contains(enabled_role); }); - if (in_except_list) - return false; - - return true; + return !in_except_list; } @@ -292,10 +289,7 @@ bool GenericRoleSet::match(const UUID & user_id, const boost::container::flat_se bool in_except_list = std::any_of( enabled_roles.begin(), enabled_roles.end(), [this](const UUID & enabled_role) { return except_ids.contains(enabled_role); }); - if (in_except_list) - return false; - - return true; + return !in_except_list; } diff --git a/dbms/src/Access/MemoryAccessStorage.cpp b/dbms/src/Access/MemoryAccessStorage.cpp index 51614c587ea..ce668bcd8b3 100644 --- a/dbms/src/Access/MemoryAccessStorage.cpp +++ b/dbms/src/Access/MemoryAccessStorage.cpp @@ -11,9 +11,6 @@ MemoryAccessStorage::MemoryAccessStorage(const String & storage_name_) } -MemoryAccessStorage::~MemoryAccessStorage() {} - - std::optional MemoryAccessStorage::findImpl(std::type_index type, const String & name) const { std::lock_guard lock{mutex}; diff --git a/dbms/src/Access/MemoryAccessStorage.h b/dbms/src/Access/MemoryAccessStorage.h index e060e953b13..b93c2868d34 100644 --- a/dbms/src/Access/MemoryAccessStorage.h +++ b/dbms/src/Access/MemoryAccessStorage.h @@ -14,7 +14,6 @@ class MemoryAccessStorage : public IAccessStorage { public: MemoryAccessStorage(const String & storage_name_ = "memory"); - ~MemoryAccessStorage() override; /// Sets all entities at once. void setAll(const std::vector & all_entities); diff --git a/dbms/src/Access/MultipleAccessStorage.cpp b/dbms/src/Access/MultipleAccessStorage.cpp index 316df11fad9..3474ab9dbd5 100644 --- a/dbms/src/Access/MultipleAccessStorage.cpp +++ b/dbms/src/Access/MultipleAccessStorage.cpp @@ -38,11 +38,6 @@ MultipleAccessStorage::MultipleAccessStorage( } -MultipleAccessStorage::~MultipleAccessStorage() -{ -} - - std::vector MultipleAccessStorage::findMultiple(std::type_index type, const String & name) const { std::vector ids; diff --git a/dbms/src/Access/MultipleAccessStorage.h b/dbms/src/Access/MultipleAccessStorage.h index bdbb6988c8d..898d55d30de 100644 --- a/dbms/src/Access/MultipleAccessStorage.h +++ b/dbms/src/Access/MultipleAccessStorage.h @@ -14,7 +14,6 @@ public: using Storage = IAccessStorage; MultipleAccessStorage(std::vector> nested_storages_); - ~MultipleAccessStorage() override; std::vector findMultiple(std::type_index type, const String & name) const; diff --git a/dbms/src/Access/QuotaContextFactory.cpp b/dbms/src/Access/QuotaContextFactory.cpp index f986ee86c01..7c585bdddee 100644 --- a/dbms/src/Access/QuotaContextFactory.cpp +++ b/dbms/src/Access/QuotaContextFactory.cpp @@ -170,9 +170,7 @@ QuotaContextFactory::QuotaContextFactory(const AccessControlManager & access_con } -QuotaContextFactory::~QuotaContextFactory() -{ -} +QuotaContextFactory::~QuotaContextFactory() = default; QuotaContextPtr QuotaContextFactory::createContext(const String & user_name, const UUID & user_id, const std::vector & enabled_roles, const Poco::Net::IPAddress & address, const String & client_key) diff --git a/dbms/src/Access/RowPolicyContextFactory.cpp b/dbms/src/Access/RowPolicyContextFactory.cpp index 49a23c4d61a..e891f43b5eb 100644 --- a/dbms/src/Access/RowPolicyContextFactory.cpp +++ b/dbms/src/Access/RowPolicyContextFactory.cpp @@ -93,7 +93,7 @@ namespace using ConditionIndex = RowPolicy::ConditionIndex; - static constexpr size_t MAX_CONDITION_INDEX = RowPolicy::MAX_CONDITION_INDEX; + constexpr size_t MAX_CONDITION_INDEX = RowPolicy::MAX_CONDITION_INDEX; /// Accumulates conditions from multiple row policies and joins them using the AND logical operation. diff --git a/dbms/src/Access/UsersConfigAccessStorage.cpp b/dbms/src/Access/UsersConfigAccessStorage.cpp index 72d46468229..20ee2a628a6 100644 --- a/dbms/src/Access/UsersConfigAccessStorage.cpp +++ b/dbms/src/Access/UsersConfigAccessStorage.cpp @@ -339,9 +339,6 @@ UsersConfigAccessStorage::UsersConfigAccessStorage() : IAccessStorage("users.xml } -UsersConfigAccessStorage::~UsersConfigAccessStorage() {} - - void UsersConfigAccessStorage::setConfiguration(const Poco::Util::AbstractConfiguration & config) { std::vector> all_entities; diff --git a/dbms/src/Access/UsersConfigAccessStorage.h b/dbms/src/Access/UsersConfigAccessStorage.h index 672d8ee2adb..773d8caa570 100644 --- a/dbms/src/Access/UsersConfigAccessStorage.h +++ b/dbms/src/Access/UsersConfigAccessStorage.h @@ -19,7 +19,6 @@ class UsersConfigAccessStorage : public IAccessStorage { public: UsersConfigAccessStorage(); - ~UsersConfigAccessStorage() override; void setConfiguration(const Poco::Util::AbstractConfiguration & config); diff --git a/dbms/src/Dictionaries/getDictionaryConfigurationFromAST.cpp b/dbms/src/Dictionaries/getDictionaryConfigurationFromAST.cpp index dbbcc0e41a8..a1d898b2621 100644 --- a/dbms/src/Dictionaries/getDictionaryConfigurationFromAST.cpp +++ b/dbms/src/Dictionaries/getDictionaryConfigurationFromAST.cpp @@ -311,9 +311,9 @@ NamesToTypeNames buildDictionaryAttributesConfiguration( { const auto & children = dictionary_attributes->children; NamesToTypeNames attributes_names_and_types; - for (size_t i = 0; i < children.size(); ++i) + for (const auto & child : children) { - const ASTDictionaryAttributeDeclaration * dict_attr = children[i]->as(); + const ASTDictionaryAttributeDeclaration * dict_attr = child->as(); if (!dict_attr->type) throw Exception("Dictionary attribute must has type", ErrorCodes::INCORRECT_DICTIONARY_DEFINITION); diff --git a/dbms/src/Interpreters/ExpressionActions.cpp b/dbms/src/Interpreters/ExpressionActions.cpp index edfc2a5b2be..dafb270defa 100644 --- a/dbms/src/Interpreters/ExpressionActions.cpp +++ b/dbms/src/Interpreters/ExpressionActions.cpp @@ -64,7 +64,7 @@ ExpressionAction ExpressionAction::applyFunction( const std::vector & argument_names_, std::string result_name_) { - if (result_name_ == "") + if (result_name_.empty()) { result_name_ = function_->getName() + "("; for (size_t i = 0 ; i < argument_names_.size(); ++i) diff --git a/dbms/src/Parsers/ASTDictionary.cpp b/dbms/src/Parsers/ASTDictionary.cpp index 630764fddb9..5c477c2aab7 100644 --- a/dbms/src/Parsers/ASTDictionary.cpp +++ b/dbms/src/Parsers/ASTDictionary.cpp @@ -76,7 +76,7 @@ ASTPtr ASTDictionaryLayout::clone() const void ASTDictionaryLayout::formatImpl(const FormatSettings & settings, FormatState & state, - FormatStateStacked expected) const + FormatStateStacked frame) const { settings.ostr << (settings.hilite ? hilite_keyword : "") << "LAYOUT" @@ -94,7 +94,7 @@ void ASTDictionaryLayout::formatImpl(const FormatSettings & settings, << (settings.hilite ? hilite_none : "") << " "; - parameter->second->formatImpl(settings, state, expected); + parameter->second->formatImpl(settings, state, frame); } settings.ostr << ")"; settings.ostr << ")"; diff --git a/dbms/src/Parsers/ASTIdentifier.cpp b/dbms/src/Parsers/ASTIdentifier.cpp index 6307db675fa..8d8ba936153 100644 --- a/dbms/src/Parsers/ASTIdentifier.cpp +++ b/dbms/src/Parsers/ASTIdentifier.cpp @@ -34,10 +34,10 @@ ASTIdentifier::ASTIdentifier(const String & name_, std::vector && name_p , name_parts(name_parts_) , semantic(std::make_shared()) { - if (name_parts.size() && name_parts[0] == "") + if (!name_parts.empty() && name_parts[0].empty()) name_parts.erase(name_parts.begin()); - if (name == "") + if (name.empty()) { if (name_parts.size() == 2) name = name_parts[0] + '.' + name_parts[1]; diff --git a/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp b/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp index 93e46032a60..65fac6d87f7 100644 --- a/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp +++ b/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp @@ -315,7 +315,7 @@ void AvroSerializer::serializeRow(const Columns & columns, size_t row_num, avro: static avro::Codec getCodec(const std::string & codec_name) { - if (codec_name == "") + if (codec_name.empty()) { #ifdef SNAPPY_CODEC_AVAILABLE return avro::Codec::SNAPPY_CODEC; diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndices.cpp b/dbms/src/Storages/MergeTree/MergeTreeIndices.cpp index 9fe475a00c7..93046f6fe2f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndices.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeIndices.cpp @@ -43,11 +43,12 @@ std::unique_ptr MergeTreeIndexFactory::get( throw Exception( "Unknown Index type '" + node->type->name + "'. Available index types: " + std::accumulate(indexes.cbegin(), indexes.cend(), std::string{}, - [] (auto && lft, const auto & rht) -> std::string { - if (lft == "") - return rht.first; + [] (auto && left, const auto & right) -> std::string + { + if (left.empty()) + return right.first; else - return lft + ", " + rht.first; + return left + ", " + right.first; }), ErrorCodes::INCORRECT_QUERY); From 2a4d5459dbfa1eb8b1b4008f1614475ec0e66ad9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 9 Mar 2020 01:08:39 +0300 Subject: [PATCH 140/712] clang-tidy, part 6 --- .../CachedCompressedReadBuffer.cpp | 2 +- .../CompressionCodecDoubleDelta.cpp | 2 +- .../Compression/CompressionCodecMultiple.h | 2 +- dbms/src/Compression/CompressionCodecT64.h | 4 +- .../src/Compression/LZ4_decompress_faster.cpp | 10 ++-- dbms/src/Core/ExternalTable.cpp | 10 ++-- dbms/src/Core/Field.cpp | 60 +++++++++---------- dbms/src/Core/SettingsCollection.cpp | 2 +- .../AddingDefaultsBlockInputStream.cpp | 6 +- .../AddingDefaultsBlockInputStream.h | 2 +- .../AggregatingBlockInputStream.cpp | 2 +- .../AggregatingSortedBlockInputStream.cpp | 5 +- .../CollapsingFinalBlockInputStream.cpp | 4 +- .../src/DataStreams/FillingBlockInputStream.h | 2 +- dbms/src/Parsers/ExpressionListParsers.cpp | 5 +- dbms/src/Parsers/ParserCase.cpp | 5 +- dbms/src/Parsers/ParserCreateQuery.cpp | 5 +- 17 files changed, 58 insertions(+), 70 deletions(-) diff --git a/dbms/src/Compression/CachedCompressedReadBuffer.cpp b/dbms/src/Compression/CachedCompressedReadBuffer.cpp index f1bbc2188f0..878c980d53f 100644 --- a/dbms/src/Compression/CachedCompressedReadBuffer.cpp +++ b/dbms/src/Compression/CachedCompressedReadBuffer.cpp @@ -61,7 +61,7 @@ bool CachedCompressedReadBuffer::nextImpl() cache->set(key, owned_cell); } - if (owned_cell->data.empty()) + if (owned_cell->data.size() == 0) return false; working_buffer = Buffer(owned_cell->data.data(), owned_cell->data.data() + owned_cell->data.size() - owned_cell->additional_bytes); diff --git a/dbms/src/Compression/CompressionCodecDoubleDelta.cpp b/dbms/src/Compression/CompressionCodecDoubleDelta.cpp index 6bf434c5163..95fa51d1bd0 100644 --- a/dbms/src/Compression/CompressionCodecDoubleDelta.cpp +++ b/dbms/src/Compression/CompressionCodecDoubleDelta.cpp @@ -52,7 +52,7 @@ struct WriteSpec }; // delta size prefix and data lengths based on few high bits peeked from binary stream -static const WriteSpec WRITE_SPEC_LUT[32] = { +const WriteSpec WRITE_SPEC_LUT[32] = { // 0b0 - 1-bit prefix, no data to read /* 00000 */ {1, 0b0, 0}, /* 00001 */ {1, 0b0, 0}, diff --git a/dbms/src/Compression/CompressionCodecMultiple.h b/dbms/src/Compression/CompressionCodecMultiple.h index b8c33bd4f6b..68d2c934fa3 100644 --- a/dbms/src/Compression/CompressionCodecMultiple.h +++ b/dbms/src/Compression/CompressionCodecMultiple.h @@ -22,7 +22,7 @@ public: protected: UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override; - void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override; + void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 decompressed_size) const override; private: Codecs codecs; diff --git a/dbms/src/Compression/CompressionCodecT64.h b/dbms/src/Compression/CompressionCodecT64.h index 8bfdcc766cf..c95e14e860d 100644 --- a/dbms/src/Compression/CompressionCodecT64.h +++ b/dbms/src/Compression/CompressionCodecT64.h @@ -39,8 +39,8 @@ public: void useInfoAboutType(DataTypePtr data_type) override; protected: - UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override; - void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override; + UInt32 doCompressData(const char * src, UInt32 src_size, char * dst) const override; + void doDecompressData(const char * src, UInt32 src_size, char * dst, UInt32 uncompressed_size) const override; UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override { diff --git a/dbms/src/Compression/LZ4_decompress_faster.cpp b/dbms/src/Compression/LZ4_decompress_faster.cpp index cdc5148b199..989b34b97bf 100644 --- a/dbms/src/Compression/LZ4_decompress_faster.cpp +++ b/dbms/src/Compression/LZ4_decompress_faster.cpp @@ -30,7 +30,7 @@ namespace template [[maybe_unused]] void copy(UInt8 * dst, const UInt8 * src); template [[maybe_unused]] void wildCopy(UInt8 * dst, const UInt8 * src, UInt8 * dst_end); -template [[maybe_unused]] void copyOverlap(UInt8 * op, const UInt8 *& match, const size_t offset); +template [[maybe_unused]] void copyOverlap(UInt8 * op, const UInt8 *& match, size_t offset); inline void copy8(UInt8 * dst, const UInt8 * src) @@ -38,7 +38,7 @@ inline void copy8(UInt8 * dst, const UInt8 * src) memcpy(dst, src, 8); } -inline void wildCopy8(UInt8 * dst, const UInt8 * src, UInt8 * dst_end) +inline void wildCopy8(UInt8 * dst, const UInt8 * src, const UInt8 * dst_end) { /// Unrolling with clang is doing >10% performance degrade. #if defined(__clang__) @@ -52,7 +52,7 @@ inline void wildCopy8(UInt8 * dst, const UInt8 * src, UInt8 * dst_end) } while (dst < dst_end); } -inline void copyOverlap8(UInt8 * op, const UInt8 *& match, const size_t offset) +inline void copyOverlap8(UInt8 * op, const UInt8 *& match, size_t offset) { /// 4 % n. /// Or if 4 % n is zero, we use n. @@ -223,7 +223,7 @@ inline void copy16(UInt8 * dst, const UInt8 * src) #endif } -inline void wildCopy16(UInt8 * dst, const UInt8 * src, UInt8 * dst_end) +inline void wildCopy16(UInt8 * dst, const UInt8 * src, const UInt8 * dst_end) { /// Unrolling with clang is doing >10% performance degrade. #if defined(__clang__) @@ -360,7 +360,7 @@ inline void copy32(UInt8 * dst, const UInt8 * src) #endif } -inline void wildCopy32(UInt8 * dst, const UInt8 * src, UInt8 * dst_end) +inline void wildCopy32(UInt8 * dst, const UInt8 * src, const UInt8 * dst_end) { /// Unrolling with clang is doing >10% performance degrade. #if defined(__clang__) diff --git a/dbms/src/Core/ExternalTable.cpp b/dbms/src/Core/ExternalTable.cpp index 3858054bcb2..f991e73ae48 100644 --- a/dbms/src/Core/ExternalTable.cpp +++ b/dbms/src/Core/ExternalTable.cpp @@ -57,8 +57,8 @@ void BaseExternalTable::write() std::cerr << "name " << name << std::endl; std::cerr << "format " << format << std::endl; std::cerr << "structure: \n"; - for (size_t i = 0; i < structure.size(); ++i) - std::cerr << "\t" << structure[i].first << " " << structure[i].second << std::endl; + for (const auto & elem : structure) + std::cerr << "\t" << elem.first << " " << elem.second << std::endl; } std::vector BaseExternalTable::split(const std::string & s, const std::string & d) @@ -91,11 +91,11 @@ void BaseExternalTable::initSampleBlock() { const DataTypeFactory & data_type_factory = DataTypeFactory::instance(); - for (size_t i = 0; i < structure.size(); ++i) + for (const auto & elem : structure) { ColumnWithTypeAndName column; - column.name = structure[i].first; - column.type = data_type_factory.get(structure[i].second); + column.name = elem.first; + column.type = data_type_factory.get(elem.second); column.column = column.type->createColumn(); sample_block.insert(std::move(column)); } diff --git a/dbms/src/Core/Field.cpp b/dbms/src/Core/Field.cpp index 861a75cd28d..fe835c21b8d 100644 --- a/dbms/src/Core/Field.cpp +++ b/dbms/src/Core/Field.cpp @@ -96,50 +96,50 @@ namespace DB DB::writeBinary(type, buf); DB::writeBinary(size, buf); - for (Array::const_iterator it = x.begin(); it != x.end(); ++it) + for (const auto & elem : x) { switch (type) { case Field::Types::Null: break; case Field::Types::UInt64: { - DB::writeVarUInt(get(*it), buf); + DB::writeVarUInt(get(elem), buf); break; } case Field::Types::UInt128: { - DB::writeBinary(get(*it), buf); + DB::writeBinary(get(elem), buf); break; } case Field::Types::Int64: { - DB::writeVarInt(get(*it), buf); + DB::writeVarInt(get(elem), buf); break; } case Field::Types::Float64: { - DB::writeFloatBinary(get(*it), buf); + DB::writeFloatBinary(get(elem), buf); break; } case Field::Types::String: { - DB::writeStringBinary(get(*it), buf); + DB::writeStringBinary(get(elem), buf); break; } case Field::Types::Array: { - DB::writeBinary(get(*it), buf); + DB::writeBinary(get(elem), buf); break; } case Field::Types::Tuple: { - DB::writeBinary(get(*it), buf); + DB::writeBinary(get(elem), buf); break; } case Field::Types::AggregateFunctionState: { - DB::writeStringBinary(it->get().name, buf); - DB::writeStringBinary(it->get().data, buf); + DB::writeStringBinary(elem.get().name, buf); + DB::writeStringBinary(elem.get().data, buf); break; } } @@ -235,9 +235,9 @@ namespace DB const size_t size = x.size(); DB::writeBinary(size, buf); - for (auto it = x.begin(); it != x.end(); ++it) + for (const auto & elem : x) { - const UInt8 type = it->getType(); + const UInt8 type = elem.getType(); DB::writeBinary(type, buf); switch (type) @@ -245,43 +245,43 @@ namespace DB case Field::Types::Null: break; case Field::Types::UInt64: { - DB::writeVarUInt(get(*it), buf); + DB::writeVarUInt(get(elem), buf); break; } case Field::Types::UInt128: { - DB::writeBinary(get(*it), buf); + DB::writeBinary(get(elem), buf); break; } case Field::Types::Int64: { - DB::writeVarInt(get(*it), buf); + DB::writeVarInt(get(elem), buf); break; } case Field::Types::Float64: { - DB::writeFloatBinary(get(*it), buf); + DB::writeFloatBinary(get(elem), buf); break; } case Field::Types::String: { - DB::writeStringBinary(get(*it), buf); + DB::writeStringBinary(get(elem), buf); break; } case Field::Types::Array: { - DB::writeBinary(get(*it), buf); + DB::writeBinary(get(elem), buf); break; } case Field::Types::Tuple: { - DB::writeBinary(get(*it), buf); + DB::writeBinary(get(elem), buf); break; } case Field::Types::AggregateFunctionState: { - DB::writeStringBinary(it->get().name, buf); - DB::writeStringBinary(it->get().data, buf); + DB::writeStringBinary(elem.get().name, buf); + DB::writeStringBinary(elem.get().data, buf); break; } } @@ -321,15 +321,15 @@ namespace DB return Comparator::compare(x, y, x_scale, y_scale); } - template <> bool decimalEqual(Decimal32 x, Decimal32 y, UInt32 xs, UInt32 ys) { return decEqual(x, y, xs, ys); } - template <> bool decimalLess(Decimal32 x, Decimal32 y, UInt32 xs, UInt32 ys) { return decLess(x, y, xs, ys); } - template <> bool decimalLessOrEqual(Decimal32 x, Decimal32 y, UInt32 xs, UInt32 ys) { return decLessOrEqual(x, y, xs, ys); } + template <> bool decimalEqual(Decimal32 x, Decimal32 y, UInt32 x_scale, UInt32 y_scale) { return decEqual(x, y, x_scale, y_scale); } + template <> bool decimalLess(Decimal32 x, Decimal32 y, UInt32 x_scale, UInt32 y_scale) { return decLess(x, y, x_scale, y_scale); } + template <> bool decimalLessOrEqual(Decimal32 x, Decimal32 y, UInt32 x_scale, UInt32 y_scale) { return decLessOrEqual(x, y, x_scale, y_scale); } - template <> bool decimalEqual(Decimal64 x, Decimal64 y, UInt32 xs, UInt32 ys) { return decEqual(x, y, xs, ys); } - template <> bool decimalLess(Decimal64 x, Decimal64 y, UInt32 xs, UInt32 ys) { return decLess(x, y, xs, ys); } - template <> bool decimalLessOrEqual(Decimal64 x, Decimal64 y, UInt32 xs, UInt32 ys) { return decLessOrEqual(x, y, xs, ys); } + template <> bool decimalEqual(Decimal64 x, Decimal64 y, UInt32 x_scale, UInt32 y_scale) { return decEqual(x, y, x_scale, y_scale); } + template <> bool decimalLess(Decimal64 x, Decimal64 y, UInt32 x_scale, UInt32 y_scale) { return decLess(x, y, x_scale, y_scale); } + template <> bool decimalLessOrEqual(Decimal64 x, Decimal64 y, UInt32 x_scale, UInt32 y_scale) { return decLessOrEqual(x, y, x_scale, y_scale); } - template <> bool decimalEqual(Decimal128 x, Decimal128 y, UInt32 xs, UInt32 ys) { return decEqual(x, y, xs, ys); } - template <> bool decimalLess(Decimal128 x, Decimal128 y, UInt32 xs, UInt32 ys) { return decLess(x, y, xs, ys); } - template <> bool decimalLessOrEqual(Decimal128 x, Decimal128 y, UInt32 xs, UInt32 ys) { return decLessOrEqual(x, y, xs, ys); } + template <> bool decimalEqual(Decimal128 x, Decimal128 y, UInt32 x_scale, UInt32 y_scale) { return decEqual(x, y, x_scale, y_scale); } + template <> bool decimalLess(Decimal128 x, Decimal128 y, UInt32 x_scale, UInt32 y_scale) { return decLess(x, y, x_scale, y_scale); } + template <> bool decimalLessOrEqual(Decimal128 x, Decimal128 y, UInt32 x_scale, UInt32 y_scale) { return decLessOrEqual(x, y, x_scale, y_scale); } } diff --git a/dbms/src/Core/SettingsCollection.cpp b/dbms/src/Core/SettingsCollection.cpp index 1053101afbf..f2ac331b028 100644 --- a/dbms/src/Core/SettingsCollection.cpp +++ b/dbms/src/Core/SettingsCollection.cpp @@ -489,7 +489,7 @@ IMPLEMENT_SETTING_ENUM(LoadBalancing, LOAD_BALANCING_LIST_OF_NAMES, ErrorCodes:: M(Unspecified, "") \ M(ALL, "ALL") \ M(ANY, "ANY") -IMPLEMENT_SETTING_ENUM(JoinStrictness, JOIN_STRICTNESS_LIST_OF_NAMES, ErrorCodes::UNKNOWN_JOIN) +IMPLEMENT_SETTING_ENUM(JoinStrictness, JOIN_STRICTNESS_LIST_OF_NAMES, ErrorCodes::UNKNOWN_JOIN) // NOLINT #define JOIN_ALGORITHM_NAMES(M) \ M(AUTO, "auto") \ diff --git a/dbms/src/DataStreams/AddingDefaultsBlockInputStream.cpp b/dbms/src/DataStreams/AddingDefaultsBlockInputStream.cpp index 031419a7095..d685b0225e6 100644 --- a/dbms/src/DataStreams/AddingDefaultsBlockInputStream.cpp +++ b/dbms/src/DataStreams/AddingDefaultsBlockInputStream.cpp @@ -140,7 +140,7 @@ void AddingDefaultsBlockInputStream::checkCalculated(const ColumnWithTypeAndName throw Exception("Mismach column types while adding defaults", ErrorCodes::TYPE_MISMATCH); } -void AddingDefaultsBlockInputStream::mixNumberColumns(TypeIndex type_idx, MutableColumnPtr & column_mixed, const ColumnPtr & column_defs, +void AddingDefaultsBlockInputStream::mixNumberColumns(TypeIndex type_idx, MutableColumnPtr & column_mixed, const ColumnPtr & col_defaults, const BlockMissingValues::RowsBitMask & defaults_mask) const { auto call = [&](const auto & types) -> bool @@ -159,7 +159,7 @@ void AddingDefaultsBlockInputStream::mixNumberColumns(TypeIndex type_idx, Mutabl typename ColVecType::Container & dst = col_read->getData(); - if (auto const_col_defs = checkAndGetColumnConst(column_defs.get())) + if (auto const_col_defs = checkAndGetColumnConst(col_defaults.get())) { FieldType value = checkAndGetColumn(const_col_defs->getDataColumnPtr().get())->getData()[0]; @@ -169,7 +169,7 @@ void AddingDefaultsBlockInputStream::mixNumberColumns(TypeIndex type_idx, Mutabl return true; } - else if (auto col_defs = checkAndGetColumn(column_defs.get())) + else if (auto col_defs = checkAndGetColumn(col_defaults.get())) { auto & src = col_defs->getData(); for (size_t i = 0; i < defaults_mask.size(); ++i) diff --git a/dbms/src/DataStreams/AddingDefaultsBlockInputStream.h b/dbms/src/DataStreams/AddingDefaultsBlockInputStream.h index 436beb9f032..0d6f36861a4 100644 --- a/dbms/src/DataStreams/AddingDefaultsBlockInputStream.h +++ b/dbms/src/DataStreams/AddingDefaultsBlockInputStream.h @@ -31,7 +31,7 @@ private: void checkCalculated(const ColumnWithTypeAndName & col_read, const ColumnWithTypeAndName & col_defaults, size_t needed) const; MutableColumnPtr mixColumns(const ColumnWithTypeAndName & col_read, const ColumnWithTypeAndName & col_defaults, const BlockMissingValues::RowsBitMask & defaults_mask) const; - void mixNumberColumns(TypeIndex type_idx, MutableColumnPtr & col_mixed, const ColumnPtr & col_defaults, + void mixNumberColumns(TypeIndex type_idx, MutableColumnPtr & column_mixed, const ColumnPtr & col_defaults, const BlockMissingValues::RowsBitMask & defaults_mask) const; }; diff --git a/dbms/src/DataStreams/AggregatingBlockInputStream.cpp b/dbms/src/DataStreams/AggregatingBlockInputStream.cpp index f288e202c80..04447474add 100644 --- a/dbms/src/DataStreams/AggregatingBlockInputStream.cpp +++ b/dbms/src/DataStreams/AggregatingBlockInputStream.cpp @@ -48,7 +48,7 @@ Block AggregatingBlockInputStream::readImpl() if (!isCancelled()) { /// Flush data in the RAM to disk also. It's easier than merging on-disk and RAM data. - if (data_variants->size()) + if (data_variants->size()) // NOLINT aggregator.writeToTemporaryFile(*data_variants); } diff --git a/dbms/src/DataStreams/AggregatingSortedBlockInputStream.cpp b/dbms/src/DataStreams/AggregatingSortedBlockInputStream.cpp index 03da7f7c528..c36ba4968d3 100644 --- a/dbms/src/DataStreams/AggregatingSortedBlockInputStream.cpp +++ b/dbms/src/DataStreams/AggregatingSortedBlockInputStream.cpp @@ -180,11 +180,8 @@ void AggregatingSortedBlockInputStream::merge(MutableColumns & merged_columns, S current_key.swap(next_key); /// We will write the data for the group. We copy the values of ordinary columns. - for (size_t i = 0, size = column_numbers_not_to_aggregate.size(); i < size; ++i) - { - size_t j = column_numbers_not_to_aggregate[i]; + for (size_t j : column_numbers_not_to_aggregate) merged_columns[j]->insertFrom(*current->all_columns[j], current->pos); - } /// Add the empty aggregation state to the aggregate columns. The state will be updated in the `addRow` function. for (auto & column_to_aggregate : columns_to_aggregate) diff --git a/dbms/src/DataStreams/CollapsingFinalBlockInputStream.cpp b/dbms/src/DataStreams/CollapsingFinalBlockInputStream.cpp index 599c38bcfeb..c81b72abec4 100644 --- a/dbms/src/DataStreams/CollapsingFinalBlockInputStream.cpp +++ b/dbms/src/DataStreams/CollapsingFinalBlockInputStream.cpp @@ -21,8 +21,8 @@ CollapsingFinalBlockInputStream::~CollapsingFinalBlockInputStream() c.block.cancel(); } - for (size_t i = 0; i < output_blocks.size(); ++i) - delete output_blocks[i]; + for (auto & block : output_blocks) + delete block; } void CollapsingFinalBlockInputStream::reportBadCounts() diff --git a/dbms/src/DataStreams/FillingBlockInputStream.h b/dbms/src/DataStreams/FillingBlockInputStream.h index 3cc4702e374..b84a1d5485f 100644 --- a/dbms/src/DataStreams/FillingBlockInputStream.h +++ b/dbms/src/DataStreams/FillingBlockInputStream.h @@ -13,7 +13,7 @@ namespace DB class FillingBlockInputStream : public IBlockInputStream { public: - FillingBlockInputStream(const BlockInputStreamPtr & input, const SortDescription & fill_description_); + FillingBlockInputStream(const BlockInputStreamPtr & input, const SortDescription & sort_description_); String getName() const override { return "Filling"; } diff --git a/dbms/src/Parsers/ExpressionListParsers.cpp b/dbms/src/Parsers/ExpressionListParsers.cpp index 51e70577fc0..4503b433e6a 100644 --- a/dbms/src/Parsers/ExpressionListParsers.cpp +++ b/dbms/src/Parsers/ExpressionListParsers.cpp @@ -110,10 +110,7 @@ bool ParserList::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) } } - if (!allow_empty && first) - return false; - - return true; + return allow_empty || first; } diff --git a/dbms/src/Parsers/ParserCase.cpp b/dbms/src/Parsers/ParserCase.cpp index ed1afd7c204..521eec212da 100644 --- a/dbms/src/Parsers/ParserCase.cpp +++ b/dbms/src/Parsers/ParserCase.cpp @@ -64,10 +64,7 @@ bool ParserCase::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) } args.push_back(expr_else); - if (!s_end.ignore(pos, expected)) - return false; - - return true; + return s_end.ignore(pos, expected); }; if (has_case_expr) diff --git a/dbms/src/Parsers/ParserCreateQuery.cpp b/dbms/src/Parsers/ParserCreateQuery.cpp index 0a03a5ca342..3ff31b31530 100644 --- a/dbms/src/Parsers/ParserCreateQuery.cpp +++ b/dbms/src/Parsers/ParserCreateQuery.cpp @@ -58,10 +58,7 @@ bool ParserIdentifierWithParameters::parseImpl(Pos & pos, ASTPtr & node, Expecte return true; ParserNestedTable nested; - if (nested.parse(pos, node, expected)) - return true; - - return false; + return nested.parse(pos, node, expected); } bool ParserNameTypePairList::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) From c24333a4370b84daac82ebb38e6f2a86ff5ece09 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 9 Mar 2020 01:38:12 +0300 Subject: [PATCH 141/712] clang-tidy, part 7 --- .../GraphiteRollupSortedBlockInputStream.cpp | 5 +---- .../GraphiteRollupSortedBlockInputStream.h | 2 +- dbms/src/DataStreams/IBlockInputStream.cpp | 12 ++++-------- ...ngAggregatedMemoryEfficientBlockInputStream.cpp | 4 ++-- ...gingAggregatedMemoryEfficientBlockInputStream.h | 2 +- .../ParallelAggregatingBlockInputStream.cpp | 4 ++-- .../ParallelParsingBlockInputStream.cpp | 2 +- .../DataStreams/ParallelParsingBlockInputStream.h | 2 +- .../DataStreams/SummingSortedBlockInputStream.h | 2 +- .../DataTypeCustomSimpleTextSerialization.cpp | 8 ++------ .../DataTypeCustomSimpleTextSerialization.h | 2 -- dbms/src/DataTypes/DataTypeDateTime.cpp | 2 +- dbms/src/DataTypes/DataTypeDateTime64.cpp | 2 +- dbms/src/DataTypes/DataTypeFactory.cpp | 3 --- dbms/src/DataTypes/DataTypeFactory.h | 1 - dbms/src/DataTypes/DataTypeNullable.cpp | 2 +- dbms/src/DataTypes/DataTypeTuple.h | 2 +- dbms/src/DataTypes/FieldToDataType.h | 2 +- dbms/src/DataTypes/IDataType.cpp | 4 +--- dbms/src/Databases/DatabaseDictionary.cpp | 4 ++-- dbms/src/Databases/DatabaseOnDisk.cpp | 8 ++++---- dbms/src/Databases/DatabaseWithDictionaries.cpp | 7 ++++--- dbms/src/Disks/DiskMemory.cpp | 4 ++-- dbms/src/Disks/DiskS3.cpp | 14 +++++--------- dbms/src/Disks/DiskSpaceMonitor.cpp | 14 +++++++------- dbms/src/Parsers/ParserGrantQuery.cpp | 4 +--- dbms/src/Parsers/parseUserName.cpp | 8 ++++---- 27 files changed, 51 insertions(+), 75 deletions(-) diff --git a/dbms/src/DataStreams/GraphiteRollupSortedBlockInputStream.cpp b/dbms/src/DataStreams/GraphiteRollupSortedBlockInputStream.cpp index 64fc0ca2b83..1b81881ab1a 100644 --- a/dbms/src/DataStreams/GraphiteRollupSortedBlockInputStream.cpp +++ b/dbms/src/DataStreams/GraphiteRollupSortedBlockInputStream.cpp @@ -281,11 +281,8 @@ void GraphiteRollupSortedBlockInputStream::startNextGroup(MutableColumns & merge const Graphite::AggregationPattern * aggregation_pattern = std::get<1>(next_rule); /// Copy unmodified column values (including path column). - for (size_t i = 0, size = unmodified_column_numbers.size(); i < size; ++i) - { - size_t j = unmodified_column_numbers[i]; + for (size_t j : unmodified_column_numbers) merged_columns[j]->insertFrom(*cursor->all_columns[j], cursor->pos); - } if (aggregation_pattern) { diff --git a/dbms/src/DataStreams/GraphiteRollupSortedBlockInputStream.h b/dbms/src/DataStreams/GraphiteRollupSortedBlockInputStream.h index 0dfdf7c300c..bfaeff7733b 100644 --- a/dbms/src/DataStreams/GraphiteRollupSortedBlockInputStream.h +++ b/dbms/src/DataStreams/GraphiteRollupSortedBlockInputStream.h @@ -229,7 +229,7 @@ private: /// Insert the values into the resulting columns, which will not be changed in the future. template - void startNextGroup(MutableColumns & merged_columns, TSortCursor & cursor, Graphite::RollupRule next_pattern); + void startNextGroup(MutableColumns & merged_columns, TSortCursor & cursor, Graphite::RollupRule next_rule); /// Insert the calculated `time`, `value`, `version` values into the resulting columns by the last group of rows. void finishCurrentGroup(MutableColumns & merged_columns); diff --git a/dbms/src/DataStreams/IBlockInputStream.cpp b/dbms/src/DataStreams/IBlockInputStream.cpp index 71cf8fc536a..4bccdff6848 100644 --- a/dbms/src/DataStreams/IBlockInputStream.cpp +++ b/dbms/src/DataStreams/IBlockInputStream.cpp @@ -336,9 +336,7 @@ Block IBlockInputStream::getTotals() forEachChild([&] (IBlockInputStream & child) { res = child.getTotals(); - if (res) - return true; - return false; + return bool(res); }); return res; } @@ -353,9 +351,7 @@ Block IBlockInputStream::getExtremes() forEachChild([&] (IBlockInputStream & child) { res = child.getExtremes(); - if (res) - return true; - return false; + return bool(res); }); return res; } @@ -391,9 +387,9 @@ size_t IBlockInputStream::checkDepthImpl(size_t max_depth, size_t level) const throw Exception("Query pipeline is too deep. Maximum: " + toString(max_depth), ErrorCodes::TOO_DEEP_PIPELINE); size_t res = 0; - for (BlockInputStreams::const_iterator it = children.begin(); it != children.end(); ++it) + for (const auto & child : children) { - size_t child_depth = (*it)->checkDepth(level + 1); + size_t child_depth = child->checkDepth(level + 1); if (child_depth > res) res = child_depth; } diff --git a/dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.cpp b/dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.cpp index 65e742929da..b241f9fd00b 100644 --- a/dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.cpp +++ b/dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.cpp @@ -113,8 +113,8 @@ void MergingAggregatedMemoryEfficientBlockInputStream::readSuffix() finalize(); - for (size_t i = 0; i < children.size(); ++i) - children[i]->readSuffix(); + for (auto & child : children) + child->readSuffix(); } diff --git a/dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.h b/dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.h index 9fe322c3f43..4a6eccfd2a4 100644 --- a/dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.h +++ b/dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.h @@ -150,7 +150,7 @@ private: std::unique_ptr parallel_merge_data; - void mergeThread(ThreadGroupStatusPtr main_thread); + void mergeThread(ThreadGroupStatusPtr thread_group); void finalize(); }; diff --git a/dbms/src/DataStreams/ParallelAggregatingBlockInputStream.cpp b/dbms/src/DataStreams/ParallelAggregatingBlockInputStream.cpp index 3ce3254bed0..951ba7d5f2e 100644 --- a/dbms/src/DataStreams/ParallelAggregatingBlockInputStream.cpp +++ b/dbms/src/DataStreams/ParallelAggregatingBlockInputStream.cpp @@ -120,7 +120,7 @@ void ParallelAggregatingBlockInputStream::Handler::onFinishThread(size_t thread_ if (data.isConvertibleToTwoLevel()) data.convertToTwoLevel(); - if (data.size()) + if (!data.empty()) parent.aggregator.writeToTemporaryFile(data); } } @@ -136,7 +136,7 @@ void ParallelAggregatingBlockInputStream::Handler::onFinish() if (data->isConvertibleToTwoLevel()) data->convertToTwoLevel(); - if (data->size()) + if (!data->empty()) parent.aggregator.writeToTemporaryFile(*data); } } diff --git a/dbms/src/DataStreams/ParallelParsingBlockInputStream.cpp b/dbms/src/DataStreams/ParallelParsingBlockInputStream.cpp index e25e31e33b8..536b7bd77f4 100644 --- a/dbms/src/DataStreams/ParallelParsingBlockInputStream.cpp +++ b/dbms/src/DataStreams/ParallelParsingBlockInputStream.cpp @@ -83,7 +83,7 @@ void ParallelParsingBlockInputStream::parserThreadFunction(size_t current_ticket // We suppose we will get at least some blocks for a non-empty buffer, // except at the end of file. Also see a matching assert in readImpl(). - assert(unit.is_last || unit.block_ext.block.size() > 0); + assert(unit.is_last || !unit.block_ext.block.empty()); std::unique_lock lock(mutex); unit.status = READY_TO_READ; diff --git a/dbms/src/DataStreams/ParallelParsingBlockInputStream.h b/dbms/src/DataStreams/ParallelParsingBlockInputStream.h index 1b2bfbd52e2..03f0d508227 100644 --- a/dbms/src/DataStreams/ParallelParsingBlockInputStream.h +++ b/dbms/src/DataStreams/ParallelParsingBlockInputStream.h @@ -242,7 +242,7 @@ private: } void segmentatorThreadFunction(); - void parserThreadFunction(size_t bucket_num); + void parserThreadFunction(size_t current_ticket_number); // Save/log a background exception, set termination flag, wake up all // threads. This function is used by segmentator and parsed threads. diff --git a/dbms/src/DataStreams/SummingSortedBlockInputStream.h b/dbms/src/DataStreams/SummingSortedBlockInputStream.h index 005ef5cb751..bdd68c991cc 100644 --- a/dbms/src/DataStreams/SummingSortedBlockInputStream.h +++ b/dbms/src/DataStreams/SummingSortedBlockInputStream.h @@ -147,7 +147,7 @@ private: void insertCurrentRowIfNeeded(MutableColumns & merged_columns); /// Returns true if merge result is not empty - bool mergeMap(const MapDescription & map, Row & row, SortCursor & cursor); + bool mergeMap(const MapDescription & desc, Row & row, SortCursor & cursor); // Add the row under the cursor to the `row`. void addRow(SortCursor & cursor); diff --git a/dbms/src/DataTypes/DataTypeCustomSimpleTextSerialization.cpp b/dbms/src/DataTypes/DataTypeCustomSimpleTextSerialization.cpp index 18b8798fe77..75d0194c524 100644 --- a/dbms/src/DataTypes/DataTypeCustomSimpleTextSerialization.cpp +++ b/dbms/src/DataTypes/DataTypeCustomSimpleTextSerialization.cpp @@ -9,7 +9,7 @@ namespace { using namespace DB; -static String serializeToString(const DataTypeCustomSimpleTextSerialization & domain, const IColumn & column, size_t row_num, const FormatSettings & settings) +String serializeToString(const DataTypeCustomSimpleTextSerialization & domain, const IColumn & column, size_t row_num, const FormatSettings & settings) { WriteBufferFromOwnString buffer; domain.serializeText(column, row_num, buffer, settings); @@ -17,7 +17,7 @@ static String serializeToString(const DataTypeCustomSimpleTextSerialization & do return buffer.str(); } -static void deserializeFromString(const DataTypeCustomSimpleTextSerialization & domain, IColumn & column, const String & s, const FormatSettings & settings) +void deserializeFromString(const DataTypeCustomSimpleTextSerialization & domain, IColumn & column, const String & s, const FormatSettings & settings) { ReadBufferFromString istr(s); domain.deserializeText(column, istr, settings); @@ -28,10 +28,6 @@ static void deserializeFromString(const DataTypeCustomSimpleTextSerialization & namespace DB { -DataTypeCustomSimpleTextSerialization::~DataTypeCustomSimpleTextSerialization() -{ -} - void DataTypeCustomSimpleTextSerialization::deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { String str; diff --git a/dbms/src/DataTypes/DataTypeCustomSimpleTextSerialization.h b/dbms/src/DataTypes/DataTypeCustomSimpleTextSerialization.h index e1c08d28738..d983b66eecc 100644 --- a/dbms/src/DataTypes/DataTypeCustomSimpleTextSerialization.h +++ b/dbms/src/DataTypes/DataTypeCustomSimpleTextSerialization.h @@ -15,8 +15,6 @@ class IColumn; class DataTypeCustomSimpleTextSerialization : public IDataTypeCustomTextSerialization { public: - virtual ~DataTypeCustomSimpleTextSerialization() override; - // Methods that subclasses must override in order to get full serialization/deserialization support. virtual void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override = 0; virtual void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const = 0; diff --git a/dbms/src/DataTypes/DataTypeDateTime.cpp b/dbms/src/DataTypes/DataTypeDateTime.cpp index 5589a372732..1af8d5b3ee4 100644 --- a/dbms/src/DataTypes/DataTypeDateTime.cpp +++ b/dbms/src/DataTypes/DataTypeDateTime.cpp @@ -20,7 +20,7 @@ namespace { using namespace DB; -static inline void readText(time_t & x, ReadBuffer & istr, const FormatSettings & settings, const DateLUTImpl & time_zone, const DateLUTImpl & utc_time_zone) +inline void readText(time_t & x, ReadBuffer & istr, const FormatSettings & settings, const DateLUTImpl & time_zone, const DateLUTImpl & utc_time_zone) { switch (settings.date_time_input_format) { diff --git a/dbms/src/DataTypes/DataTypeDateTime64.cpp b/dbms/src/DataTypes/DataTypeDateTime64.cpp index 40909c1000b..228cfbd17c8 100644 --- a/dbms/src/DataTypes/DataTypeDateTime64.cpp +++ b/dbms/src/DataTypes/DataTypeDateTime64.cpp @@ -235,7 +235,7 @@ getArgument(const ASTPtr & arguments, size_t argument_index, const char * argume static DataTypePtr create64(const String & /*type_name*/, const ASTPtr & arguments) { - if (!arguments || arguments->empty()) + if (!arguments || arguments->size() == 0) return std::make_shared(DataTypeDateTime64::default_scale); const auto scale = getArgument(arguments, 0, "scale", "DateType64"); diff --git a/dbms/src/DataTypes/DataTypeFactory.cpp b/dbms/src/DataTypes/DataTypeFactory.cpp index c073f00a0b7..3852a3200f4 100644 --- a/dbms/src/DataTypes/DataTypeFactory.cpp +++ b/dbms/src/DataTypes/DataTypeFactory.cpp @@ -180,9 +180,6 @@ DataTypeFactory::DataTypeFactory() registerDataTypeDomainSimpleAggregateFunction(*this); } -DataTypeFactory::~DataTypeFactory() -{} - DataTypeFactory & DataTypeFactory::instance() { static DataTypeFactory ret; diff --git a/dbms/src/DataTypes/DataTypeFactory.h b/dbms/src/DataTypes/DataTypeFactory.h index 94f5fc4b4a4..662c4d291d0 100644 --- a/dbms/src/DataTypes/DataTypeFactory.h +++ b/dbms/src/DataTypes/DataTypeFactory.h @@ -55,7 +55,6 @@ private: DataTypesDictionary case_insensitive_data_types; DataTypeFactory(); - ~DataTypeFactory() override; const DataTypesDictionary & getCreatorMap() const override { return data_types; } diff --git a/dbms/src/DataTypes/DataTypeNullable.cpp b/dbms/src/DataTypes/DataTypeNullable.cpp index d219e27ed8f..45cbfebadce 100644 --- a/dbms/src/DataTypes/DataTypeNullable.cpp +++ b/dbms/src/DataTypes/DataTypeNullable.cpp @@ -207,7 +207,7 @@ static ReturnType safeDeserialize( void DataTypeNullable::deserializeBinary(IColumn & column, ReadBuffer & istr) const { safeDeserialize(column, *nested_data_type, - [&istr] { bool is_null = 0; readBinary(is_null, istr); return is_null; }, + [&istr] { bool is_null = false; readBinary(is_null, istr); return is_null; }, [this, &istr] (IColumn & nested) { nested_data_type->deserializeBinary(nested, istr); }); } diff --git a/dbms/src/DataTypes/DataTypeTuple.h b/dbms/src/DataTypes/DataTypeTuple.h index 06f0f62026e..a8d16c28fa5 100644 --- a/dbms/src/DataTypes/DataTypeTuple.h +++ b/dbms/src/DataTypes/DataTypeTuple.h @@ -78,7 +78,7 @@ public: DeserializeBinaryBulkStatePtr & state) const override; void serializeProtobuf(const IColumn & column, size_t row_num, ProtobufWriter & protobuf, size_t & value_index) const override; - void deserializeProtobuf(IColumn & column, ProtobufReader & reader, bool allow_add_row, bool & row_added) const override; + void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override; MutableColumnPtr createColumn() const override; diff --git a/dbms/src/DataTypes/FieldToDataType.h b/dbms/src/DataTypes/FieldToDataType.h index 1edcdf3c11d..767bf92f383 100644 --- a/dbms/src/DataTypes/FieldToDataType.h +++ b/dbms/src/DataTypes/FieldToDataType.h @@ -24,7 +24,7 @@ public: DataTypePtr operator() (const Float64 & x) const; DataTypePtr operator() (const String & x) const; DataTypePtr operator() (const Array & x) const; - DataTypePtr operator() (const Tuple & x) const; + DataTypePtr operator() (const Tuple & tuple) const; DataTypePtr operator() (const DecimalField & x) const; DataTypePtr operator() (const DecimalField & x) const; DataTypePtr operator() (const DecimalField & x) const; diff --git a/dbms/src/DataTypes/IDataType.cpp b/dbms/src/DataTypes/IDataType.cpp index 39d269d8613..c31b80b7973 100644 --- a/dbms/src/DataTypes/IDataType.cpp +++ b/dbms/src/DataTypes/IDataType.cpp @@ -27,9 +27,7 @@ IDataType::IDataType() : custom_name(nullptr), custom_text_serialization(nullptr { } -IDataType::~IDataType() -{ -} +IDataType::~IDataType() = default; String IDataType::getName() const { diff --git a/dbms/src/Databases/DatabaseDictionary.cpp b/dbms/src/Databases/DatabaseDictionary.cpp index 9bf36005af1..006eb1656a2 100644 --- a/dbms/src/Databases/DatabaseDictionary.cpp +++ b/dbms/src/Databases/DatabaseDictionary.cpp @@ -75,9 +75,9 @@ StoragePtr DatabaseDictionary::tryGetTable( return {}; } -DatabaseTablesIteratorPtr DatabaseDictionary::getTablesIterator(const Context & context, const FilterByNameFunction & filter_by_name) +DatabaseTablesIteratorPtr DatabaseDictionary::getTablesIterator(const Context & context, const FilterByNameFunction & filter_by_table_name) { - return std::make_unique(listTables(context, filter_by_name)); + return std::make_unique(listTables(context, filter_by_table_name)); } bool DatabaseDictionary::empty(const Context & context) const diff --git a/dbms/src/Databases/DatabaseOnDisk.cpp b/dbms/src/Databases/DatabaseOnDisk.cpp index 4ca866b006f..902933c58e1 100644 --- a/dbms/src/Databases/DatabaseOnDisk.cpp +++ b/dbms/src/Databases/DatabaseOnDisk.cpp @@ -290,14 +290,14 @@ void DatabaseOnDisk::drop(const Context & context) Poco::File(getMetadataPath()).remove(false); } -String DatabaseOnDisk::getObjectMetadataPath(const String & table_name) const +String DatabaseOnDisk::getObjectMetadataPath(const String & object_name) const { - return getMetadataPath() + escapeForFileName(table_name) + ".sql"; + return getMetadataPath() + escapeForFileName(object_name) + ".sql"; } -time_t DatabaseOnDisk::getObjectMetadataModificationTime(const String & table_name) const +time_t DatabaseOnDisk::getObjectMetadataModificationTime(const String & object_name) const { - String table_metadata_path = getObjectMetadataPath(table_name); + String table_metadata_path = getObjectMetadataPath(object_name); Poco::File meta_file(table_metadata_path); if (meta_file.exists()) diff --git a/dbms/src/Databases/DatabaseWithDictionaries.cpp b/dbms/src/Databases/DatabaseWithDictionaries.cpp index cd72b703ecc..d4d95184490 100644 --- a/dbms/src/Databases/DatabaseWithDictionaries.cpp +++ b/dbms/src/Databases/DatabaseWithDictionaries.cpp @@ -168,11 +168,12 @@ StoragePtr DatabaseWithDictionaries::tryGetTable(const Context & context, const return {}; } -DatabaseTablesIteratorPtr DatabaseWithDictionaries::getTablesWithDictionaryTablesIterator(const Context & context, const FilterByNameFunction & filter_by_name) +DatabaseTablesIteratorPtr DatabaseWithDictionaries::getTablesWithDictionaryTablesIterator( + const Context & context, const FilterByNameFunction & filter_by_dictionary_name) { /// NOTE: it's not atomic - auto tables_it = getTablesIterator(context, filter_by_name); - auto dictionaries_it = getDictionariesIterator(context, filter_by_name); + auto tables_it = getTablesIterator(context, filter_by_dictionary_name); + auto dictionaries_it = getDictionariesIterator(context, filter_by_dictionary_name); Tables result; while (tables_it && tables_it->isValid()) diff --git a/dbms/src/Disks/DiskMemory.cpp b/dbms/src/Disks/DiskMemory.cpp index e8634147959..9d85b43a979 100644 --- a/dbms/src/Disks/DiskMemory.cpp +++ b/dbms/src/Disks/DiskMemory.cpp @@ -44,7 +44,7 @@ class ReadIndirectBuffer : public ReadBufferFromFileBase { public: ReadIndirectBuffer(String path_, const String & data_) - : ReadBufferFromFileBase(), impl(ReadBufferFromString(data_)), path(std::move(path_)) + : impl(ReadBufferFromString(data_)), path(std::move(path_)) { internal_buffer = impl.buffer(); working_buffer = internal_buffer; @@ -73,7 +73,7 @@ class WriteIndirectBuffer : public WriteBufferFromFileBase { public: WriteIndirectBuffer(DiskMemory * disk_, String path_, WriteMode mode_, size_t buf_size) - : WriteBufferFromFileBase(buf_size, nullptr, 0), impl(), disk(disk_), path(std::move(path_)), mode(mode_) + : WriteBufferFromFileBase(buf_size, nullptr, 0), disk(disk_), path(std::move(path_)), mode(mode_) { } diff --git a/dbms/src/Disks/DiskS3.cpp b/dbms/src/Disks/DiskS3.cpp index e91dbf55f8d..2f0c9720934 100644 --- a/dbms/src/Disks/DiskS3.cpp +++ b/dbms/src/Disks/DiskS3.cpp @@ -144,14 +144,10 @@ namespace public: ReadIndirectBufferFromS3( std::shared_ptr client_ptr_, const String & bucket_, Metadata metadata_, size_t buf_size_) - : ReadBufferFromFileBase() - , client_ptr(std::move(client_ptr_)) - , bucket(bucket_) - , metadata(std::move(metadata_)) - , buf_size(buf_size_) - , absolute_position(0) - , current_buf_idx(0) - , current_buf(nullptr) + : client_ptr(std::move(client_ptr_)), + bucket(bucket_), + metadata(std::move(metadata_)), + buf_size(buf_size_) { } @@ -235,7 +231,7 @@ namespace size_t buf_size; size_t absolute_position = 0; - UInt32 current_buf_idx; + UInt32 current_buf_idx = 0; std::unique_ptr current_buf; }; diff --git a/dbms/src/Disks/DiskSpaceMonitor.cpp b/dbms/src/Disks/DiskSpaceMonitor.cpp index 0318891fe4e..c602baccece 100644 --- a/dbms/src/Disks/DiskSpaceMonitor.cpp +++ b/dbms/src/Disks/DiskSpaceMonitor.cpp @@ -191,11 +191,11 @@ DiskPtr Volume::getNextDisk() return disks[index]; } -ReservationPtr Volume::reserve(UInt64 expected_size) +ReservationPtr Volume::reserve(UInt64 bytes) { /// This volume can not store files which size greater than max_data_part_size - if (max_data_part_size != 0 && expected_size > max_data_part_size) + if (max_data_part_size != 0 && bytes > max_data_part_size) return {}; size_t start_from = last_used.fetch_add(1u, std::memory_order_relaxed); @@ -204,7 +204,7 @@ ReservationPtr Volume::reserve(UInt64 expected_size) { size_t index = (start_from + i) % disks_num; - auto reservation = disks[index]->reserve(expected_size); + auto reservation = disks[index]->reserve(bytes); if (reservation) return reservation; @@ -354,12 +354,12 @@ UInt64 StoragePolicy::getMaxUnreservedFreeSpace() const } -ReservationPtr StoragePolicy::reserve(UInt64 expected_size, size_t min_volume_index) const +ReservationPtr StoragePolicy::reserve(UInt64 bytes, size_t min_volume_index) const { for (size_t i = min_volume_index; i < volumes.size(); ++i) { const auto & volume = volumes[i]; - auto reservation = volume->reserve(expected_size); + auto reservation = volume->reserve(bytes); if (reservation) return reservation; } @@ -367,9 +367,9 @@ ReservationPtr StoragePolicy::reserve(UInt64 expected_size, size_t min_volume_in } -ReservationPtr StoragePolicy::reserve(UInt64 expected_size) const +ReservationPtr StoragePolicy::reserve(UInt64 bytes) const { - return reserve(expected_size, 0); + return reserve(bytes, 0); } diff --git a/dbms/src/Parsers/ParserGrantQuery.cpp b/dbms/src/Parsers/ParserGrantQuery.cpp index ae37369474d..dc2fbc5f260 100644 --- a/dbms/src/Parsers/ParserGrantQuery.cpp +++ b/dbms/src/Parsers/ParserGrantQuery.cpp @@ -33,9 +33,7 @@ namespace if (pos_->type != TokenType::BareWord) return false; std::string_view word{pos_->begin, pos_->size()}; - if (boost::iequals(word, "ON") || boost::iequals(word, "TO") || boost::iequals(word, "FROM")) - return false; - return true; + return !(boost::iequals(word, "ON") || boost::iequals(word, "TO") || boost::iequals(word, "FROM")); }; expected.add(pos, "access type"); diff --git a/dbms/src/Parsers/parseUserName.cpp b/dbms/src/Parsers/parseUserName.cpp index ed7bb45b6fb..3993935e386 100644 --- a/dbms/src/Parsers/parseUserName.cpp +++ b/dbms/src/Parsers/parseUserName.cpp @@ -36,15 +36,15 @@ namespace } -bool parseUserName(IParser::Pos & pos, Expected & expected, String & result) +bool parseUserName(IParser::Pos & pos, Expected & expected, String & user_name) { - return parseUserNameImpl(pos, expected, result, nullptr); + return parseUserNameImpl(pos, expected, user_name, nullptr); } -bool parseUserName(IParser::Pos & pos, Expected & expected, String & result, String & host_like_pattern) +bool parseUserName(IParser::Pos & pos, Expected & expected, String & user_name, String & host_like_pattern) { - return parseUserNameImpl(pos, expected, result, &host_like_pattern); + return parseUserNameImpl(pos, expected, user_name, &host_like_pattern); } From 889ae0305b0ccdf879f70e3c957075030a3b5ff8 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 9 Mar 2020 02:48:08 +0300 Subject: [PATCH 142/712] clang-tidy, part 8 --- dbms/src/Interpreters/ActionsVisitor.cpp | 16 ++--- dbms/src/Interpreters/Aggregator.cpp | 32 +++++----- dbms/src/Interpreters/AnalyzedJoin.h | 2 +- dbms/src/Interpreters/BloomFilter.cpp | 3 - dbms/src/Interpreters/BloomFilter.h | 1 - dbms/src/Interpreters/CatBoostModel.cpp | 2 +- dbms/src/Interpreters/Cluster.cpp | 4 +- dbms/src/Interpreters/Context.cpp | 30 ++------- dbms/src/Interpreters/Context.h | 10 ++- .../Interpreters/CrossToInnerJoinVisitor.cpp | 9 +-- dbms/src/Interpreters/DDLWorker.h | 4 +- .../ExecuteScalarSubqueriesVisitor.cpp | 2 +- dbms/src/Interpreters/ExpressionActions.cpp | 61 +++++++++---------- dbms/src/Interpreters/ExpressionAnalyzer.cpp | 39 +++++------- dbms/src/Interpreters/ExpressionAnalyzer.h | 2 +- dbms/src/Interpreters/ExpressionJIT.cpp | 12 ++-- 16 files changed, 95 insertions(+), 134 deletions(-) diff --git a/dbms/src/Interpreters/ActionsVisitor.cpp b/dbms/src/Interpreters/ActionsVisitor.cpp index 3a20ae6ce24..79c660689d7 100644 --- a/dbms/src/Interpreters/ActionsVisitor.cpp +++ b/dbms/src/Interpreters/ActionsVisitor.cpp @@ -222,11 +222,11 @@ void ScopeStack::pushLevel(const NamesAndTypesList & input_columns) ColumnsWithTypeAndName all_columns; NameSet new_names; - for (NamesAndTypesList::const_iterator it = input_columns.begin(); it != input_columns.end(); ++it) + for (const auto & input_column : input_columns) { - all_columns.emplace_back(nullptr, it->type, it->name); - new_names.insert(it->name); - stack.back().new_columns.insert(it->name); + all_columns.emplace_back(nullptr, input_column.type, input_column.name); + new_names.insert(input_column.name); + stack.back().new_columns.insert(input_column.name); } const Block & prev_sample_block = prev.actions->getSampleBlock(); @@ -253,17 +253,17 @@ void ScopeStack::addAction(const ExpressionAction & action) { size_t level = 0; Names required = action.getNeededColumns(); - for (size_t i = 0; i < required.size(); ++i) - level = std::max(level, getColumnLevel(required[i])); + for (const auto & elem : required) + level = std::max(level, getColumnLevel(elem)); Names added; stack[level].actions->add(action, added); stack[level].new_columns.insert(added.begin(), added.end()); - for (size_t i = 0; i < added.size(); ++i) + for (const auto & elem : added) { - const ColumnWithTypeAndName & col = stack[level].actions->getSampleBlock().getByName(added[i]); + const ColumnWithTypeAndName & col = stack[level].actions->getSampleBlock().getByName(elem); for (size_t j = level + 1; j < stack.size(); ++j) stack[j].actions->addInput(col); } diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index f4a833879cb..30d2afdb2d8 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -649,7 +649,7 @@ bool Aggregator::executeOnBlock(Columns columns, UInt64 num_rows, AggregatedData executeImpl(*result.NAME, result.aggregates_pool, num_rows, key_columns, aggregate_functions_instructions.data(), \ no_more_keys, overflow_row_ptr); - if (false) {} + if (false) {} // NOLINT APPLY_FOR_AGGREGATED_VARIANTS(M) #undef M } @@ -717,7 +717,7 @@ void Aggregator::writeToTemporaryFile(AggregatedDataVariants & data_variants, co else if (data_variants.type == AggregatedDataVariants::Type::NAME) \ writeToTemporaryFileImpl(data_variants, *data_variants.NAME, block_out); - if (false) {} + if (false) {} // NOLINT APPLY_FOR_VARIANTS_TWO_LEVEL(M) #undef M else @@ -798,7 +798,7 @@ Block Aggregator::mergeAndConvertOneBucketToBlock( auto method = merged_data.type; Block block; - if (false) {} + if (false) {} // NOLINT #define M(NAME) \ else if (method == AggregatedDataVariants::Type::NAME) \ { \ @@ -1048,8 +1048,8 @@ Block Aggregator::prepareBlockAndFill( /// The ColumnAggregateFunction column captures the shared ownership of the arena with the aggregate function states. ColumnAggregateFunction & column_aggregate_func = assert_cast(*aggregate_columns[i]); - for (size_t j = 0; j < data_variants.aggregates_pools.size(); ++j) - column_aggregate_func.addArena(data_variants.aggregates_pools[j]); + for (auto & pool : data_variants.aggregates_pools) + column_aggregate_func.addArena(pool); aggregate_columns_data[i] = &column_aggregate_func.getData(); aggregate_columns_data[i]->reserve(rows); @@ -1064,8 +1064,8 @@ Block Aggregator::prepareBlockAndFill( /// The ColumnAggregateFunction column captures the shared ownership of the arena with aggregate function states. ColumnAggregateFunction & column_aggregate_func = assert_cast(*final_aggregate_columns[i]); - for (size_t j = 0; j < data_variants.aggregates_pools.size(); ++j) - column_aggregate_func.addArena(data_variants.aggregates_pools[j]); + for (auto & pool : data_variants.aggregates_pools) + column_aggregate_func.addArena(pool); } } } @@ -1152,7 +1152,7 @@ Block Aggregator::prepareBlockAndFillSingleLevel(AggregatedDataVariants & data_v convertToBlockImpl(*data_variants.NAME, data_variants.NAME->data, \ key_columns, aggregate_columns, final_aggregate_columns, final_); - if (false) {} + if (false) {} // NOLINT APPLY_FOR_VARIANTS_SINGLE_LEVEL(M) #undef M else @@ -1169,7 +1169,7 @@ BlocksList Aggregator::prepareBlocksAndFillTwoLevel(AggregatedDataVariants & dat else if (data_variants.type == AggregatedDataVariants::Type::NAME) \ return prepareBlocksAndFillTwoLevelImpl(data_variants, *data_variants.NAME, final, thread_pool); - if (false) {} + if (false) {} // NOLINT APPLY_FOR_VARIANTS_TWO_LEVEL(M) #undef M else @@ -1587,7 +1587,7 @@ protected: #define M(NAME) \ else if (first->type == AggregatedDataVariants::Type::NAME) \ aggregator.mergeSingleLevelDataImplNAME)::element_type>(data); - if (false) {} + if (false) {} // NOLINT APPLY_FOR_VARIANTS_SINGLE_LEVEL(M) #undef M else @@ -1687,7 +1687,7 @@ private: size_t thread_number = static_cast(bucket_num) % threads; Arena * arena = merged_data.aggregates_pools.at(thread_number).get(); - if (false) {} + if (false) {} // NOLINT #define M(NAME) \ else if (method == AggregatedDataVariants::Type::NAME) \ { \ @@ -1993,7 +1993,7 @@ void Aggregator::mergeBlocks(BucketToBlocks bucket_to_blocks, AggregatedDataVari else if (result.type == AggregatedDataVariants::Type::NAME) \ mergeStreamsImpl(block, aggregates_pool, *result.NAME, result.NAME->data.impls[bucket], nullptr, false); - if (false) {} + if (false) {} // NOLINT APPLY_FOR_VARIANTS_TWO_LEVEL(M) #undef M else @@ -2249,7 +2249,7 @@ std::vector Aggregator::convertBlockToTwoLevel(const Block & block) else if (type == AggregatedDataVariants::Type::NAME) \ type = AggregatedDataVariants::Type::NAME ## _two_level; - if (false) {} + if (false) {} // NOLINT APPLY_FOR_VARIANTS_CONVERTIBLE_TO_TWO_LEVEL(M) #undef M else @@ -2263,7 +2263,7 @@ std::vector Aggregator::convertBlockToTwoLevel(const Block & block) else if (data.type == AggregatedDataVariants::Type::NAME) \ num_buckets = data.NAME->data.NUM_BUCKETS; - if (false) {} + if (false) {} // NOLINT APPLY_FOR_VARIANTS_TWO_LEVEL(M) #undef M else @@ -2276,7 +2276,7 @@ std::vector Aggregator::convertBlockToTwoLevel(const Block & block) convertBlockToTwoLevelImpl(*data.NAME, data.aggregates_pool, \ key_columns, block, splitted_blocks); - if (false) {} + if (false) {} // NOLINT APPLY_FOR_VARIANTS_TWO_LEVEL(M) #undef M else @@ -2337,7 +2337,7 @@ void Aggregator::destroyAllAggregateStates(AggregatedDataVariants & result) else if (result.type == AggregatedDataVariants::Type::NAME) \ destroyImpl(result.NAME->data); - if (false) {} + if (false) {} // NOLINT APPLY_FOR_AGGREGATED_VARIANTS(M) #undef M else if (result.type != AggregatedDataVariants::Type::without_key) diff --git a/dbms/src/Interpreters/AnalyzedJoin.h b/dbms/src/Interpreters/AnalyzedJoin.h index 6c5c65cde54..f1341a16a4c 100644 --- a/dbms/src/Interpreters/AnalyzedJoin.h +++ b/dbms/src/Interpreters/AnalyzedJoin.h @@ -109,7 +109,7 @@ public: NameSet getQualifiedColumnsSet() const; NamesWithAliases getNamesWithAliases(const NameSet & required_columns) const; - NamesWithAliases getRequiredColumns(const Block & sample, const Names & action_columns) const; + NamesWithAliases getRequiredColumns(const Block & sample, const Names & action_required_columns) const; void deduplicateAndQualifyColumnNames(const NameSet & left_table_columns, const String & right_table_prefix); size_t rightKeyInclusion(const String & name) const; diff --git a/dbms/src/Interpreters/BloomFilter.cpp b/dbms/src/Interpreters/BloomFilter.cpp index d489d7c200e..010f29d8449 100644 --- a/dbms/src/Interpreters/BloomFilter.cpp +++ b/dbms/src/Interpreters/BloomFilter.cpp @@ -21,9 +21,6 @@ static constexpr UInt64 SEED_GEN_B = 217728422; BloomFilter::BloomFilter(size_t size_, size_t hashes_, size_t seed_) : size(size_), hashes(hashes_), seed(seed_), words((size + sizeof(UnderType) - 1) / sizeof(UnderType)), filter(words, 0) {} -BloomFilter::BloomFilter(const BloomFilter & bloom_filter) - : size(bloom_filter.size), hashes(bloom_filter.hashes), seed(bloom_filter.seed), words(bloom_filter.words), filter(bloom_filter.filter) {} - bool BloomFilter::find(const char * data, size_t len) { size_t hash1 = CityHash_v1_0_2::CityHash64WithSeed(data, len, seed); diff --git a/dbms/src/Interpreters/BloomFilter.h b/dbms/src/Interpreters/BloomFilter.h index c9a515dc1e7..46ab3b6e82d 100644 --- a/dbms/src/Interpreters/BloomFilter.h +++ b/dbms/src/Interpreters/BloomFilter.h @@ -23,7 +23,6 @@ public: /// hashes -- number of used hash functions. /// seed -- random seed for hash functions generation. BloomFilter(size_t size_, size_t hashes_, size_t seed_); - BloomFilter(const BloomFilter & bloom_filter); bool find(const char * data, size_t len); void add(const char * data, size_t len); diff --git a/dbms/src/Interpreters/CatBoostModel.cpp b/dbms/src/Interpreters/CatBoostModel.cpp index b2e2a4c3d73..552905cfd5a 100644 --- a/dbms/src/Interpreters/CatBoostModel.cpp +++ b/dbms/src/Interpreters/CatBoostModel.cpp @@ -29,7 +29,7 @@ extern const int CANNOT_APPLY_CATBOOST_MODEL; /// CatBoost wrapper interface functions. struct CatBoostWrapperAPI { - typedef void ModelCalcerHandle; + using ModelCalcerHandle = void; ModelCalcerHandle * (* ModelCalcerCreate)(); diff --git a/dbms/src/Interpreters/Cluster.cpp b/dbms/src/Interpreters/Cluster.cpp index 3261c0caf59..97e8b7432bf 100644 --- a/dbms/src/Interpreters/Cluster.cpp +++ b/dbms/src/Interpreters/Cluster.cpp @@ -28,7 +28,7 @@ namespace { /// Default shard weight. -static constexpr UInt32 default_weight = 1; +constexpr UInt32 default_weight = 1; inline bool isLocalImpl(const Cluster::Address & address, const Poco::Net::SocketAddress & resolved_address, UInt16 clickhouse_port) { @@ -483,7 +483,6 @@ std::unique_ptr Cluster::getClusterWithMultipleShards(const std::vector } Cluster::Cluster(Cluster::ReplicasAsShardsTag, const Cluster & from, const Settings & settings) - : shards_info{}, addresses_with_failover{} { if (from.addresses_with_failover.empty()) throw Exception("Cluster is empty", ErrorCodes::LOGICAL_ERROR); @@ -525,7 +524,6 @@ Cluster::Cluster(Cluster::ReplicasAsShardsTag, const Cluster & from, const Setti Cluster::Cluster(Cluster::SubclusterTag, const Cluster & from, const std::vector & indices) - : shards_info{} { for (size_t index : indices) { diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index dbc963e0a27..6a1c2805fc0 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -502,13 +502,13 @@ MergeList & Context::getMergeList() { return shared->merge_list; } const MergeList & Context::getMergeList() const { return shared->merge_list; } -const Databases Context::getDatabases() const +const Databases & Context::getDatabases() const { auto lock = getLock(); return shared->databases; } -Databases Context::getDatabases() +Databases & Context::getDatabases() { auto lock = getLock(); return shared->databases; @@ -538,7 +538,7 @@ static String resolveDatabase(const String & database_name, const String & curre } -const DatabasePtr Context::getDatabase(const String & database_name) const +DatabasePtr Context::getDatabase(const String & database_name) const { auto lock = getLock(); String db = resolveDatabase(database_name, current_database); @@ -546,25 +546,7 @@ const DatabasePtr Context::getDatabase(const String & database_name) const return shared->databases[db]; } -DatabasePtr Context::getDatabase(const String & database_name) -{ - auto lock = getLock(); - String db = resolveDatabase(database_name, current_database); - assertDatabaseExists(db); - return shared->databases[db]; -} - -const DatabasePtr Context::tryGetDatabase(const String & database_name) const -{ - auto lock = getLock(); - String db = resolveDatabase(database_name, current_database); - auto it = shared->databases.find(db); - if (it == shared->databases.end()) - return {}; - return it->second; -} - -DatabasePtr Context::tryGetDatabase(const String & database_name) +DatabasePtr Context::tryGetDatabase(const String & database_name) const { auto lock = getLock(); String db = resolveDatabase(database_name, current_database); @@ -644,7 +626,7 @@ VolumePtr Context::setTemporaryStorage(const String & path, const String & polic shared->tmp_volume = tmp_policy->getVolume(0); } - if (!shared->tmp_volume->disks.size()) + if (shared->tmp_volume->disks.empty()) throw Exception("No disks volume for temporary files", ErrorCodes::NO_ELEMENTS_IN_CONFIG); return shared->tmp_volume; @@ -1053,7 +1035,7 @@ StoragePtr Context::getTable(const StorageID & table_id) const std::optional exc; auto res = getTableImpl(table_id, &exc); if (!res) - throw *exc; + throw Exception(*exc); return res; } diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index 5b5b8bdabd5..7c9925f8241 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -414,13 +414,11 @@ public: /// Get query for the CREATE table. ASTPtr getCreateExternalTableQuery(const String & table_name) const; - const DatabasePtr getDatabase(const String & database_name) const; - DatabasePtr getDatabase(const String & database_name); - const DatabasePtr tryGetDatabase(const String & database_name) const; - DatabasePtr tryGetDatabase(const String & database_name); + DatabasePtr getDatabase(const String & database_name) const; + DatabasePtr tryGetDatabase(const String & database_name) const; - const Databases getDatabases() const; - Databases getDatabases(); + const Databases & getDatabases() const; + Databases & getDatabases(); /// Allow to use named sessions. The thread will be run to cleanup sessions after timeout has expired. /// The method must be called at the server startup. diff --git a/dbms/src/Interpreters/CrossToInnerJoinVisitor.cpp b/dbms/src/Interpreters/CrossToInnerJoinVisitor.cpp index 21cd688cd61..c05e617f84d 100644 --- a/dbms/src/Interpreters/CrossToInnerJoinVisitor.cpp +++ b/dbms/src/Interpreters/CrossToInnerJoinVisitor.cpp @@ -277,10 +277,7 @@ bool getTables(ASTSelectQuery & select, std::vector & joined_tabl if (num_comma && (num_comma != (joined_tables.size() - 1))) throw Exception("Mix of COMMA and other JOINS is not supported", ErrorCodes::NOT_IMPLEMENTED); - if (num_array_join || num_using) - return false; - - return true; + return !(num_array_join || num_using); } } @@ -288,9 +285,7 @@ bool getTables(ASTSelectQuery & select, std::vector & joined_tabl bool CrossToInnerJoinMatcher::needChildVisit(ASTPtr & node, const ASTPtr &) { - if (node->as()) - return false; - return true; + return !node->as(); } void CrossToInnerJoinMatcher::visit(ASTPtr & ast, Data & data) diff --git a/dbms/src/Interpreters/DDLWorker.h b/dbms/src/Interpreters/DDLWorker.h index 95180d0cfb7..39ae768d7d8 100644 --- a/dbms/src/Interpreters/DDLWorker.h +++ b/dbms/src/Interpreters/DDLWorker.h @@ -62,7 +62,7 @@ private: bool taskShouldBeExecutedOnLeader(const ASTPtr ast_ddl, StoragePtr storage) const; /// Check that shard has consistent config with table - void checkShardConfig(const String & table, const DDLTask & taks, StoragePtr storage) const; + void checkShardConfig(const String & table, const DDLTask & task, StoragePtr storage) const; /// Executes query only on leader replica in case of replicated table. /// Queries like TRUNCATE/ALTER .../OPTIMIZE have to be executed only on one node of shard. @@ -84,7 +84,7 @@ private: void cleanupQueue(Int64 current_time_seconds, const ZooKeeperPtr & zookeeper); /// Init task node - void createStatusDirs(const std::string & node_name, const ZooKeeperPtr & zookeeper); + void createStatusDirs(const std::string & node_path, const ZooKeeperPtr & zookeeper); void runMainThread(); diff --git a/dbms/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp b/dbms/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp index 6baaa7e5a1a..6c5b3ed6425 100644 --- a/dbms/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp +++ b/dbms/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp @@ -84,7 +84,7 @@ void ExecuteScalarSubqueriesMatcher::visit(const ASTSubquery & subquery, ASTPtr Context subquery_context = data.context; Settings subquery_settings = data.context.getSettings(); subquery_settings.max_result_rows = 1; - subquery_settings.extremes = 0; + subquery_settings.extremes = false; subquery_context.setSettings(subquery_settings); ASTPtr subquery_select = subquery.children.at(0); diff --git a/dbms/src/Interpreters/ExpressionActions.cpp b/dbms/src/Interpreters/ExpressionActions.cpp index dafb270defa..67899995c4a 100644 --- a/dbms/src/Interpreters/ExpressionActions.cpp +++ b/dbms/src/Interpreters/ExpressionActions.cpp @@ -253,12 +253,12 @@ void ExpressionAction::prepare(Block & sample_block, const Settings & settings, { Block new_block; - for (size_t i = 0; i < projection.size(); ++i) + for (const auto & elem : projection) { - const std::string & name = projection[i].first; - const std::string & alias = projection[i].second; + const std::string & name = elem.first; + const std::string & alias = elem.second; ColumnWithTypeAndName column = sample_block.getByName(name); - if (alias != "") + if (!alias.empty()) column.name = alias; new_block.insert(std::move(column)); } @@ -269,12 +269,12 @@ void ExpressionAction::prepare(Block & sample_block, const Settings & settings, case ADD_ALIASES: { - for (size_t i = 0; i < projection.size(); ++i) + for (const auto & elem : projection) { - const std::string & name = projection[i].first; - const std::string & alias = projection[i].second; + const std::string & name = elem.first; + const std::string & alias = elem.second; const ColumnWithTypeAndName & column = sample_block.getByName(name); - if (alias != "" && !sample_block.has(alias)) + if (!alias.empty() && !sample_block.has(alias)) sample_block.insert({column.column, column.type, alias}); } break; @@ -371,12 +371,12 @@ void ExpressionAction::execute(Block & block, bool dry_run, ExtraBlockPtr & not_ { Block new_block; - for (size_t i = 0; i < projection.size(); ++i) + for (const auto & elem : projection) { - const std::string & name = projection[i].first; - const std::string & alias = projection[i].second; + const std::string & name = elem.first; + const std::string & alias = elem.second; ColumnWithTypeAndName column = block.getByName(name); - if (alias != "") + if (!alias.empty()) column.name = alias; new_block.insert(std::move(column)); } @@ -388,12 +388,12 @@ void ExpressionAction::execute(Block & block, bool dry_run, ExtraBlockPtr & not_ case ADD_ALIASES: { - for (size_t i = 0; i < projection.size(); ++i) + for (const auto & elem : projection) { - const std::string & name = projection[i].first; - const std::string & alias = projection[i].second; + const std::string & name = elem.first; + const std::string & alias = elem.second; const ColumnWithTypeAndName & column = block.getByName(name); - if (alias != "" && !block.has(alias)) + if (!alias.empty() && !block.has(alias)) block.insert({column.column, column.type, alias}); } break; @@ -497,7 +497,7 @@ std::string ExpressionAction::toString() const if (i) ss << ", "; ss << projection[i].first; - if (projection[i].second != "" && projection[i].second != projection[i].first) + if (!projection[i].second.empty() && projection[i].second != projection[i].first) ss << " AS " << projection[i].second; } break; @@ -558,7 +558,7 @@ void ExpressionActions::add(const ExpressionAction & action) void ExpressionActions::addImpl(ExpressionAction action, Names & new_names) { - if (action.result_name != "") + if (!action.result_name.empty()) new_names.push_back(action.result_name); if (action.array_join) @@ -720,9 +720,8 @@ std::string ExpressionActions::getSmallestColumn(const NamesAndTypesList & colum void ExpressionActions::finalize(const Names & output_columns) { NameSet final_columns; - for (size_t i = 0; i < output_columns.size(); ++i) + for (const auto & name : output_columns) { - const std::string & name = output_columns[i]; if (!sample_block.has(name)) throw Exception("Unknown column: " + name + ", there are only columns " + sample_block.dumpNames(), ErrorCodes::UNKNOWN_IDENTIFIER); @@ -743,8 +742,8 @@ void ExpressionActions::finalize(const Names & output_columns) { NamesAndTypesList sample_columns = sample_block.getNamesAndTypesList(); - for (NamesAndTypesList::iterator it = sample_columns.begin(); it != sample_columns.end(); ++it) - unmodified_columns.insert(it->name); + for (const auto & sample_column : sample_columns) + unmodified_columns.insert(sample_column.name); } /// Let's go from the end and maintain set of required columns at this stage. @@ -942,17 +941,17 @@ std::string ExpressionActions::dumpActions() const std::stringstream ss; ss << "input:\n"; - for (NamesAndTypesList::const_iterator it = input_columns.begin(); it != input_columns.end(); ++it) - ss << it->name << " " << it->type->getName() << "\n"; + for (const auto & input_column : input_columns) + ss << input_column.name << " " << input_column.type->getName() << "\n"; ss << "\nactions:\n"; - for (size_t i = 0; i < actions.size(); ++i) - ss << actions[i].toString() << '\n'; + for (const auto & action : actions) + ss << action.toString() << '\n'; ss << "\noutput:\n"; NamesAndTypesList output_columns = sample_block.getNamesAndTypesList(); - for (NamesAndTypesList::const_iterator it = output_columns.begin(); it != output_columns.end(); ++it) - ss << it->name << " " << it->type->getName() << "\n"; + for (const auto & output_column : output_columns) + ss << output_column.name << " " << output_column.type->getName() << "\n"; return ss.str(); } @@ -991,9 +990,9 @@ void ExpressionActions::optimizeArrayJoin() needed = actions[i].getNeededColumns(); - for (size_t j = 0; j < needed.size(); ++j) + for (const auto & elem : needed) { - if (array_joined_columns.count(needed[j])) + if (array_joined_columns.count(elem)) { depends_on_array_join = true; break; @@ -1006,7 +1005,7 @@ void ExpressionActions::optimizeArrayJoin() if (first_array_join == NONE) first_array_join = i; - if (actions[i].result_name != "") + if (!actions[i].result_name.empty()) array_joined_columns.insert(actions[i].result_name); if (actions[i].array_join) array_joined_columns.insert(actions[i].array_join->columns.begin(), actions[i].array_join->columns.end()); diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp index 4dee6565b52..251eee93644 100644 --- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp +++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp @@ -242,15 +242,12 @@ void ExpressionAnalyzer::analyzeAggregation() if (group_asts.empty()) { select_query->setExpression(ASTSelectQuery::Expression::GROUP_BY, {}); - has_aggregation = select_query->having() || aggregate_descriptions.size(); + has_aggregation = select_query->having() || !aggregate_descriptions.empty(); } } - for (size_t i = 0; i < aggregate_descriptions.size(); ++i) - { - AggregateDescription & desc = aggregate_descriptions[i]; + for (const auto & desc : aggregate_descriptions) aggregated_columns.emplace_back(desc.column_name, desc.function->getReturnType()); - } } else { @@ -717,10 +714,10 @@ bool SelectQueryExpressionAnalyzer::appendGroupBy(ExpressionActionsChain & chain ExpressionActionsChain::Step & step = chain.steps.back(); ASTs asts = select_query->groupBy()->children; - for (size_t i = 0; i < asts.size(); ++i) + for (const auto & ast : asts) { - step.required_output.push_back(asts[i]->getColumnName()); - getRootActions(asts[i], only_types, step.actions); + step.required_output.emplace_back(ast->getColumnName()); + getRootActions(ast, only_types, step.actions); } return true; @@ -733,13 +730,9 @@ void SelectQueryExpressionAnalyzer::appendAggregateFunctionsArguments(Expression initChain(chain, sourceColumns()); ExpressionActionsChain::Step & step = chain.steps.back(); - for (size_t i = 0; i < aggregate_descriptions.size(); ++i) - { - for (size_t j = 0; j < aggregate_descriptions[i].argument_names.size(); ++j) - { - step.required_output.push_back(aggregate_descriptions[i].argument_names[j]); - } - } + for (const auto & desc : aggregate_descriptions) + for (const auto & name : desc.argument_names) + step.required_output.emplace_back(name); /// Collect aggregates removing duplicates by node.getColumnName() /// It's not clear why we recollect aggregates (for query parts) while we're able to use previously collected ones (for entire query) @@ -804,7 +797,7 @@ bool SelectQueryExpressionAnalyzer::appendOrderBy(ExpressionActionsChain & chain for (auto & child : select_query->orderBy()->children) { const auto * ast = child->as(); - if (!ast || ast->children.size() < 1) + if (!ast || ast->children.empty()) throw Exception("Bad order expression AST", ErrorCodes::UNKNOWN_TYPE_OF_AST_NODE); ASTPtr order_expression = ast->children.at(0); step.required_output.push_back(order_expression->getColumnName()); @@ -862,12 +855,12 @@ void SelectQueryExpressionAnalyzer::appendProjectResult(ExpressionActionsChain & NamesWithAliases result_columns; ASTs asts = select_query->select()->children; - for (size_t i = 0; i < asts.size(); ++i) + for (const auto & ast : asts) { - String result_name = asts[i]->getAliasOrColumnName(); + String result_name = ast->getAliasOrColumnName(); if (required_result_columns.empty() || required_result_columns.count(result_name)) { - result_columns.emplace_back(asts[i]->getColumnName(), result_name); + result_columns.emplace_back(ast->getColumnName(), result_name); step.required_output.push_back(result_columns.back().second); } } @@ -898,17 +891,17 @@ ExpressionActionsPtr ExpressionAnalyzer::getActions(bool add_aliases, bool proje else asts = ASTs(1, query); - for (size_t i = 0; i < asts.size(); ++i) + for (const auto & ast : asts) { - std::string name = asts[i]->getColumnName(); + std::string name = ast->getColumnName(); std::string alias; if (add_aliases) - alias = asts[i]->getAliasOrColumnName(); + alias = ast->getAliasOrColumnName(); else alias = name; result_columns.emplace_back(name, alias); result_names.push_back(alias); - getRootActions(asts[i], false, actions); + getRootActions(ast, false, actions); } if (add_aliases) diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.h b/dbms/src/Interpreters/ExpressionAnalyzer.h index ac48bfbd5cd..7caed9a69e1 100644 --- a/dbms/src/Interpreters/ExpressionAnalyzer.h +++ b/dbms/src/Interpreters/ExpressionAnalyzer.h @@ -272,7 +272,7 @@ private: * Because while making set we will read data from StorageSet which is not allowed. * Returns valid SetPtr from StorageSet if the latter is used after IN or nullptr otherwise. */ - SetPtr isPlainStorageSetInSubquery(const ASTPtr & subquery_of_table_name); + SetPtr isPlainStorageSetInSubquery(const ASTPtr & subquery_or_table_name); JoinPtr makeTableJoin(const ASTTablesInSelectQueryElement & join_element); void makeSubqueryForJoin(const ASTTablesInSelectQueryElement & join_element, NamesWithAliases && required_columns_with_aliases, diff --git a/dbms/src/Interpreters/ExpressionJIT.cpp b/dbms/src/Interpreters/ExpressionJIT.cpp index 3385c089788..656df317554 100644 --- a/dbms/src/Interpreters/ExpressionJIT.cpp +++ b/dbms/src/Interpreters/ExpressionJIT.cpp @@ -152,7 +152,7 @@ struct SymbolResolver : public llvm::orc::SymbolResolver bool has_resolved = false; impl.lookup({*symbol}, [&](llvm::Expected resolved) { - if (resolved && resolved->size()) + if (resolved && !resolved->empty()) { query->notifySymbolMetRequiredState(symbol, resolved->begin()->second); has_resolved = true; @@ -198,7 +198,7 @@ struct LLVMContext /// returns used memory void compileAllFunctionsToNativeCode() { - if (!module->size()) + if (module->empty()) return; llvm::PassManagerBuilder pass_manager_builder; llvm::legacy::PassManager mpm; @@ -346,7 +346,7 @@ static void compileFunctionToLLVMByteCode(LLVMContext & context, const IFunction } } ValuePlaceholders arguments(arg_types.size()); - for (size_t i = 0; i < arguments.size(); ++i) + for (size_t i = 0; i < arguments.size(); ++i) // NOLINT { arguments[i] = [&b, &col = columns[i], &type = arg_types[i]]() -> llvm::Value * { @@ -699,10 +699,10 @@ void compileFunctions( fused[*dep].insert(fused[*dep].end(), fused[i].begin(), fused[i].end()); } - for (size_t i = 0; i < actions.size(); ++i) + for (auto & action : actions) { - if (actions[i].type == ExpressionAction::APPLY_FUNCTION && actions[i].is_function_compiled) - actions[i].function = actions[i].function_base->prepare({}, {}, 0); /// Arguments are not used for LLVMFunction. + if (action.type == ExpressionAction::APPLY_FUNCTION && action.is_function_compiled) + action.function = action.function_base->prepare({}, {}, 0); /// Arguments are not used for LLVMFunction. } } From 63cf9f8c5b8bad310e8b1a44b13b2dffb569c8e8 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 9 Mar 2020 03:08:02 +0300 Subject: [PATCH 143/712] clang-tidy, part 9 --- .../ExternalLoaderXMLConfigRepository.h | 2 +- dbms/src/Interpreters/ExternalModelsLoader.h | 2 +- dbms/src/Interpreters/FillingRow.cpp | 6 +++--- dbms/src/Interpreters/IdentifierSemantic.h | 2 +- .../InJoinSubqueriesPreprocessor.cpp | 4 +--- .../InJoinSubqueriesPreprocessor.h | 2 +- dbms/src/Interpreters/InterpreterDropQuery.h | 2 +- .../Interpreters/InterpreterSelectQuery.cpp | 21 ++++++------------- .../src/Interpreters/InterpreterSelectQuery.h | 2 +- .../src/TableFunctions/TableFunctionMySQL.cpp | 4 ++-- 10 files changed, 18 insertions(+), 29 deletions(-) diff --git a/dbms/src/Interpreters/ExternalLoaderXMLConfigRepository.h b/dbms/src/Interpreters/ExternalLoaderXMLConfigRepository.h index 808fa66fdbf..75d9be8bb42 100644 --- a/dbms/src/Interpreters/ExternalLoaderXMLConfigRepository.h +++ b/dbms/src/Interpreters/ExternalLoaderXMLConfigRepository.h @@ -27,7 +27,7 @@ public: Poco::Timestamp getUpdateTime(const std::string & definition_entity_name) override; /// May contain definition about several entities (several dictionaries in one .xml file) - LoadablesConfigurationPtr load(const std::string & definition_entity_name) override; + LoadablesConfigurationPtr load(const std::string & config_file) override; private: const String name; diff --git a/dbms/src/Interpreters/ExternalModelsLoader.h b/dbms/src/Interpreters/ExternalModelsLoader.h index 4d09cebc307..3f09512338d 100644 --- a/dbms/src/Interpreters/ExternalModelsLoader.h +++ b/dbms/src/Interpreters/ExternalModelsLoader.h @@ -27,7 +27,7 @@ public: protected: LoadablePtr create(const std::string & name, const Poco::Util::AbstractConfiguration & config, - const std::string & key_in_config, const std::string & repository_name) const override; + const std::string & config_prefix, const std::string & repository_name) const override; friend class StorageSystemModels; private: diff --git a/dbms/src/Interpreters/FillingRow.cpp b/dbms/src/Interpreters/FillingRow.cpp index 9d4c81dc70b..dc48b5347c4 100644 --- a/dbms/src/Interpreters/FillingRow.cpp +++ b/dbms/src/Interpreters/FillingRow.cpp @@ -17,7 +17,7 @@ bool equals(const Field & lhs, const Field & rhs) } -FillingRow::FillingRow(const SortDescription & description_) : description(description_) +FillingRow::FillingRow(const SortDescription & sort_description) : description(sort_description) { row.resize(description.size()); } @@ -114,8 +114,8 @@ void insertFromFillingRow(MutableColumns & filling_columns, MutableColumns & oth filling_columns[i]->insert(filling_row[i]); } - for (size_t i = 0; i < other_columns.size(); ++i) - other_columns[i]->insertDefault(); + for (const auto & other_column : other_columns) + other_column->insertDefault(); } void copyRowFromColumns(MutableColumns & dest, const Columns & source, size_t row_num) diff --git a/dbms/src/Interpreters/IdentifierSemantic.h b/dbms/src/Interpreters/IdentifierSemantic.h index 0b92b7b7716..9e4d9ef6ccb 100644 --- a/dbms/src/Interpreters/IdentifierSemantic.h +++ b/dbms/src/Interpreters/IdentifierSemantic.h @@ -47,7 +47,7 @@ struct IdentifierSemantic static void setColumnShortName(ASTIdentifier & identifier, const DatabaseAndTableWithAlias & db_and_table); static void setColumnLongName(ASTIdentifier & identifier, const DatabaseAndTableWithAlias & db_and_table); static bool canBeAlias(const ASTIdentifier & identifier); - static void setMembership(ASTIdentifier &, size_t table_no); + static void setMembership(ASTIdentifier &, size_t table_pos); static void coverName(ASTIdentifier &, const String & alias); static std::optional uncover(const ASTIdentifier & identifier); static std::optional getMembership(const ASTIdentifier & identifier); diff --git a/dbms/src/Interpreters/InJoinSubqueriesPreprocessor.cpp b/dbms/src/Interpreters/InJoinSubqueriesPreprocessor.cpp index 149565beadc..4d250881cdd 100644 --- a/dbms/src/Interpreters/InJoinSubqueriesPreprocessor.cpp +++ b/dbms/src/Interpreters/InJoinSubqueriesPreprocessor.cpp @@ -139,9 +139,7 @@ public: return false; /// Processed, process others /// Descent into all children, but not into subqueries of other kind (scalar subqueries), that are irrelevant to us. - if (child->as()) - return false; - return true; + return !child->as(); } private: diff --git a/dbms/src/Interpreters/InJoinSubqueriesPreprocessor.h b/dbms/src/Interpreters/InJoinSubqueriesPreprocessor.h index 34d49888528..ff39d812dee 100644 --- a/dbms/src/Interpreters/InJoinSubqueriesPreprocessor.h +++ b/dbms/src/Interpreters/InJoinSubqueriesPreprocessor.h @@ -50,7 +50,7 @@ public: , checker(std::move(_checker)) {} - void visit(ASTPtr & query) const; + void visit(ASTPtr & ast) const; private: const Context & context; diff --git a/dbms/src/Interpreters/InterpreterDropQuery.h b/dbms/src/Interpreters/InterpreterDropQuery.h index 99a6a34474f..6a9c249973e 100644 --- a/dbms/src/Interpreters/InterpreterDropQuery.h +++ b/dbms/src/Interpreters/InterpreterDropQuery.h @@ -32,7 +32,7 @@ private: BlockIO executeToTable(const String & database_name, const String & table_name, ASTDropQuery::Kind kind, bool if_exists, bool is_temporary, bool no_ddl_lock); - BlockIO executeToDictionary(const String & database_name, const String & table_name, ASTDropQuery::Kind kind, bool if_exists, bool is_temporary, bool no_ddl_lock); + BlockIO executeToDictionary(const String & database_name, const String & dictionary_name, ASTDropQuery::Kind kind, bool if_exists, bool is_temporary, bool no_ddl_lock); DatabasePtr tryGetDatabase(const String & database_name, bool exists); diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index 8285978c7bd..bd17050bcb9 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -204,7 +204,7 @@ static Context getSubqueryContext(const Context & context) subquery_settings.max_result_rows = 0; subquery_settings.max_result_bytes = 0; /// The calculation of extremes does not make sense and is not necessary (if you do it, then the extremes of the subquery can be taken for whole query). - subquery_settings.extremes = 0; + subquery_settings.extremes = false; subquery_context.setSettings(subquery_settings); return subquery_context; } @@ -1518,14 +1518,6 @@ void InterpreterSelectQuery::executeFetchColumns( pipes.emplace_back(std::move(source)); } - /// Pin sources for merge tree tables. -// bool pin_sources = dynamic_cast(storage.get()) != nullptr; -// if (pin_sources) -// { -// for (size_t i = 0; i < pipes.size(); ++i) -// pipes[i].pinSources(i); -// } - for (auto & pipe : pipes) pipe.enableQuota(); @@ -1558,19 +1550,19 @@ void InterpreterSelectQuery::executeFetchColumns( } -void InterpreterSelectQuery::executeWhere(Pipeline & pipeline, const ExpressionActionsPtr & expression, bool remove_fiter) +void InterpreterSelectQuery::executeWhere(Pipeline & pipeline, const ExpressionActionsPtr & expression, bool remove_filter) { pipeline.transform([&](auto & stream) { - stream = std::make_shared(stream, expression, getSelectQuery().where()->getColumnName(), remove_fiter); + stream = std::make_shared(stream, expression, getSelectQuery().where()->getColumnName(), remove_filter); }); } -void InterpreterSelectQuery::executeWhere(QueryPipeline & pipeline, const ExpressionActionsPtr & expression, bool remove_fiter) +void InterpreterSelectQuery::executeWhere(QueryPipeline & pipeline, const ExpressionActionsPtr & expression, bool remove_filter) { pipeline.addSimpleTransform([&](const Block & block) { - return std::make_shared(block, expression, getSelectQuery().where()->getColumnName(), remove_fiter); + return std::make_shared(block, expression, getSelectQuery().where()->getColumnName(), remove_filter); }); } @@ -2541,9 +2533,8 @@ void InterpreterSelectQuery::unifyStreams(Pipeline & pipeline, Block header) if (header.columns() > 1 && header.has("_dummy")) header.erase("_dummy"); - for (size_t i = 0; i < pipeline.streams.size(); ++i) + for (auto & stream : pipeline.streams) { - auto & stream = pipeline.streams[i]; auto stream_header = stream->getHeader(); auto mode = ConvertingBlockInputStream::MatchColumnsMode::Name; diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.h b/dbms/src/Interpreters/InterpreterSelectQuery.h index 882c5a71e2a..94d847ea550 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.h +++ b/dbms/src/Interpreters/InterpreterSelectQuery.h @@ -182,7 +182,7 @@ private: void executeSubqueriesInSetsAndJoins(Pipeline & pipeline, const std::unordered_map & subqueries_for_sets); void executeMergeSorted(Pipeline & pipeline, const SortDescription & sort_description, UInt64 limit); - void executeWhere(QueryPipeline & pipeline, const ExpressionActionsPtr & expression, bool remove_fiter); + void executeWhere(QueryPipeline & pipeline, const ExpressionActionsPtr & expression, bool remove_filter); void executeAggregation(QueryPipeline & pipeline, const ExpressionActionsPtr & expression, bool overflow_row, bool final); void executeMergeAggregated(QueryPipeline & pipeline, bool overflow_row, bool final); void executeTotalsAndHaving(QueryPipeline & pipeline, bool has_having, const ExpressionActionsPtr & expression, bool overflow_row, bool final); diff --git a/dbms/src/TableFunctions/TableFunctionMySQL.cpp b/dbms/src/TableFunctions/TableFunctionMySQL.cpp index 4a88537414e..cc8e17d0d3e 100644 --- a/dbms/src/TableFunctions/TableFunctionMySQL.cpp +++ b/dbms/src/TableFunctions/TableFunctionMySQL.cpp @@ -48,8 +48,8 @@ StoragePtr TableFunctionMySQL::executeImpl(const ASTPtr & ast_function, const Co throw Exception("Table function 'mysql' requires 5-7 parameters: MySQL('host:port', database, table, 'user', 'password'[, replace_query, 'on_duplicate_clause']).", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - for (size_t i = 0; i < args.size(); ++i) - args[i] = evaluateConstantExpressionOrIdentifierAsLiteral(args[i], context); + for (auto & arg : args) + arg = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context); std::string host_port = args[0]->as().value.safeGet(); std::string remote_database_name = args[1]->as().value.safeGet(); From 48c92cc1cdd306c9eb487b5db0a6d43adbf55c69 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 9 Mar 2020 03:28:05 +0300 Subject: [PATCH 144/712] clang-tidy, part 10 --- dbms/src/Dictionaries/CacheDictionary.cpp | 2 +- .../ClusterProxy/SelectStreamFactory.cpp | 4 ++-- dbms/src/Interpreters/Join.cpp | 2 +- dbms/src/Interpreters/JoinSwitcher.cpp | 2 +- .../JoinToSubqueryTransformVisitor.cpp | 4 +--- .../src/Interpreters/MutationsInterpreter.cpp | 2 +- .../PredicateExpressionsOptimizer.cpp | 4 ++-- .../Interpreters/PredicateRewriteVisitor.cpp | 8 ++++---- dbms/src/Interpreters/QueryAliasesVisitor.cpp | 4 +--- dbms/src/Interpreters/QueryNormalizer.cpp | 4 +--- dbms/src/Interpreters/QueryNormalizer.h | 2 +- dbms/src/Interpreters/Set.cpp | 10 +++++----- dbms/src/Interpreters/Set.h | 2 +- dbms/src/Interpreters/SyntaxAnalyzer.cpp | 2 +- .../TranslateQualifiedNamesVisitor.h | 8 ++++---- .../Interpreters/evaluateConstantExpression.h | 2 +- dbms/src/Interpreters/executeQuery.cpp | 4 ++-- dbms/src/Interpreters/interpretSubquery.cpp | 2 +- dbms/src/Interpreters/sortBlock.cpp | 20 +++++++++---------- .../MergeTree/MergeTreeWhereOptimizer.cpp | 7 ++----- .../MergeTree/MergeTreeWhereOptimizer.h | 2 +- 21 files changed, 44 insertions(+), 53 deletions(-) diff --git a/dbms/src/Dictionaries/CacheDictionary.cpp b/dbms/src/Dictionaries/CacheDictionary.cpp index aba58e96bba..2294c99c111 100644 --- a/dbms/src/Dictionaries/CacheDictionary.cpp +++ b/dbms/src/Dictionaries/CacheDictionary.cpp @@ -748,7 +748,7 @@ void CacheDictionary::updateThreadFunction() UpdateUnitPtr current_unit_ptr; - while (update_request.size() && update_queue.tryPop(current_unit_ptr)) + while (!update_request.empty() && update_queue.tryPop(current_unit_ptr)) update_request.emplace_back(std::move(current_unit_ptr)); BunchUpdateUnit bunch_update_unit(update_request); diff --git a/dbms/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/dbms/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp index b6ece8d0176..9d12a308b1f 100644 --- a/dbms/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp +++ b/dbms/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp @@ -111,10 +111,10 @@ Pipe createLocalStream(const ASTPtr & query_ast, const Block & header, const Con return std::move(pipeline).getPipe(); } -static String formattedAST(const ASTPtr & ast) +String formattedAST(const ASTPtr & ast) { if (!ast) - return ""; + return {}; std::stringstream ss; formatAST(*ast, ss, false, true); return ss.str(); diff --git a/dbms/src/Interpreters/Join.cpp b/dbms/src/Interpreters/Join.cpp index 4693cfb4ec2..802fddf1406 100644 --- a/dbms/src/Interpreters/Join.cpp +++ b/dbms/src/Interpreters/Join.cpp @@ -92,7 +92,7 @@ static ColumnWithTypeAndName correctNullability(ColumnWithTypeAndName && column, if (nullable) { JoinCommon::convertColumnToNullable(column); - if (column.type->isNullable() && negative_null_map.size()) + if (column.type->isNullable() && !negative_null_map.empty()) { MutableColumnPtr mutable_column = (*std::move(column.column)).mutate(); assert_cast(*mutable_column).applyNegatedNullMap(negative_null_map); diff --git a/dbms/src/Interpreters/JoinSwitcher.cpp b/dbms/src/Interpreters/JoinSwitcher.cpp index 142a7a1af94..5636022b563 100644 --- a/dbms/src/Interpreters/JoinSwitcher.cpp +++ b/dbms/src/Interpreters/JoinSwitcher.cpp @@ -59,7 +59,7 @@ void JoinSwitcher::switchJoin() /// names to positions optimization std::vector positions; std::vector is_nullable; - if (right_blocks.size()) + if (!right_blocks.empty()) { positions.reserve(right_sample_block.columns()); const Block & tmp_block = *right_blocks.begin(); diff --git a/dbms/src/Interpreters/JoinToSubqueryTransformVisitor.cpp b/dbms/src/Interpreters/JoinToSubqueryTransformVisitor.cpp index 83066996cff..39037613dda 100644 --- a/dbms/src/Interpreters/JoinToSubqueryTransformVisitor.cpp +++ b/dbms/src/Interpreters/JoinToSubqueryTransformVisitor.cpp @@ -183,9 +183,7 @@ struct ColumnAliasesMatcher static bool needChildVisit(const ASTPtr & node, const ASTPtr &) { - if (node->as()) - return false; - return true; + return !node->as(); } static void visit(const ASTPtr & ast, Data & data) diff --git a/dbms/src/Interpreters/MutationsInterpreter.cpp b/dbms/src/Interpreters/MutationsInterpreter.cpp index a8b124447e6..3acd04d99f6 100644 --- a/dbms/src/Interpreters/MutationsInterpreter.cpp +++ b/dbms/src/Interpreters/MutationsInterpreter.cpp @@ -638,7 +638,7 @@ ASTPtr MutationsInterpreter::prepareInterpreterSelectQuery(std::vector & bool empty = false; /// In all other cases we cannot have empty key if (auto key_function = key_expr->as()) - empty = key_function->arguments->children.size() == 0; + empty = key_function->arguments->children.empty(); /// Not explicitely spicified empty key if (!empty) diff --git a/dbms/src/Interpreters/PredicateExpressionsOptimizer.cpp b/dbms/src/Interpreters/PredicateExpressionsOptimizer.cpp index d2f3ecfa30d..8791008ac09 100644 --- a/dbms/src/Interpreters/PredicateExpressionsOptimizer.cpp +++ b/dbms/src/Interpreters/PredicateExpressionsOptimizer.cpp @@ -100,8 +100,8 @@ std::vector PredicateExpressionsOptimizer::extractTablesPredicates(const A tables_predicates[*expression_info.unique_reference_tables_pos.begin()].emplace_back(predicate_expression); else if (expression_info.unique_reference_tables_pos.empty()) { - for (size_t index = 0; index < tables_predicates.size(); ++index) - tables_predicates[index].emplace_back(predicate_expression); + for (auto & predicate : tables_predicates) + predicate.emplace_back(predicate_expression); } } } diff --git a/dbms/src/Interpreters/PredicateRewriteVisitor.cpp b/dbms/src/Interpreters/PredicateRewriteVisitor.cpp index 6bd16ddc066..a834e68172b 100644 --- a/dbms/src/Interpreters/PredicateRewriteVisitor.cpp +++ b/dbms/src/Interpreters/PredicateRewriteVisitor.cpp @@ -26,7 +26,7 @@ void PredicateRewriteVisitorData::visit(ASTSelectWithUnionQuery & union_select_q { auto & internal_select_list = union_select_query.list_of_selects->children; - if (internal_select_list.size() > 0) + if (!internal_select_list.empty()) visitFirstInternalSelect(*internal_select_list[0]->as(), internal_select_list[0]); for (size_t index = 1; index < internal_select_list.size(); ++index) @@ -96,15 +96,15 @@ bool PredicateRewriteVisitorData::rewriteSubquery(ASTSelectQuery & subquery, con ASTPtr optimize_predicate = predicate->clone(); cleanAliasAndCollectIdentifiers(optimize_predicate, identifiers); - for (size_t index = 0; index < identifiers.size(); ++index) + for (const auto & identifier : identifiers) { - const auto & column_name = identifiers[index]->shortName(); + const auto & column_name = identifier->shortName(); const auto & outer_column_iterator = std::find(outer_columns.begin(), outer_columns.end(), column_name); /// For lambda functions, we can't always find them in the list of columns /// For example: SELECT * FROM system.one WHERE arrayMap(x -> x, [dummy]) = [0] if (outer_column_iterator != outer_columns.end()) - identifiers[index]->setShortName(inner_columns[outer_column_iterator - outer_columns.begin()]); + identifier->setShortName(inner_columns[outer_column_iterator - outer_columns.begin()]); } /// We only need to push all the predicates to subquery having diff --git a/dbms/src/Interpreters/QueryAliasesVisitor.cpp b/dbms/src/Interpreters/QueryAliasesVisitor.cpp index 6d8f0266670..9caed01ca6d 100644 --- a/dbms/src/Interpreters/QueryAliasesVisitor.cpp +++ b/dbms/src/Interpreters/QueryAliasesVisitor.cpp @@ -33,9 +33,7 @@ static String wrongAliasMessage(const ASTPtr & ast, const ASTPtr & prev_ast, con bool QueryAliasesMatcher::needChildVisit(const ASTPtr & node, const ASTPtr &) { /// Don't descent into table functions and subqueries and special case for ArrayJoin. - if (node->as() || node->as() || node->as()) - return false; - return true; + return !(node->as() || node->as() || node->as()); } void QueryAliasesMatcher::visit(const ASTPtr & ast, Data & data) diff --git a/dbms/src/Interpreters/QueryNormalizer.cpp b/dbms/src/Interpreters/QueryNormalizer.cpp index 363d5ba64b1..93474b7b2ce 100644 --- a/dbms/src/Interpreters/QueryNormalizer.cpp +++ b/dbms/src/Interpreters/QueryNormalizer.cpp @@ -129,9 +129,7 @@ void QueryNormalizer::visit(ASTTablesInSelectQueryElement & node, const ASTPtr & static bool needVisitChild(const ASTPtr & child) { - if (child->as() || child->as()) - return false; - return true; + return !(child->as() || child->as()); } /// special visitChildren() for ASTSelectQuery diff --git a/dbms/src/Interpreters/QueryNormalizer.h b/dbms/src/Interpreters/QueryNormalizer.h index b842ae3f018..5bd0064c002 100644 --- a/dbms/src/Interpreters/QueryNormalizer.h +++ b/dbms/src/Interpreters/QueryNormalizer.h @@ -63,7 +63,7 @@ public: private: Data & visitor_data; - static void visit(ASTPtr & query, Data & data); + static void visit(ASTPtr & ast, Data & data); static void visit(ASTIdentifier &, ASTPtr &, Data &); static void visit(ASTTablesInSelectQueryElement &, const ASTPtr &, Data &); diff --git a/dbms/src/Interpreters/Set.cpp b/dbms/src/Interpreters/Set.cpp index 52207a0ebac..6b6a5a58c8a 100644 --- a/dbms/src/Interpreters/Set.cpp +++ b/dbms/src/Interpreters/Set.cpp @@ -103,14 +103,14 @@ void NO_INLINE Set::insertFromBlockImplCase( } -void Set::setHeader(const Block & block) +void Set::setHeader(const Block & header) { std::unique_lock lock(rwlock); if (!empty()) return; - keys_size = block.columns(); + keys_size = header.columns(); ColumnRawPtrs key_columns; key_columns.reserve(keys_size); data_types.reserve(keys_size); @@ -122,10 +122,10 @@ void Set::setHeader(const Block & block) /// Remember the columns we will work with for (size_t i = 0; i < keys_size; ++i) { - materialized_columns.emplace_back(block.safeGetByPosition(i).column->convertToFullColumnIfConst()); + materialized_columns.emplace_back(header.safeGetByPosition(i).column->convertToFullColumnIfConst()); key_columns.emplace_back(materialized_columns.back().get()); - data_types.emplace_back(block.safeGetByPosition(i).type); - set_elements_types.emplace_back(block.safeGetByPosition(i).type); + data_types.emplace_back(header.safeGetByPosition(i).type); + set_elements_types.emplace_back(header.safeGetByPosition(i).type); /// Convert low cardinality column to full. if (auto * low_cardinality_type = typeid_cast(data_types.back().get())) diff --git a/dbms/src/Interpreters/Set.h b/dbms/src/Interpreters/Set.h index f7a2f18d2ad..c9605d4e11e 100644 --- a/dbms/src/Interpreters/Set.h +++ b/dbms/src/Interpreters/Set.h @@ -223,7 +223,7 @@ public: std::vector functions; }; - MergeTreeSetIndex(const Columns & set_elements, std::vector && indexes_mapping_); + MergeTreeSetIndex(const Columns & set_elements, std::vector && index_mapping_); size_t size() const { return ordered_set.at(0)->size(); } diff --git a/dbms/src/Interpreters/SyntaxAnalyzer.cpp b/dbms/src/Interpreters/SyntaxAnalyzer.cpp index 187fea9087a..ce6f163ab5b 100644 --- a/dbms/src/Interpreters/SyntaxAnalyzer.cpp +++ b/dbms/src/Interpreters/SyntaxAnalyzer.cpp @@ -702,7 +702,7 @@ void SyntaxAnalyzerResult::collectUsedColumns(const ASTPtr & query) columns.emplace_back(ColumnSizeTuple{c->second.data_compressed, type_size, c->second.data_uncompressed, source_column.name}); } } - if (columns.size()) + if (!columns.empty()) required.insert(std::min_element(columns.begin(), columns.end())->name); else /// If we have no information about columns sizes, choose a column of minimum size of its data type. diff --git a/dbms/src/Interpreters/TranslateQualifiedNamesVisitor.h b/dbms/src/Interpreters/TranslateQualifiedNamesVisitor.h index c30836e7ca2..bcd032938aa 100644 --- a/dbms/src/Interpreters/TranslateQualifiedNamesVisitor.h +++ b/dbms/src/Interpreters/TranslateQualifiedNamesVisitor.h @@ -38,17 +38,17 @@ public: bool hasColumn(const String & name) const { return source_columns.count(name); } bool hasTable() const { return !tables.empty(); } bool processAsterisks() const { return hasTable() && has_columns; } - bool unknownColumn(size_t table_pos, const ASTIdentifier & node) const; + bool unknownColumn(size_t table_pos, const ASTIdentifier & identifier) const; }; static void visit(ASTPtr & ast, Data & data); static bool needChildVisit(ASTPtr & node, const ASTPtr & child); private: - static void visit(ASTIdentifier & node, ASTPtr & ast, Data &); + static void visit(ASTIdentifier & identifier, ASTPtr & ast, Data &); static void visit(const ASTQualifiedAsterisk & node, const ASTPtr & ast, Data &); - static void visit(ASTTableJoin & node, const ASTPtr & ast, Data &); - static void visit(ASTSelectQuery & node, const ASTPtr & ast, Data &); + static void visit(ASTTableJoin & join, const ASTPtr & ast, Data &); + static void visit(ASTSelectQuery & select, const ASTPtr & ast, Data &); static void visit(ASTExpressionList &, const ASTPtr &, Data &); static void visit(ASTFunction &, const ASTPtr &, Data &); diff --git a/dbms/src/Interpreters/evaluateConstantExpression.h b/dbms/src/Interpreters/evaluateConstantExpression.h index 422afd33d4b..f3ed1b730dc 100644 --- a/dbms/src/Interpreters/evaluateConstantExpression.h +++ b/dbms/src/Interpreters/evaluateConstantExpression.h @@ -50,6 +50,6 @@ ASTPtr evaluateConstantExpressionForDatabaseName(const ASTPtr & node, const Cont * or empty blocks if condition is always false, * or nothing if condition can't be folded to a set of constants. */ -std::optional evaluateExpressionOverConstantCondition(const ASTPtr & condition, const ExpressionActionsPtr & target_expr); +std::optional evaluateExpressionOverConstantCondition(const ASTPtr & node, const ExpressionActionsPtr & target_expr); } diff --git a/dbms/src/Interpreters/executeQuery.cpp b/dbms/src/Interpreters/executeQuery.cpp index ca5146d1bc1..a6b9a54b9ca 100644 --- a/dbms/src/Interpreters/executeQuery.cpp +++ b/dbms/src/Interpreters/executeQuery.cpp @@ -249,7 +249,7 @@ static std::tuple executeQueryImpl( BlockIO res; QueryPipeline & pipeline = res.pipeline; - String query_for_logging = ""; + String query_for_logging; try { @@ -598,7 +598,7 @@ void executeQuery( const char * end; /// If 'istr' is empty now, fetch next data into buffer. - if (istr.buffer().empty()) + if (istr.buffer().size() == 0) istr.next(); size_t max_query_size = context.getSettingsRef().max_query_size; diff --git a/dbms/src/Interpreters/interpretSubquery.cpp b/dbms/src/Interpreters/interpretSubquery.cpp index 95bedfbfdfd..811550cd6eb 100644 --- a/dbms/src/Interpreters/interpretSubquery.cpp +++ b/dbms/src/Interpreters/interpretSubquery.cpp @@ -63,7 +63,7 @@ std::shared_ptr interpretSubquery( subquery_settings.max_result_rows = 0; subquery_settings.max_result_bytes = 0; /// The calculation of `extremes` does not make sense and is not necessary (if you do it, then the `extremes` of the subquery can be taken instead of the whole query). - subquery_settings.extremes = 0; + subquery_settings.extremes = false; subquery_context.setSettings(subquery_settings); auto subquery_options = options.subquery(); diff --git a/dbms/src/Interpreters/sortBlock.cpp b/dbms/src/Interpreters/sortBlock.cpp index d00a9deaab2..7f5af84ab0b 100644 --- a/dbms/src/Interpreters/sortBlock.cpp +++ b/dbms/src/Interpreters/sortBlock.cpp @@ -48,13 +48,13 @@ struct PartialSortingLess bool operator() (size_t a, size_t b) const { - for (ColumnsWithSortDescriptions::const_iterator it = columns.begin(); it != columns.end(); ++it) + for (const auto & elem : columns) { int res; - if (it->column_const) + if (elem.column_const) res = 0; else - res = it->description.direction * it->column->compareAt(a, b, *it->column, it->description.nulls_direction); + res = elem.description.direction * elem.column->compareAt(a, b, *elem.column, elem.description.nulls_direction); if (res < 0) return true; else if (res > 0) @@ -76,22 +76,22 @@ struct PartialSortingLessWithCollation bool operator() (size_t a, size_t b) const { - for (ColumnsWithSortDescriptions::const_iterator it = columns.begin(); it != columns.end(); ++it) + for (const auto & elem : columns) { int res; - if (it->column_const) + if (elem.column_const) { res = 0; } - else if (isCollationRequired(it->description)) + else if (isCollationRequired(elem.description)) { - const ColumnString & column_string = assert_cast(*it->column); - res = column_string.compareAtWithCollation(a, b, *it->column, *it->description.collator); + const ColumnString & column_string = assert_cast(*elem.column); + res = column_string.compareAtWithCollation(a, b, *elem.column, *elem.description.collator); } else - res = it->column->compareAt(a, b, *it->column, it->description.nulls_direction); - res *= it->description.direction; + res = elem.column->compareAt(a, b, *elem.column, elem.description.nulls_direction); + res *= elem.description.direction; if (res < 0) return true; else if (res > 0) diff --git a/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp b/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp index e18c9197217..b3660674362 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp @@ -302,11 +302,8 @@ bool MergeTreeWhereOptimizer::isConstant(const ASTPtr & expr) const { const auto column_name = expr->getColumnName(); - if (expr->as() - || (block_with_constants.has(column_name) && isColumnConst(*block_with_constants.getByName(column_name).column))) - return true; - - return false; + return expr->as() + || (block_with_constants.has(column_name) && isColumnConst(*block_with_constants.getByName(column_name).column)); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.h b/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.h index bdfcaad6a51..6ec81592ee1 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.h +++ b/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.h @@ -31,7 +31,7 @@ public: SelectQueryInfo & query_info, const Context & context, const MergeTreeData & data, - const Names & queried_column_names_, + const Names & queried_columns_, Poco::Logger * log_); private: From 6320f59abb36633ba59ff04e509ed67f195a7fcf Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 9 Mar 2020 04:03:43 +0300 Subject: [PATCH 145/712] clang-tidy, part 11 --- dbms/src/Columns/ColumnAggregateFunction.cpp | 4 ++-- dbms/src/Columns/ColumnAggregateFunction.h | 2 +- dbms/src/Columns/ColumnArray.cpp | 2 +- dbms/src/Columns/ColumnDecimal.h | 2 +- dbms/src/Columns/ColumnLowCardinality.h | 2 +- dbms/src/IO/ReadBufferFromHDFS.cpp | 2 ++ dbms/src/IO/ReadBufferFromHDFS.h | 1 + dbms/src/Storages/AlterCommands.cpp | 6 +++--- dbms/src/Storages/ColumnDefault.h | 2 +- dbms/src/Storages/IStorage.cpp | 6 +++--- dbms/src/Storages/StorageDistributed.cpp | 8 ++++---- dbms/src/Storages/StorageFile.cpp | 8 ++++---- dbms/src/Storages/StorageFile.h | 2 +- dbms/src/Storages/StorageGenerateRandom.cpp | 2 +- 14 files changed, 26 insertions(+), 23 deletions(-) diff --git a/dbms/src/Columns/ColumnAggregateFunction.cpp b/dbms/src/Columns/ColumnAggregateFunction.cpp index 96dd789ffc3..507ff6c2db9 100644 --- a/dbms/src/Columns/ColumnAggregateFunction.cpp +++ b/dbms/src/Columns/ColumnAggregateFunction.cpp @@ -423,9 +423,9 @@ void ColumnAggregateFunction::insertDefault() pushBackAndCreateState(data, arena, func.get()); } -StringRef ColumnAggregateFunction::serializeValueIntoArena(size_t n, Arena & dst, const char *& begin) const +StringRef ColumnAggregateFunction::serializeValueIntoArena(size_t n, Arena & arena, const char *& begin) const { - WriteBufferFromArena out(dst, begin); + WriteBufferFromArena out(arena, begin); func->serialize(data[n], out); return out.finish(); } diff --git a/dbms/src/Columns/ColumnAggregateFunction.h b/dbms/src/Columns/ColumnAggregateFunction.h index 8e17a28cf4e..9a744b141f5 100644 --- a/dbms/src/Columns/ColumnAggregateFunction.h +++ b/dbms/src/Columns/ColumnAggregateFunction.h @@ -156,7 +156,7 @@ public: StringRef serializeValueIntoArena(size_t n, Arena & arena, char const *& begin) const override; - const char * deserializeAndInsertFromArena(const char * pos) override; + const char * deserializeAndInsertFromArena(const char * src_arena) override; void updateHashWithValue(size_t n, SipHash & hash) const override; diff --git a/dbms/src/Columns/ColumnArray.cpp b/dbms/src/Columns/ColumnArray.cpp index e0e5caef3bf..2c98e873cb6 100644 --- a/dbms/src/Columns/ColumnArray.cpp +++ b/dbms/src/Columns/ColumnArray.cpp @@ -331,7 +331,7 @@ bool ColumnArray::hasEqualOffsets(const ColumnArray & other) const const Offsets & offsets1 = getOffsets(); const Offsets & offsets2 = other.getOffsets(); return offsets1.size() == offsets2.size() - && (offsets1.size() == 0 || 0 == memcmp(offsets1.data(), offsets2.data(), sizeof(offsets1[0]) * offsets1.size())); + && (offsets1.empty() || 0 == memcmp(offsets1.data(), offsets2.data(), sizeof(offsets1[0]) * offsets1.size())); } diff --git a/dbms/src/Columns/ColumnDecimal.h b/dbms/src/Columns/ColumnDecimal.h index 57259176096..038e96016f1 100644 --- a/dbms/src/Columns/ColumnDecimal.h +++ b/dbms/src/Columns/ColumnDecimal.h @@ -94,7 +94,7 @@ public: void reserve(size_t n) override { data.reserve(n); } void insertFrom(const IColumn & src, size_t n) override { data.push_back(static_cast(src).getData()[n]); } - void insertData(const char * pos, size_t /*length*/) override; + void insertData(const char * src, size_t /*length*/) override; void insertDefault() override { data.push_back(T()); } virtual void insertManyDefaults(size_t length) override { data.resize_fill(data.size() + length); } void insert(const Field & x) override { data.push_back(DB::get>(x)); } diff --git a/dbms/src/Columns/ColumnLowCardinality.h b/dbms/src/Columns/ColumnLowCardinality.h index b690bf1463d..621fffb4a19 100644 --- a/dbms/src/Columns/ColumnLowCardinality.h +++ b/dbms/src/Columns/ColumnLowCardinality.h @@ -266,7 +266,7 @@ private: /// Dictionary may be shared for several mutable columns. /// Immutable columns may have the same column unique, which isn't necessarily shared dictionary. - void setShared(const ColumnPtr & dictionary); + void setShared(const ColumnPtr & column_unique_); bool isShared() const { return shared; } /// Create new dictionary with only keys that are mentioned in positions. diff --git a/dbms/src/IO/ReadBufferFromHDFS.cpp b/dbms/src/IO/ReadBufferFromHDFS.cpp index 5d9fafac86e..42419be3117 100644 --- a/dbms/src/IO/ReadBufferFromHDFS.cpp +++ b/dbms/src/IO/ReadBufferFromHDFS.cpp @@ -13,6 +13,8 @@ namespace ErrorCodes extern const int CANNOT_OPEN_FILE; } +ReadBufferFromHDFS::~ReadBufferFromHDFS() = default; + struct ReadBufferFromHDFS::ReadBufferFromHDFSImpl { std::string hdfs_uri; diff --git a/dbms/src/IO/ReadBufferFromHDFS.h b/dbms/src/IO/ReadBufferFromHDFS.h index f9b684718b7..e27159f2e98 100644 --- a/dbms/src/IO/ReadBufferFromHDFS.h +++ b/dbms/src/IO/ReadBufferFromHDFS.h @@ -20,6 +20,7 @@ class ReadBufferFromHDFS : public BufferWithOwnMemory public: ReadBufferFromHDFS(const std::string & hdfs_name_, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE); ReadBufferFromHDFS(ReadBufferFromHDFS &&) = default; + ~ReadBufferFromHDFS() override; bool nextImpl() override; }; diff --git a/dbms/src/Storages/AlterCommands.cpp b/dbms/src/Storages/AlterCommands.cpp index b6afe84c8c9..86edf8ce73c 100644 --- a/dbms/src/Storages/AlterCommands.cpp +++ b/dbms/src/Storages/AlterCommands.cpp @@ -89,7 +89,7 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ else if (command_ast->type == ASTAlterCommand::DROP_COLUMN && !command_ast->partition) { if (command_ast->clear_column) - throw Exception("\"ALTER TABLE table CLEAR COLUMN column\" queries are not supported yet. Use \"CLEAR COLUMN column IN PARTITION\".", ErrorCodes::NOT_IMPLEMENTED); + throw Exception(R"("ALTER TABLE table CLEAR COLUMN column" queries are not supported yet. Use "CLEAR COLUMN column IN PARTITION".)", ErrorCodes::NOT_IMPLEMENTED); AlterCommand command; command.ast = command_ast->clone(); @@ -189,7 +189,7 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ else if (command_ast->type == ASTAlterCommand::DROP_CONSTRAINT && !command_ast->partition) { if (command_ast->clear_column) - throw Exception("\"ALTER TABLE table CLEAR COLUMN column\" queries are not supported yet. Use \"CLEAR COLUMN column IN PARTITION\".", ErrorCodes::NOT_IMPLEMENTED); + throw Exception(R"("ALTER TABLE table CLEAR COLUMN column" queries are not supported yet. Use "CLEAR COLUMN column IN PARTITION".)", ErrorCodes::NOT_IMPLEMENTED); AlterCommand command; command.ast = command_ast->clone(); @@ -202,7 +202,7 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ else if (command_ast->type == ASTAlterCommand::DROP_INDEX && !command_ast->partition) { if (command_ast->clear_column) - throw Exception("\"ALTER TABLE table CLEAR INDEX index\" queries are not supported yet. Use \"CLEAR INDEX index IN PARTITION\".", ErrorCodes::NOT_IMPLEMENTED); + throw Exception(R"("ALTER TABLE table CLEAR INDEX index" queries are not supported yet. Use "CLEAR INDEX index IN PARTITION".)", ErrorCodes::NOT_IMPLEMENTED); AlterCommand command; command.ast = command_ast->clone(); diff --git a/dbms/src/Storages/ColumnDefault.h b/dbms/src/Storages/ColumnDefault.h index 4dcfa5da789..1035bfcc834 100644 --- a/dbms/src/Storages/ColumnDefault.h +++ b/dbms/src/Storages/ColumnDefault.h @@ -18,7 +18,7 @@ enum class ColumnDefaultKind ColumnDefaultKind columnDefaultKindFromString(const std::string & str); -std::string toString(const ColumnDefaultKind type); +std::string toString(const ColumnDefaultKind kind); struct ColumnDefault diff --git a/dbms/src/Storages/IStorage.cpp b/dbms/src/Storages/IStorage.cpp index 51ea13a8643..ef86bc3013d 100644 --- a/dbms/src/Storages/IStorage.cpp +++ b/dbms/src/Storages/IStorage.cpp @@ -278,10 +278,10 @@ void IStorage::check(const Block & block, bool need_all) const if (need_all && names_in_block.size() < columns_map.size()) { - for (auto it = available_columns.begin(); it != available_columns.end(); ++it) + for (const auto & available_column : available_columns) { - if (!names_in_block.count(it->name)) - throw Exception("Expected column " + it->name, ErrorCodes::NOT_FOUND_COLUMN_IN_BLOCK); + if (!names_in_block.count(available_column.name)) + throw Exception("Expected column " + available_column.name, ErrorCodes::NOT_FOUND_COLUMN_IN_BLOCK); } } } diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp index 8ed5c121011..3e6e9075c3d 100644 --- a/dbms/src/Storages/StorageDistributed.cpp +++ b/dbms/src/Storages/StorageDistributed.cpp @@ -53,8 +53,8 @@ namespace { -static const UInt64 FORCE_OPTIMIZE_SKIP_UNUSED_SHARDS_HAS_SHARDING_KEY = 1; -static const UInt64 FORCE_OPTIMIZE_SKIP_UNUSED_SHARDS_ALWAYS = 2; +const UInt64 FORCE_OPTIMIZE_SKIP_UNUSED_SHARDS_HAS_SHARDING_KEY = 1; +const UInt64 FORCE_OPTIMIZE_SKIP_UNUSED_SHARDS_ALWAYS = 2; } namespace DB @@ -663,8 +663,8 @@ void StorageDistributed::flushClusterNodesAllData() std::lock_guard lock(cluster_nodes_mutex); /// TODO: Maybe it should be executed in parallel - for (auto it = cluster_nodes_data.begin(); it != cluster_nodes_data.end(); ++it) - it->second.flushAllData(); + for (auto & node : cluster_nodes_data) + node.second.flushAllData(); } void StorageDistributed::rename(const String & new_path_to_table_data, const String & new_database_name, const String & new_table_name, diff --git a/dbms/src/Storages/StorageFile.cpp b/dbms/src/Storages/StorageFile.cpp index 95958b99bc6..92d9c3ccfe2 100644 --- a/dbms/src/Storages/StorageFile.cpp +++ b/dbms/src/Storages/StorageFile.cpp @@ -60,7 +60,7 @@ namespace /* Recursive directory listing with matched paths as a result. * Have the same method in StorageHDFS. */ -static std::vector listFilesWithRegexpMatching(const std::string & path_for_ls, const std::string & for_match) +std::vector listFilesWithRegexpMatching(const std::string & path_for_ls, const std::string & for_match) { const size_t first_glob = for_match.find_first_of("*?{"); @@ -105,13 +105,13 @@ static std::vector listFilesWithRegexpMatching(const std::string & return result; } -static std::string getTablePath(const std::string & table_dir_path, const std::string & format_name) +std::string getTablePath(const std::string & table_dir_path, const std::string & format_name) { return table_dir_path + "/data." + escapeForFileName(format_name); } /// Both db_dir_path and table_path must be converted to absolute paths (in particular, path cannot contain '..'). -static void checkCreationIsAllowed(const Context & context_global, const std::string & db_dir_path, const std::string & table_path) +void checkCreationIsAllowed(const Context & context_global, const std::string & db_dir_path, const std::string & table_path) { if (context_global.getApplicationType() != Context::ApplicationType::SERVER) return; @@ -546,7 +546,7 @@ void registerStorageFile(StorageFactory & factory) { ASTs & engine_args = args.engine_args; - if (!(engine_args.size() >= 1 && engine_args.size() <= 3)) + if (!(engine_args.size() >= 1 && engine_args.size() <= 3)) // NOLINT throw Exception( "Storage File requires from 1 to 3 arguments: name of used format, source and compression_method.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); diff --git a/dbms/src/Storages/StorageFile.h b/dbms/src/Storages/StorageFile.h index ec24653af62..6134dff5db5 100644 --- a/dbms/src/Storages/StorageFile.h +++ b/dbms/src/Storages/StorageFile.h @@ -60,7 +60,7 @@ protected: StorageFile(int table_fd_, CommonArguments args); /// From user's file - StorageFile(const std::string & table_path_, const std::string & user_files_absolute_path, CommonArguments args); + StorageFile(const std::string & table_path_, const std::string & user_files_path, CommonArguments args); /// From table in database StorageFile(const std::string & relative_table_dir_path, CommonArguments args); diff --git a/dbms/src/Storages/StorageGenerateRandom.cpp b/dbms/src/Storages/StorageGenerateRandom.cpp index 12c5d1ac09f..f2a67210f3f 100644 --- a/dbms/src/Storages/StorageGenerateRandom.cpp +++ b/dbms/src/Storages/StorageGenerateRandom.cpp @@ -385,7 +385,7 @@ void registerStorageGenerateRandom(StorageFactory & factory) UInt64 max_string_length = 10; UInt64 max_array_length = 10; - if (engine_args.size() >= 1) + if (!engine_args.empty()) { const Field & value = engine_args[0]->as().value; if (!value.isNull()) From cde492a7848fa766bddb116cd6ef916f3699dfd3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 9 Mar 2020 04:22:33 +0300 Subject: [PATCH 146/712] clang-tidy, part 12 --- .../Storages/Distributed/DirectoryMonitor.cpp | 4 ++-- .../Storages/MergeTree/DataPartsExchange.cpp | 4 ++-- .../Storages/MergeTree/IMergeTreeDataPart.h | 4 ++-- dbms/src/Storages/StorageLog.cpp | 4 ++-- dbms/src/Storages/StorageLogSettings.cpp | 8 +++---- dbms/src/Storages/StorageMergeTree.cpp | 6 ++--- dbms/src/Storages/StorageMySQL.cpp | 4 ++-- .../Storages/StorageReplicatedMergeTree.cpp | 24 +++++++------------ .../src/Storages/StorageReplicatedMergeTree.h | 4 ++-- dbms/src/Storages/StorageS3.cpp | 4 ++-- dbms/src/Storages/VirtualColumnUtils.cpp | 8 +++---- .../transformQueryForExternalDatabase.cpp | 12 +++------- 12 files changed, 36 insertions(+), 50 deletions(-) diff --git a/dbms/src/Storages/Distributed/DirectoryMonitor.cpp b/dbms/src/Storages/Distributed/DirectoryMonitor.cpp index 912973f1d7d..098a06e43fc 100644 --- a/dbms/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/dbms/src/Storages/Distributed/DirectoryMonitor.cpp @@ -46,7 +46,7 @@ namespace ErrorCodes namespace { - static constexpr const std::chrono::minutes decrease_error_count_period{5}; + constexpr const std::chrono::minutes decrease_error_count_period{5}; template ConnectionPoolPtrs createPoolsForAddresses(const std::string & name, PoolFactory && factory) @@ -238,7 +238,7 @@ bool StorageDistributedDirectoryMonitor::processFiles() const auto & file_path_str = it->path(); Poco::Path file_path{file_path_str}; - if (!it->isDirectory() && startsWith(file_path.getExtension().data(), "bin")) + if (!it->isDirectory() && startsWith(file_path.getExtension(), "bin")) files[parse(file_path.getBaseName())] = file_path_str; } diff --git a/dbms/src/Storages/MergeTree/DataPartsExchange.cpp b/dbms/src/Storages/MergeTree/DataPartsExchange.cpp index 11fa64739d3..dc2678c92c2 100644 --- a/dbms/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/dbms/src/Storages/MergeTree/DataPartsExchange.cpp @@ -33,8 +33,8 @@ namespace DataPartsExchange namespace { -static constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_SIZE = 1; -static constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_SIZE_AND_TTL_INFOS = 2; +constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_SIZE = 1; +constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_SIZE_AND_TTL_INFOS = 2; std::string getEndpointId(const std::string & node_id) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h index 61b5ba43165..8af45f477f4 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -268,8 +268,8 @@ public: { } - void load(const MergeTreeData & storage, const String & part_path); - void store(const MergeTreeData & storage, const String & part_path, Checksums & checksums) const; + void load(const MergeTreeData & data, const String & part_path); + void store(const MergeTreeData & data, const String & part_path, Checksums & checksums) const; void store(const Names & column_names, const DataTypes & data_types, const String & part_path, Checksums & checksums) const; void update(const Block & block, const Names & column_names); diff --git a/dbms/src/Storages/StorageLog.cpp b/dbms/src/Storages/StorageLog.cpp index 20def7d1b1d..d908b9d459f 100644 --- a/dbms/src/Storages/StorageLog.cpp +++ b/dbms/src/Storages/StorageLog.cpp @@ -500,12 +500,12 @@ void StorageLog::loadMarks() std::unique_ptr marks_rb = disk->readFile(marks_file_path, 32768); while (!marks_rb->eof()) { - for (size_t i = 0; i < files_by_index.size(); ++i) + for (auto & file : files_by_index) { Mark mark; readIntBinary(mark.rows, *marks_rb); readIntBinary(mark.offset, *marks_rb); - files_by_index[i]->second.marks.push_back(mark); + file->second.marks.push_back(mark); } } } diff --git a/dbms/src/Storages/StorageLogSettings.cpp b/dbms/src/Storages/StorageLogSettings.cpp index 19cd0dd34e3..900e1070eac 100644 --- a/dbms/src/Storages/StorageLogSettings.cpp +++ b/dbms/src/Storages/StorageLogSettings.cpp @@ -9,11 +9,9 @@ String getDiskName(ASTStorage & storage_def) if (storage_def.settings) { SettingsChanges changes = storage_def.settings->changes; - for (auto it = changes.begin(); it != changes.end(); ++it) - { - if (it->name == "disk") - return it->value.safeGet(); - } + for (const auto & change : changes) + if (change.name == "disk") + return change.value.safeGet(); } return "default"; } diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index fdd4e3c6f00..7317e6baddf 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -253,7 +253,7 @@ std::vector StorageMergeTree::prepar } void StorageMergeTree::alter( - const AlterCommands & params, + const AlterCommands & commands, const Context & context, TableStructureWriteLockHolder & table_lock_holder) { @@ -263,7 +263,7 @@ void StorageMergeTree::alter( StorageInMemoryMetadata metadata = getInMemoryMetadata(); - params.apply(metadata); + commands.apply(metadata); /// Update metdata in memory auto update_metadata = [&metadata, &table_lock_holder, this]() @@ -277,7 +277,7 @@ void StorageMergeTree::alter( }; /// This alter can be performed at metadata level only - if (!params.isModifyingData()) + if (!commands.isModifyingData()) { lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); diff --git a/dbms/src/Storages/StorageMySQL.cpp b/dbms/src/Storages/StorageMySQL.cpp index 2e3272b2159..1991215ad12 100644 --- a/dbms/src/Storages/StorageMySQL.cpp +++ b/dbms/src/Storages/StorageMySQL.cpp @@ -214,8 +214,8 @@ void registerStorageMySQL(StorageFactory & factory) "Storage MySQL requires 5-7 parameters: MySQL('host:port', database, table, 'user', 'password'[, replace_query, 'on_duplicate_clause']).", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - for (size_t i = 0; i < engine_args.size(); ++i) - engine_args[i] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[i], args.local_context); + for (auto & engine_arg : engine_args) + engine_arg = evaluateConstantExpressionOrIdentifierAsLiteral(engine_arg, args.local_context); /// 3306 is the default MySQL port. auto parsed_host_port = parseAddress(engine_args[0]->as().value.safeGet(), 3306); diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 48ba57d0c72..b7f3385185c 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -1308,10 +1308,10 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) Coordination::Requests ops; - for (size_t i = 0, size = replicas.size(); i < size; ++i) + for (const auto & path_part : replicas) { Coordination::Stat stat; - String path = zookeeper_path + "/replicas/" + replicas[i] + "/host"; + String path = zookeeper_path + "/replicas/" + path_part + "/host"; zookeeper->get(path, &stat); ops.emplace_back(zkutil::makeCheckRequest(path, stat.version)); } @@ -4079,7 +4079,6 @@ void StorageReplicatedMergeTree::sendRequestToLeaderReplica(const ASTPtr & query NullBlockOutputStream output({}); copyData(stream, output); - return; } @@ -4089,14 +4088,13 @@ std::optional StorageReplicatedMergeTree::findClusterAddress(c { const auto & shards = iter.second->getShardsAddresses(); - for (size_t shard_num = 0; shard_num < shards.size(); ++shard_num) + for (const auto & shard : shards) { - for (size_t replica_num = 0; replica_num < shards[shard_num].size(); ++replica_num) + for (const auto & replica : shard) { - const Cluster::Address & address = shards[shard_num][replica_num]; /// user is actually specified, not default - if (address.host_name == leader_address.host && address.port == leader_address.queries_port && address.user_specified) - return address; + if (replica.host_name == leader_address.host && replica.port == leader_address.queries_port && replica.user_specified) + return replica; } } } @@ -4873,14 +4871,12 @@ void StorageReplicatedMergeTree::replacePartitionFrom(const StoragePtr & source_ } } - for (size_t i = 0; i < src_all_parts.size(); ++i) + for (const auto & src_part : src_all_parts) { /// We also make some kind of deduplication to avoid duplicated parts in case of ATTACH PARTITION /// Assume that merges in the partition are quite rare /// Save deduplication block ids with special prefix replace_partition - auto & src_part = src_all_parts[i]; - if (!canReplacePartition(src_part)) throw Exception( "Cannot replace partition '" + partition_id + "' because part '" + src_part->name + "' has inconsistent granularity with table", @@ -5054,17 +5050,15 @@ void StorageReplicatedMergeTree::movePartitionToTable(const StoragePtr & dest_ta /// Clone parts into destination table. - for (size_t i = 0; i < src_all_parts.size(); ++i) + for (const auto & src_part : src_all_parts) { - auto & src_part = src_all_parts[i]; - if (!dest_table_storage->canReplacePartition(src_part)) throw Exception( "Cannot move partition '" + partition_id + "' because part '" + src_part->name + "' has inconsistent granularity with table", ErrorCodes::LOGICAL_ERROR); String hash_hex = src_part->checksums.getTotalChecksumHex(); - String block_id_path = ""; + String block_id_path; auto lock = dest_table_storage->allocateBlockNumber(partition_id, zookeeper, block_id_path); if (!lock) diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.h b/dbms/src/Storages/StorageReplicatedMergeTree.h index 182c2a3a249..928938ccfc6 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.h +++ b/dbms/src/Storages/StorageReplicatedMergeTree.h @@ -106,7 +106,7 @@ public: void alterPartition(const ASTPtr & query, const PartitionCommands & commands, const Context & query_context) override; void mutate(const MutationCommands & commands, const Context & context) override; - void waitMutation(const String & znode_name, size_t mutation_sync) const; + void waitMutation(const String & znode_name, size_t mutations_sync) const; std::vector getMutationsStatus() const override; CancellationCode killMutation(const String & mutation_id) override; @@ -522,7 +522,7 @@ private: void dropPartition(const ASTPtr & query, const ASTPtr & partition, bool detach, const Context & query_context); void attachPartition(const ASTPtr & partition, bool part, const Context & query_context); void replacePartitionFrom(const StoragePtr & source_table, const ASTPtr & partition, bool replace, const Context & query_context); - void movePartitionToTable(const StoragePtr & source_table, const ASTPtr & partition, const Context & query_context); + void movePartitionToTable(const StoragePtr & dest_table, const ASTPtr & partition, const Context & query_context); void fetchPartition(const ASTPtr & partition, const String & from, const Context & query_context); /// Check granularity of already existing replicated table in zookeeper if it exists diff --git a/dbms/src/Storages/StorageS3.cpp b/dbms/src/Storages/StorageS3.cpp index 128fbafeffa..962150ebc29 100644 --- a/dbms/src/Storages/StorageS3.cpp +++ b/dbms/src/Storages/StorageS3.cpp @@ -321,8 +321,8 @@ void registerStorageS3(StorageFactory & factory) throw Exception( "Storage S3 requires 2 to 5 arguments: url, [access_key_id, secret_access_key], name of used format and [compression_method].", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - for (size_t i = 0; i < engine_args.size(); ++i) - engine_args[i] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[i], args.local_context); + for (auto & engine_arg : engine_args) + engine_arg = evaluateConstantExpressionOrIdentifierAsLiteral(engine_arg, args.local_context); String url = engine_args[0]->as().value.safeGet(); Poco::URI uri (url); diff --git a/dbms/src/Storages/VirtualColumnUtils.cpp b/dbms/src/Storages/VirtualColumnUtils.cpp index b22b8b02746..2e1ce33951a 100644 --- a/dbms/src/Storages/VirtualColumnUtils.cpp +++ b/dbms/src/Storages/VirtualColumnUtils.cpp @@ -29,8 +29,8 @@ namespace /// Verifying that the function depends only on the specified columns bool isValidFunction(const ASTPtr & expression, const NameSet & columns) { - for (size_t i = 0; i < expression->children.size(); ++i) - if (!isValidFunction(expression->children[i], columns)) + for (const auto & child : expression->children) + if (!isValidFunction(child, columns)) return false; if (auto opt_name = IdentifierSemantic::getColumnName(expression)) @@ -45,8 +45,8 @@ void extractFunctions(const ASTPtr & expression, const NameSet & columns, std::v const auto * function = expression->as(); if (function && function->name == "and") { - for (size_t i = 0; i < function->arguments->children.size(); ++i) - extractFunctions(function->arguments->children[i], columns, result); + for (const auto & child : function->arguments->children) + extractFunctions(child, columns, result); } else if (isValidFunction(expression, columns)) { diff --git a/dbms/src/Storages/transformQueryForExternalDatabase.cpp b/dbms/src/Storages/transformQueryForExternalDatabase.cpp index d5d2ebb0e63..f5ab7ca9026 100644 --- a/dbms/src/Storages/transformQueryForExternalDatabase.cpp +++ b/dbms/src/Storages/transformQueryForExternalDatabase.cpp @@ -114,7 +114,7 @@ bool isCompatible(const IAST & node) return false; for (const auto & expr : function->arguments->children) - if (!isCompatible(*expr.get())) + if (!isCompatible(*expr)) return false; return true; @@ -123,16 +123,10 @@ bool isCompatible(const IAST & node) if (const auto * literal = node.as()) { /// Foreign databases often have no support for Array. But Tuple literals are passed to support IN clause. - if (literal->value.getType() == Field::Types::Array) - return false; - - return true; + return literal->value.getType() != Field::Types::Array; } - if (node.as()) - return true; - - return false; + return node.as(); } } From 7334c13de9bd5e342eac03368df69280fdebef4b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 9 Mar 2020 04:50:33 +0300 Subject: [PATCH 147/712] clang-tidy, part 13 --- .../tests/date_lut_default_timezone.cpp | 2 +- base/common/tests/gtest_find_symbols.cpp | 2 +- base/common/tests/gtest_json_test.cpp | 946 +++++++++--------- base/mysqlxx/src/tests/mysqlxx_test.cpp | 12 +- .../MergeTree/IMergeTreeDataPartWriter.cpp | 12 +- dbms/src/Storages/MergeTree/KeyCondition.cpp | 11 +- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 13 +- dbms/src/Storages/MergeTree/MergeTreeData.h | 10 +- .../MergeTree/MergeTreeDataMergerMutator.cpp | 3 +- .../MergeTree/MergeTreeDataMergerMutator.h | 4 +- .../MergeTree/MergeTreeDataPartChecksum.h | 4 +- .../MergeTree/MergeTreeDataPartTTLInfo.cpp | 4 +- 12 files changed, 506 insertions(+), 517 deletions(-) diff --git a/base/common/tests/date_lut_default_timezone.cpp b/base/common/tests/date_lut_default_timezone.cpp index e0251707537..aeefae3c9e0 100644 --- a/base/common/tests/date_lut_default_timezone.cpp +++ b/base/common/tests/date_lut_default_timezone.cpp @@ -8,7 +8,7 @@ int main(int argc, char ** argv) { const auto & date_lut = DateLUT::instance(); std::cout << "Detected default timezone: `" << date_lut.getTimeZone() << "'" << std::endl; - time_t now = time(NULL); + time_t now = time(nullptr); std::cout << "Current time: " << date_lut.timeToString(now) << ", UTC: " << DateLUT::instance("UTC").timeToString(now) << std::endl; } diff --git a/base/common/tests/gtest_find_symbols.cpp b/base/common/tests/gtest_find_symbols.cpp index 118a1f5c178..f18b4df832a 100644 --- a/base/common/tests/gtest_find_symbols.cpp +++ b/base/common/tests/gtest_find_symbols.cpp @@ -3,7 +3,7 @@ #include -TEST(find_symbols, SimpleTest) +TEST(FindSymbols, SimpleTest) { std::string s = "Hello, world! Goodbye..."; const char * begin = s.data(); diff --git a/base/common/tests/gtest_json_test.cpp b/base/common/tests/gtest_json_test.cpp index 79a55dcd1ab..d0393a850aa 100644 --- a/base/common/tests/gtest_json_test.cpp +++ b/base/common/tests/gtest_json_test.cpp @@ -26,480 +26,480 @@ TEST(JSON_Suite, SimpleTest) { std::vector test_data = { - { "\"name\""s, ResultType::Return, "name"s }, - { "\"Вафельница Vitek WX-1102 FL\""s, ResultType::Return, "Вафельница Vitek WX-1102 FL"s }, - { "\"brand\""s, ResultType::Return, "brand"s }, - { "\"184509\""s, ResultType::Return, "184509"s }, - { "\"category\""s, ResultType::Return, "category"s }, - { "\"Все для детей/Детская техника/Vitek\""s, ResultType::Return, "Все для детей/Детская техника/Vitek"s }, - { "\"variant\""s, ResultType::Return, "variant"s }, - { "\"В наличии\""s, ResultType::Return, "В наличии"s }, - { "\"price\""s, ResultType::Return, "price"s }, - { "\"2390.00\""s, ResultType::Return, "2390.00"s }, - { "\"list\""s, ResultType::Return, "list"s }, - { "\"Карточка\""s, ResultType::Return, "Карточка"s }, - { "\"position\""s, ResultType::Return, "position"s }, - { "\"detail\""s, ResultType::Return, "detail"s }, - { "\"actionField\""s, ResultType::Return, "actionField"s }, - { "\"list\""s, ResultType::Return, "list"s }, - { "\"http://www.techport.ru/q/?t=вафельница&sort=price&sdim=asc\""s, ResultType::Return, "http://www.techport.ru/q/?t=вафельница&sort=price&sdim=asc"s }, - { "\"action\""s, ResultType::Return, "action"s }, - { "\"detail\""s, ResultType::Return, "detail"s }, - { "\"products\""s, ResultType::Return, "products"s }, - { "\"name\""s, ResultType::Return, "name"s }, - { "\"Вафельница Vitek WX-1102 FL\""s, ResultType::Return, "Вафельница Vitek WX-1102 FL"s }, - { "\"id\""s, ResultType::Return, "id"s }, - { "\"184509\""s, ResultType::Return, "184509"s }, - { "\"price\""s, ResultType::Return, "price"s }, - { "\"2390.00\""s, ResultType::Return, "2390.00"s }, - { "\"brand\""s, ResultType::Return, "brand"s }, - { "\"Vitek\""s, ResultType::Return, "Vitek"s }, - { "\"category\""s, ResultType::Return, "category"s }, - { "\"Все для детей/Детская техника/Vitek\""s, ResultType::Return, "Все для детей/Детская техника/Vitek"s }, - { "\"variant\""s, ResultType::Return, "variant"s }, - { "\"В наличии\""s, ResultType::Return, "В наличии"s }, - { "\"ru\""s, ResultType::Return, "ru"s }, - { "\"experiments\""s, ResultType::Return, "experiments"s }, - { "\"lang\""s, ResultType::Return, "lang"s }, - { "\"ru\""s, ResultType::Return, "ru"s }, - { "\"los_portal\""s, ResultType::Return, "los_portal"s }, - { "\"los_level\""s, ResultType::Return, "los_level"s }, - { "\"none\""s, ResultType::Return, "none"s }, - { "\"isAuthorized\""s, ResultType::Return, "isAuthorized"s }, - { "\"isSubscriber\""s, ResultType::Return, "isSubscriber"s }, - { "\"postType\""s, ResultType::Return, "postType"s }, - { "\"Новости\""s, ResultType::Return, "Новости"s }, - { "\"experiments\""s, ResultType::Return, "experiments"s }, - { "\"lang\""s, ResultType::Return, "lang"s }, - { "\"ru\""s, ResultType::Return, "ru"s }, - { "\"los_portal\""s, ResultType::Return, "los_portal"s }, - { "\"los_level\""s, ResultType::Return, "los_level"s }, - { "\"none\""s, ResultType::Return, "none"s }, - { "\"lang\""s, ResultType::Return, "lang"s }, - { "\"ru\""s, ResultType::Return, "ru"s }, - { "\"Электроплита GEFEST Брест ЭПНД 5140-01 0001\""s, ResultType::Return, "Электроплита GEFEST Брест ЭПНД 5140-01 0001"s }, - { "\"price\""s, ResultType::Return, "price"s }, - { "\"currencyCode\""s, ResultType::Return, "currencyCode"s }, - { "\"RUB\""s, ResultType::Return, "RUB"s }, - { "\"lang\""s, ResultType::Return, "lang"s }, - { "\"ru\""s, ResultType::Return, "ru"s }, - { "\"experiments\""s, ResultType::Return, "experiments"s }, - { "\"lang\""s, ResultType::Return, "lang"s }, - { "\"ru\""s, ResultType::Return, "ru"s }, - { "\"los_portal\""s, ResultType::Return, "los_portal"s }, - { "\"los_level\""s, ResultType::Return, "los_level"s }, - { "\"none\""s, ResultType::Return, "none"s }, - { "\"trash_login\""s, ResultType::Return, "trash_login"s }, - { "\"novikoff\""s, ResultType::Return, "novikoff"s }, - { "\"trash_cat_link\""s, ResultType::Return, "trash_cat_link"s }, - { "\"progs\""s, ResultType::Return, "progs"s }, - { "\"trash_parent_link\""s, ResultType::Return, "trash_parent_link"s }, - { "\"content\""s, ResultType::Return, "content"s }, - { "\"trash_posted_parent\""s, ResultType::Return, "trash_posted_parent"s }, - { "\"content.01.2016\""s, ResultType::Return, "content.01.2016"s }, - { "\"trash_posted_cat\""s, ResultType::Return, "trash_posted_cat"s }, - { "\"progs.01.2016\""s, ResultType::Return, "progs.01.2016"s }, - { "\"trash_virus_count\""s, ResultType::Return, "trash_virus_count"s }, - { "\"trash_is_android\""s, ResultType::Return, "trash_is_android"s }, - { "\"trash_is_wp8\""s, ResultType::Return, "trash_is_wp8"s }, - { "\"trash_is_ios\""s, ResultType::Return, "trash_is_ios"s }, - { "\"trash_posted\""s, ResultType::Return, "trash_posted"s }, - { "\"01.2016\""s, ResultType::Return, "01.2016"s }, - { "\"experiments\""s, ResultType::Return, "experiments"s }, - { "\"lang\""s, ResultType::Return, "lang"s }, - { "\"ru\""s, ResultType::Return, "ru"s }, - { "\"los_portal\""s, ResultType::Return, "los_portal"s }, - { "\"los_level\""s, ResultType::Return, "los_level"s }, - { "\"none\""s, ResultType::Return, "none"s }, - { "\"merchantId\""s, ResultType::Return, "merchantId"s }, - { "\"13694_49246\""s, ResultType::Return, "13694_49246"s }, - { "\"cps-source\""s, ResultType::Return, "cps-source"s }, - { "\"wargaming\""s, ResultType::Return, "wargaming"s }, - { "\"cps_provider\""s, ResultType::Return, "cps_provider"s }, - { "\"default\""s, ResultType::Return, "default"s }, - { "\"errorReason\""s, ResultType::Return, "errorReason"s }, - { "\"no errors\""s, ResultType::Return, "no errors"s }, - { "\"scid\""s, ResultType::Return, "scid"s }, - { "\"isAuthPayment\""s, ResultType::Return, "isAuthPayment"s }, - { "\"lang\""s, ResultType::Return, "lang"s }, - { "\"ru\""s, ResultType::Return, "ru"s }, - { "\"rubric\""s, ResultType::Return, "rubric"s }, - { "\"\""s, ResultType::Return, ""s }, - { "\"rubric\""s, ResultType::Return, "rubric"s }, - { "\"Мир\""s, ResultType::Return, "Мир"s }, - { "\"lang\""s, ResultType::Return, "lang"s }, - { "\"ru\""s, ResultType::Return, "ru"s }, - { "\"experiments\""s, ResultType::Return, "experiments"s }, - { "\"lang\""s, ResultType::Return, "lang"s }, - { "\"ru\""s, ResultType::Return, "ru"s }, - { "\"los_portal\""s, ResultType::Return, "los_portal"s }, - { "\"los_level\""s, ResultType::Return, "los_level"s }, - { "\"none\""s, ResultType::Return, "none"s }, - { "\"lang\""s, ResultType::Return, "lang"s }, - { "\"ru\""s, ResultType::Return, "ru"s }, - { "\"__ym\""s, ResultType::Return, "__ym"s }, - { "\"ecommerce\""s, ResultType::Return, "ecommerce"s }, - { "\"impressions\""s, ResultType::Return, "impressions"s }, - { "\"id\""s, ResultType::Return, "id"s }, - { "\"863813\""s, ResultType::Return, "863813"s }, - { "\"name\""s, ResultType::Return, "name"s }, - { "\"Футболка детская 3D Happy, возраст 1-2 года, трикотаж\""s, ResultType::Return, "Футболка детская 3D Happy, возраст 1-2 года, трикотаж"s }, - { "\"category\""s, ResultType::Return, "category"s }, - { "\"/Летние товары/Летний текстиль/\""s, ResultType::Return, "/Летние товары/Летний текстиль/"s }, - { "\"variant\""s, ResultType::Return, "variant"s }, - { "\"\""s, ResultType::Return, ""s }, - { "\"price\""s, ResultType::Return, "price"s }, - { "\"390.00\""s, ResultType::Return, "390.00"s }, - { "\"list\""s, ResultType::Return, "list"s }, - { "\"/retailrocket/\""s, ResultType::Return, "/retailrocket/"s }, - { "\"position\""s, ResultType::Return, "position"s }, - { "\"brand\""s, ResultType::Return, "brand"s }, - { "\"/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/\""s, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"s }, - { "\"id\""s, ResultType::Return, "id"s }, - { "\"863839\""s, ResultType::Return, "863839"s }, - { "\"name\""s, ResultType::Return, "name"s }, - { "\"Футболка детская 3D Pretty kitten, возраст 1-2 года, трикотаж\""s, ResultType::Return, "Футболка детская 3D Pretty kitten, возраст 1-2 года, трикотаж"s }, - { "\"category\""s, ResultType::Return, "category"s }, - { "\"/Летние товары/Летний текстиль/\""s, ResultType::Return, "/Летние товары/Летний текстиль/"s }, - { "\"variant\""s, ResultType::Return, "variant"s }, - { "\"\""s, ResultType::Return, ""s }, - { "\"price\""s, ResultType::Return, "price"s }, - { "\"390.00\""s, ResultType::Return, "390.00"s }, - { "\"list\""s, ResultType::Return, "list"s }, - { "\"/retailrocket/\""s, ResultType::Return, "/retailrocket/"s }, - { "\"position\""s, ResultType::Return, "position"s }, - { "\"brand\""s, ResultType::Return, "brand"s }, - { "\"/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/\""s, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"s }, - { "\"id\""s, ResultType::Return, "id"s }, - { "\"863847\""s, ResultType::Return, "863847"s }, - { "\"name\""s, ResultType::Return, "name"s }, - { "\"Футболка детская 3D Little tiger, возраст 1-2 года, трикотаж\""s, ResultType::Return, "Футболка детская 3D Little tiger, возраст 1-2 года, трикотаж"s }, - { "\"category\""s, ResultType::Return, "category"s }, - { "\"/Летние товары/Летний текстиль/\""s, ResultType::Return, "/Летние товары/Летний текстиль/"s }, - { "\"variant\""s, ResultType::Return, "variant"s }, - { "\"\""s, ResultType::Return, ""s }, - { "\"price\""s, ResultType::Return, "price"s }, - { "\"390.00\""s, ResultType::Return, "390.00"s }, - { "\"list\""s, ResultType::Return, "list"s }, - { "\"/retailrocket/\""s, ResultType::Return, "/retailrocket/"s }, - { "\"position\""s, ResultType::Return, "position"s }, - { "\"brand\""s, ResultType::Return, "brand"s }, - { "\"/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/\""s, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"s }, - { "\"id\""s, ResultType::Return, "id"s }, - { "\"911480\""s, ResultType::Return, "911480"s }, - { "\"name\""s, ResultType::Return, "name"s }, - { "\"Футболка детская 3D Puppy, возраст 1-2 года, трикотаж\""s, ResultType::Return, "Футболка детская 3D Puppy, возраст 1-2 года, трикотаж"s }, - { "\"category\""s, ResultType::Return, "category"s }, - { "\"/Летние товары/Летний текстиль/\""s, ResultType::Return, "/Летние товары/Летний текстиль/"s }, - { "\"variant\""s, ResultType::Return, "variant"s }, - { "\"\""s, ResultType::Return, ""s }, - { "\"price\""s, ResultType::Return, "price"s }, - { "\"390.00\""s, ResultType::Return, "390.00"s }, - { "\"list\""s, ResultType::Return, "list"s }, - { "\"/retailrocket/\""s, ResultType::Return, "/retailrocket/"s }, - { "\"position\""s, ResultType::Return, "position"s }, - { "\"brand\""s, ResultType::Return, "brand"s }, - { "\"/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/\""s, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"s }, - { "\"id\""s, ResultType::Return, "id"s }, - { "\"911484\""s, ResultType::Return, "911484"s }, - { "\"name\""s, ResultType::Return, "name"s }, - { "\"Футболка детская 3D Little bears, возраст 1-2 года, трикотаж\""s, ResultType::Return, "Футболка детская 3D Little bears, возраст 1-2 года, трикотаж"s }, - { "\"category\""s, ResultType::Return, "category"s }, - { "\"/Летние товары/Летний текстиль/\""s, ResultType::Return, "/Летние товары/Летний текстиль/"s }, - { "\"variant\""s, ResultType::Return, "variant"s }, - { "\"\""s, ResultType::Return, ""s }, - { "\"price\""s, ResultType::Return, "price"s }, - { "\"390.00\""s, ResultType::Return, "390.00"s }, - { "\"list\""s, ResultType::Return, "list"s }, - { "\"/retailrocket/\""s, ResultType::Return, "/retailrocket/"s }, - { "\"position\""s, ResultType::Return, "position"s }, - { "\"brand\""s, ResultType::Return, "brand"s }, - { "\"/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/\""s, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"s }, - { "\"id\""s, ResultType::Return, "id"s }, - { "\"911489\""s, ResultType::Return, "911489"s }, - { "\"name\""s, ResultType::Return, "name"s }, - { "\"Футболка детская 3D Dolphin, возраст 2-4 года, трикотаж\""s, ResultType::Return, "Футболка детская 3D Dolphin, возраст 2-4 года, трикотаж"s }, - { "\"category\""s, ResultType::Return, "category"s }, - { "\"/Летние товары/Летний текстиль/\""s, ResultType::Return, "/Летние товары/Летний текстиль/"s }, - { "\"variant\""s, ResultType::Return, "variant"s }, - { "\"\""s, ResultType::Return, ""s }, - { "\"price\""s, ResultType::Return, "price"s }, - { "\"390.00\""s, ResultType::Return, "390.00"s }, - { "\"list\""s, ResultType::Return, "list"s }, - { "\"/retailrocket/\""s, ResultType::Return, "/retailrocket/"s }, - { "\"position\""s, ResultType::Return, "position"s }, - { "\"brand\""s, ResultType::Return, "brand"s }, - { "\"/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/\""s, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"s }, - { "\"id\""s, ResultType::Return, "id"s }, - { "\"911496\""s, ResultType::Return, "911496"s }, - { "\"name\""s, ResultType::Return, "name"s }, - { "\"Футболка детская 3D Pretty, возраст 1-2 года, трикотаж\""s, ResultType::Return, "Футболка детская 3D Pretty, возраст 1-2 года, трикотаж"s }, - { "\"category\""s, ResultType::Return, "category"s }, - { "\"/Летние товары/Летний текстиль/\""s, ResultType::Return, "/Летние товары/Летний текстиль/"s }, - { "\"variant\""s, ResultType::Return, "variant"s }, - { "\"\""s, ResultType::Return, ""s }, - { "\"price\""s, ResultType::Return, "price"s }, - { "\"390.00\""s, ResultType::Return, "390.00"s }, - { "\"list\""s, ResultType::Return, "list"s }, - { "\"/retailrocket/\""s, ResultType::Return, "/retailrocket/"s }, - { "\"position\""s, ResultType::Return, "position"s }, - { "\"brand\""s, ResultType::Return, "brand"s }, - { "\"/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/\""s, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"s }, - { "\"id\""s, ResultType::Return, "id"s }, - { "\"911504\""s, ResultType::Return, "911504"s }, - { "\"name\""s, ResultType::Return, "name"s }, - { "\"Футболка детская 3D Fairytale, возраст 1-2 года, трикотаж\""s, ResultType::Return, "Футболка детская 3D Fairytale, возраст 1-2 года, трикотаж"s }, - { "\"category\""s, ResultType::Return, "category"s }, - { "\"/Летние товары/Летний текстиль/\""s, ResultType::Return, "/Летние товары/Летний текстиль/"s }, - { "\"variant\""s, ResultType::Return, "variant"s }, - { "\"\""s, ResultType::Return, ""s }, - { "\"price\""s, ResultType::Return, "price"s }, - { "\"390.00\""s, ResultType::Return, "390.00"s }, - { "\"list\""s, ResultType::Return, "list"s }, - { "\"/retailrocket/\""s, ResultType::Return, "/retailrocket/"s }, - { "\"position\""s, ResultType::Return, "position"s }, - { "\"brand\""s, ResultType::Return, "brand"s }, - { "\"/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/\""s, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"s }, - { "\"id\""s, ResultType::Return, "id"s }, - { "\"911508\""s, ResultType::Return, "911508"s }, - { "\"name\""s, ResultType::Return, "name"s }, - { "\"Футболка детская 3D Kittens, возраст 1-2 года, трикотаж\""s, ResultType::Return, "Футболка детская 3D Kittens, возраст 1-2 года, трикотаж"s }, - { "\"category\""s, ResultType::Return, "category"s }, - { "\"/Летние товары/Летний текстиль/\""s, ResultType::Return, "/Летние товары/Летний текстиль/"s }, - { "\"variant\""s, ResultType::Return, "variant"s }, - { "\"\""s, ResultType::Return, ""s }, - { "\"price\""s, ResultType::Return, "price"s }, - { "\"390.00\""s, ResultType::Return, "390.00"s }, - { "\"list\""s, ResultType::Return, "list"s }, - { "\"/retailrocket/\""s, ResultType::Return, "/retailrocket/"s }, - { "\"position\""s, ResultType::Return, "position"s }, - { "\"brand\""s, ResultType::Return, "brand"s }, - { "\"/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/\""s, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"s }, - { "\"id\""s, ResultType::Return, "id"s }, - { "\"911512\""s, ResultType::Return, "911512"s }, - { "\"name\""s, ResultType::Return, "name"s }, - { "\"Футболка детская 3D Sunshine, возраст 1-2 года, трикотаж\""s, ResultType::Return, "Футболка детская 3D Sunshine, возраст 1-2 года, трикотаж"s }, - { "\"category\""s, ResultType::Return, "category"s }, - { "\"/Летние товары/Летний текстиль/\""s, ResultType::Return, "/Летние товары/Летний текстиль/"s }, - { "\"variant\""s, ResultType::Return, "variant"s }, - { "\"\""s, ResultType::Return, ""s }, - { "\"price\""s, ResultType::Return, "price"s }, - { "\"390.00\""s, ResultType::Return, "390.00"s }, - { "\"list\""s, ResultType::Return, "list"s }, - { "\"/retailrocket/\""s, ResultType::Return, "/retailrocket/"s }, - { "\"position\""s, ResultType::Return, "position"s }, - { "\"brand\""s, ResultType::Return, "brand"s }, - { "\"/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/\""s, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"s }, - { "\"id\""s, ResultType::Return, "id"s }, - { "\"911516\""s, ResultType::Return, "911516"s }, - { "\"name\""s, ResultType::Return, "name"s }, - { "\"Футболка детская 3D Dog in bag, возраст 1-2 года, трикотаж\""s, ResultType::Return, "Футболка детская 3D Dog in bag, возраст 1-2 года, трикотаж"s }, - { "\"category\""s, ResultType::Return, "category"s }, - { "\"/Летние товары/Летний текстиль/\""s, ResultType::Return, "/Летние товары/Летний текстиль/"s }, - { "\"variant\""s, ResultType::Return, "variant"s }, - { "\"\""s, ResultType::Return, ""s }, - { "\"price\""s, ResultType::Return, "price"s }, - { "\"390.00\""s, ResultType::Return, "390.00"s }, - { "\"list\""s, ResultType::Return, "list"s }, - { "\"/retailrocket/\""s, ResultType::Return, "/retailrocket/"s }, - { "\"position\""s, ResultType::Return, "position"s }, - { "\"brand\""s, ResultType::Return, "brand"s }, - { "\"/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/\""s, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"s }, - { "\"id\""s, ResultType::Return, "id"s }, - { "\"911520\""s, ResultType::Return, "911520"s }, - { "\"name\""s, ResultType::Return, "name"s }, - { "\"Футболка детская 3D Cute puppy, возраст 1-2 года, трикотаж\""s, ResultType::Return, "Футболка детская 3D Cute puppy, возраст 1-2 года, трикотаж"s }, - { "\"category\""s, ResultType::Return, "category"s }, - { "\"/Летние товары/Летний текстиль/\""s, ResultType::Return, "/Летние товары/Летний текстиль/"s }, - { "\"variant\""s, ResultType::Return, "variant"s }, - { "\"\""s, ResultType::Return, ""s }, - { "\"price\""s, ResultType::Return, "price"s }, - { "\"390.00\""s, ResultType::Return, "390.00"s }, - { "\"list\""s, ResultType::Return, "list"s }, - { "\"/retailrocket/\""s, ResultType::Return, "/retailrocket/"s }, - { "\"position\""s, ResultType::Return, "position"s }, - { "\"brand\""s, ResultType::Return, "brand"s }, - { "\"/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/\""s, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"s }, - { "\"id\""s, ResultType::Return, "id"s }, - { "\"911524\""s, ResultType::Return, "911524"s }, - { "\"name\""s, ResultType::Return, "name"s }, - { "\"Футболка детская 3D Rabbit, возраст 1-2 года, трикотаж\""s, ResultType::Return, "Футболка детская 3D Rabbit, возраст 1-2 года, трикотаж"s }, - { "\"category\""s, ResultType::Return, "category"s }, - { "\"/Летние товары/Летний текстиль/\""s, ResultType::Return, "/Летние товары/Летний текстиль/"s }, - { "\"variant\""s, ResultType::Return, "variant"s }, - { "\"\""s, ResultType::Return, ""s }, - { "\"price\""s, ResultType::Return, "price"s }, - { "\"390.00\""s, ResultType::Return, "390.00"s }, - { "\"list\""s, ResultType::Return, "list"s }, - { "\"/retailrocket/\""s, ResultType::Return, "/retailrocket/"s }, - { "\"position\""s, ResultType::Return, "position"s }, - { "\"brand\""s, ResultType::Return, "brand"s }, - { "\"/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/\""s, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"s }, - { "\"id\""s, ResultType::Return, "id"s }, - { "\"911528\""s, ResultType::Return, "911528"s }, - { "\"name\""s, ResultType::Return, "name"s }, - { "\"Футболка детская 3D Turtle, возраст 1-2 года, трикотаж\""s, ResultType::Return, "Футболка детская 3D Turtle, возраст 1-2 года, трикотаж"s }, - { "\"category\""s, ResultType::Return, "category"s }, - { "\"/Летние товары/Летний текстиль/\""s, ResultType::Return, "/Летние товары/Летний текстиль/"s }, - { "\"variant\""s, ResultType::Return, "variant"s }, - { "\"\""s, ResultType::Return, ""s }, - { "\"price\""s, ResultType::Return, "price"s }, - { "\"390.00\""s, ResultType::Return, "390.00"s }, - { "\"list\""s, ResultType::Return, "list"s }, - { "\"/retailrocket/\""s, ResultType::Return, "/retailrocket/"s }, - { "\"position\""s, ResultType::Return, "position"s }, - { "\"brand\""s, ResultType::Return, "brand"s }, - { "\"/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/\""s, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"s }, - { "\"id\""s, ResultType::Return, "id"s }, - { "\"888616\""s, ResultType::Return, "888616"s }, - { "\"name\""s, ResultType::Return, "name"s }, + { R"("name")"s, ResultType::Return, "name"s }, + { R"("Вафельница Vitek WX-1102 FL")"s, ResultType::Return, "Вафельница Vitek WX-1102 FL"s }, + { R"("brand")"s, ResultType::Return, "brand"s }, + { R"("184509")"s, ResultType::Return, "184509"s }, + { R"("category")"s, ResultType::Return, "category"s }, + { R"("Все для детей/Детская техника/Vitek")"s, ResultType::Return, "Все для детей/Детская техника/Vitek"s }, + { R"("variant")"s, ResultType::Return, "variant"s }, + { R"("В наличии")"s, ResultType::Return, "В наличии"s }, + { R"("price")"s, ResultType::Return, "price"s }, + { R"("2390.00")"s, ResultType::Return, "2390.00"s }, + { R"("list")"s, ResultType::Return, "list"s }, + { R"("Карточка")"s, ResultType::Return, "Карточка"s }, + { R"("position")"s, ResultType::Return, "position"s }, + { R"("detail")"s, ResultType::Return, "detail"s }, + { R"("actionField")"s, ResultType::Return, "actionField"s }, + { R"("list")"s, ResultType::Return, "list"s }, + { R"("http://www.techport.ru/q/?t=вафельница&sort=price&sdim=asc")"s, ResultType::Return, "http://www.techport.ru/q/?t=вафельница&sort=price&sdim=asc"s }, + { R"("action")"s, ResultType::Return, "action"s }, + { R"("detail")"s, ResultType::Return, "detail"s }, + { R"("products")"s, ResultType::Return, "products"s }, + { R"("name")"s, ResultType::Return, "name"s }, + { R"("Вафельница Vitek WX-1102 FL")"s, ResultType::Return, "Вафельница Vitek WX-1102 FL"s }, + { R"("id")"s, ResultType::Return, "id"s }, + { R"("184509")"s, ResultType::Return, "184509"s }, + { R"("price")"s, ResultType::Return, "price"s }, + { R"("2390.00")"s, ResultType::Return, "2390.00"s }, + { R"("brand")"s, ResultType::Return, "brand"s }, + { R"("Vitek")"s, ResultType::Return, "Vitek"s }, + { R"("category")"s, ResultType::Return, "category"s }, + { R"("Все для детей/Детская техника/Vitek")"s, ResultType::Return, "Все для детей/Детская техника/Vitek"s }, + { R"("variant")"s, ResultType::Return, "variant"s }, + { R"("В наличии")"s, ResultType::Return, "В наличии"s }, + { R"("ru")"s, ResultType::Return, "ru"s }, + { R"("experiments")"s, ResultType::Return, "experiments"s }, + { R"("lang")"s, ResultType::Return, "lang"s }, + { R"("ru")"s, ResultType::Return, "ru"s }, + { R"("los_portal")"s, ResultType::Return, "los_portal"s }, + { R"("los_level")"s, ResultType::Return, "los_level"s }, + { R"("none")"s, ResultType::Return, "none"s }, + { R"("isAuthorized")"s, ResultType::Return, "isAuthorized"s }, + { R"("isSubscriber")"s, ResultType::Return, "isSubscriber"s }, + { R"("postType")"s, ResultType::Return, "postType"s }, + { R"("Новости")"s, ResultType::Return, "Новости"s }, + { R"("experiments")"s, ResultType::Return, "experiments"s }, + { R"("lang")"s, ResultType::Return, "lang"s }, + { R"("ru")"s, ResultType::Return, "ru"s }, + { R"("los_portal")"s, ResultType::Return, "los_portal"s }, + { R"("los_level")"s, ResultType::Return, "los_level"s }, + { R"("none")"s, ResultType::Return, "none"s }, + { R"("lang")"s, ResultType::Return, "lang"s }, + { R"("ru")"s, ResultType::Return, "ru"s }, + { R"("Электроплита GEFEST Брест ЭПНД 5140-01 0001")"s, ResultType::Return, "Электроплита GEFEST Брест ЭПНД 5140-01 0001"s }, + { R"("price")"s, ResultType::Return, "price"s }, + { R"("currencyCode")"s, ResultType::Return, "currencyCode"s }, + { R"("RUB")"s, ResultType::Return, "RUB"s }, + { R"("lang")"s, ResultType::Return, "lang"s }, + { R"("ru")"s, ResultType::Return, "ru"s }, + { R"("experiments")"s, ResultType::Return, "experiments"s }, + { R"("lang")"s, ResultType::Return, "lang"s }, + { R"("ru")"s, ResultType::Return, "ru"s }, + { R"("los_portal")"s, ResultType::Return, "los_portal"s }, + { R"("los_level")"s, ResultType::Return, "los_level"s }, + { R"("none")"s, ResultType::Return, "none"s }, + { R"("trash_login")"s, ResultType::Return, "trash_login"s }, + { R"("novikoff")"s, ResultType::Return, "novikoff"s }, + { R"("trash_cat_link")"s, ResultType::Return, "trash_cat_link"s }, + { R"("progs")"s, ResultType::Return, "progs"s }, + { R"("trash_parent_link")"s, ResultType::Return, "trash_parent_link"s }, + { R"("content")"s, ResultType::Return, "content"s }, + { R"("trash_posted_parent")"s, ResultType::Return, "trash_posted_parent"s }, + { R"("content.01.2016")"s, ResultType::Return, "content.01.2016"s }, + { R"("trash_posted_cat")"s, ResultType::Return, "trash_posted_cat"s }, + { R"("progs.01.2016")"s, ResultType::Return, "progs.01.2016"s }, + { R"("trash_virus_count")"s, ResultType::Return, "trash_virus_count"s }, + { R"("trash_is_android")"s, ResultType::Return, "trash_is_android"s }, + { R"("trash_is_wp8")"s, ResultType::Return, "trash_is_wp8"s }, + { R"("trash_is_ios")"s, ResultType::Return, "trash_is_ios"s }, + { R"("trash_posted")"s, ResultType::Return, "trash_posted"s }, + { R"("01.2016")"s, ResultType::Return, "01.2016"s }, + { R"("experiments")"s, ResultType::Return, "experiments"s }, + { R"("lang")"s, ResultType::Return, "lang"s }, + { R"("ru")"s, ResultType::Return, "ru"s }, + { R"("los_portal")"s, ResultType::Return, "los_portal"s }, + { R"("los_level")"s, ResultType::Return, "los_level"s }, + { R"("none")"s, ResultType::Return, "none"s }, + { R"("merchantId")"s, ResultType::Return, "merchantId"s }, + { R"("13694_49246")"s, ResultType::Return, "13694_49246"s }, + { R"("cps-source")"s, ResultType::Return, "cps-source"s }, + { R"("wargaming")"s, ResultType::Return, "wargaming"s }, + { R"("cps_provider")"s, ResultType::Return, "cps_provider"s }, + { R"("default")"s, ResultType::Return, "default"s }, + { R"("errorReason")"s, ResultType::Return, "errorReason"s }, + { R"("no errors")"s, ResultType::Return, "no errors"s }, + { R"("scid")"s, ResultType::Return, "scid"s }, + { R"("isAuthPayment")"s, ResultType::Return, "isAuthPayment"s }, + { R"("lang")"s, ResultType::Return, "lang"s }, + { R"("ru")"s, ResultType::Return, "ru"s }, + { R"("rubric")"s, ResultType::Return, "rubric"s }, + { R"("")"s, ResultType::Return, ""s }, + { R"("rubric")"s, ResultType::Return, "rubric"s }, + { R"("Мир")"s, ResultType::Return, "Мир"s }, + { R"("lang")"s, ResultType::Return, "lang"s }, + { R"("ru")"s, ResultType::Return, "ru"s }, + { R"("experiments")"s, ResultType::Return, "experiments"s }, + { R"("lang")"s, ResultType::Return, "lang"s }, + { R"("ru")"s, ResultType::Return, "ru"s }, + { R"("los_portal")"s, ResultType::Return, "los_portal"s }, + { R"("los_level")"s, ResultType::Return, "los_level"s }, + { R"("none")"s, ResultType::Return, "none"s }, + { R"("lang")"s, ResultType::Return, "lang"s }, + { R"("ru")"s, ResultType::Return, "ru"s }, + { R"("__ym")"s, ResultType::Return, "__ym"s }, + { R"("ecommerce")"s, ResultType::Return, "ecommerce"s }, + { R"("impressions")"s, ResultType::Return, "impressions"s }, + { R"("id")"s, ResultType::Return, "id"s }, + { R"("863813")"s, ResultType::Return, "863813"s }, + { R"("name")"s, ResultType::Return, "name"s }, + { R"("Футболка детская 3D Happy, возраст 1-2 года, трикотаж")"s, ResultType::Return, "Футболка детская 3D Happy, возраст 1-2 года, трикотаж"s }, + { R"("category")"s, ResultType::Return, "category"s }, + { R"("/Летние товары/Летний текстиль/")"s, ResultType::Return, "/Летние товары/Летний текстиль/"s }, + { R"("variant")"s, ResultType::Return, "variant"s }, + { R"("")"s, ResultType::Return, ""s }, + { R"("price")"s, ResultType::Return, "price"s }, + { R"("390.00")"s, ResultType::Return, "390.00"s }, + { R"("list")"s, ResultType::Return, "list"s }, + { R"("/retailrocket/")"s, ResultType::Return, "/retailrocket/"s }, + { R"("position")"s, ResultType::Return, "position"s }, + { R"("brand")"s, ResultType::Return, "brand"s }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"s, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"s }, + { R"("id")"s, ResultType::Return, "id"s }, + { R"("863839")"s, ResultType::Return, "863839"s }, + { R"("name")"s, ResultType::Return, "name"s }, + { R"("Футболка детская 3D Pretty kitten, возраст 1-2 года, трикотаж")"s, ResultType::Return, "Футболка детская 3D Pretty kitten, возраст 1-2 года, трикотаж"s }, + { R"("category")"s, ResultType::Return, "category"s }, + { R"("/Летние товары/Летний текстиль/")"s, ResultType::Return, "/Летние товары/Летний текстиль/"s }, + { R"("variant")"s, ResultType::Return, "variant"s }, + { R"("")"s, ResultType::Return, ""s }, + { R"("price")"s, ResultType::Return, "price"s }, + { R"("390.00")"s, ResultType::Return, "390.00"s }, + { R"("list")"s, ResultType::Return, "list"s }, + { R"("/retailrocket/")"s, ResultType::Return, "/retailrocket/"s }, + { R"("position")"s, ResultType::Return, "position"s }, + { R"("brand")"s, ResultType::Return, "brand"s }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"s, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"s }, + { R"("id")"s, ResultType::Return, "id"s }, + { R"("863847")"s, ResultType::Return, "863847"s }, + { R"("name")"s, ResultType::Return, "name"s }, + { R"("Футболка детская 3D Little tiger, возраст 1-2 года, трикотаж")"s, ResultType::Return, "Футболка детская 3D Little tiger, возраст 1-2 года, трикотаж"s }, + { R"("category")"s, ResultType::Return, "category"s }, + { R"("/Летние товары/Летний текстиль/")"s, ResultType::Return, "/Летние товары/Летний текстиль/"s }, + { R"("variant")"s, ResultType::Return, "variant"s }, + { R"("")"s, ResultType::Return, ""s }, + { R"("price")"s, ResultType::Return, "price"s }, + { R"("390.00")"s, ResultType::Return, "390.00"s }, + { R"("list")"s, ResultType::Return, "list"s }, + { R"("/retailrocket/")"s, ResultType::Return, "/retailrocket/"s }, + { R"("position")"s, ResultType::Return, "position"s }, + { R"("brand")"s, ResultType::Return, "brand"s }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"s, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"s }, + { R"("id")"s, ResultType::Return, "id"s }, + { R"("911480")"s, ResultType::Return, "911480"s }, + { R"("name")"s, ResultType::Return, "name"s }, + { R"("Футболка детская 3D Puppy, возраст 1-2 года, трикотаж")"s, ResultType::Return, "Футболка детская 3D Puppy, возраст 1-2 года, трикотаж"s }, + { R"("category")"s, ResultType::Return, "category"s }, + { R"("/Летние товары/Летний текстиль/")"s, ResultType::Return, "/Летние товары/Летний текстиль/"s }, + { R"("variant")"s, ResultType::Return, "variant"s }, + { R"("")"s, ResultType::Return, ""s }, + { R"("price")"s, ResultType::Return, "price"s }, + { R"("390.00")"s, ResultType::Return, "390.00"s }, + { R"("list")"s, ResultType::Return, "list"s }, + { R"("/retailrocket/")"s, ResultType::Return, "/retailrocket/"s }, + { R"("position")"s, ResultType::Return, "position"s }, + { R"("brand")"s, ResultType::Return, "brand"s }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"s, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"s }, + { R"("id")"s, ResultType::Return, "id"s }, + { R"("911484")"s, ResultType::Return, "911484"s }, + { R"("name")"s, ResultType::Return, "name"s }, + { R"("Футболка детская 3D Little bears, возраст 1-2 года, трикотаж")"s, ResultType::Return, "Футболка детская 3D Little bears, возраст 1-2 года, трикотаж"s }, + { R"("category")"s, ResultType::Return, "category"s }, + { R"("/Летние товары/Летний текстиль/")"s, ResultType::Return, "/Летние товары/Летний текстиль/"s }, + { R"("variant")"s, ResultType::Return, "variant"s }, + { R"("")"s, ResultType::Return, ""s }, + { R"("price")"s, ResultType::Return, "price"s }, + { R"("390.00")"s, ResultType::Return, "390.00"s }, + { R"("list")"s, ResultType::Return, "list"s }, + { R"("/retailrocket/")"s, ResultType::Return, "/retailrocket/"s }, + { R"("position")"s, ResultType::Return, "position"s }, + { R"("brand")"s, ResultType::Return, "brand"s }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"s, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"s }, + { R"("id")"s, ResultType::Return, "id"s }, + { R"("911489")"s, ResultType::Return, "911489"s }, + { R"("name")"s, ResultType::Return, "name"s }, + { R"("Футболка детская 3D Dolphin, возраст 2-4 года, трикотаж")"s, ResultType::Return, "Футболка детская 3D Dolphin, возраст 2-4 года, трикотаж"s }, + { R"("category")"s, ResultType::Return, "category"s }, + { R"("/Летние товары/Летний текстиль/")"s, ResultType::Return, "/Летние товары/Летний текстиль/"s }, + { R"("variant")"s, ResultType::Return, "variant"s }, + { R"("")"s, ResultType::Return, ""s }, + { R"("price")"s, ResultType::Return, "price"s }, + { R"("390.00")"s, ResultType::Return, "390.00"s }, + { R"("list")"s, ResultType::Return, "list"s }, + { R"("/retailrocket/")"s, ResultType::Return, "/retailrocket/"s }, + { R"("position")"s, ResultType::Return, "position"s }, + { R"("brand")"s, ResultType::Return, "brand"s }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"s, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"s }, + { R"("id")"s, ResultType::Return, "id"s }, + { R"("911496")"s, ResultType::Return, "911496"s }, + { R"("name")"s, ResultType::Return, "name"s }, + { R"("Футболка детская 3D Pretty, возраст 1-2 года, трикотаж")"s, ResultType::Return, "Футболка детская 3D Pretty, возраст 1-2 года, трикотаж"s }, + { R"("category")"s, ResultType::Return, "category"s }, + { R"("/Летние товары/Летний текстиль/")"s, ResultType::Return, "/Летние товары/Летний текстиль/"s }, + { R"("variant")"s, ResultType::Return, "variant"s }, + { R"("")"s, ResultType::Return, ""s }, + { R"("price")"s, ResultType::Return, "price"s }, + { R"("390.00")"s, ResultType::Return, "390.00"s }, + { R"("list")"s, ResultType::Return, "list"s }, + { R"("/retailrocket/")"s, ResultType::Return, "/retailrocket/"s }, + { R"("position")"s, ResultType::Return, "position"s }, + { R"("brand")"s, ResultType::Return, "brand"s }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"s, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"s }, + { R"("id")"s, ResultType::Return, "id"s }, + { R"("911504")"s, ResultType::Return, "911504"s }, + { R"("name")"s, ResultType::Return, "name"s }, + { R"("Футболка детская 3D Fairytale, возраст 1-2 года, трикотаж")"s, ResultType::Return, "Футболка детская 3D Fairytale, возраст 1-2 года, трикотаж"s }, + { R"("category")"s, ResultType::Return, "category"s }, + { R"("/Летние товары/Летний текстиль/")"s, ResultType::Return, "/Летние товары/Летний текстиль/"s }, + { R"("variant")"s, ResultType::Return, "variant"s }, + { R"("")"s, ResultType::Return, ""s }, + { R"("price")"s, ResultType::Return, "price"s }, + { R"("390.00")"s, ResultType::Return, "390.00"s }, + { R"("list")"s, ResultType::Return, "list"s }, + { R"("/retailrocket/")"s, ResultType::Return, "/retailrocket/"s }, + { R"("position")"s, ResultType::Return, "position"s }, + { R"("brand")"s, ResultType::Return, "brand"s }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"s, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"s }, + { R"("id")"s, ResultType::Return, "id"s }, + { R"("911508")"s, ResultType::Return, "911508"s }, + { R"("name")"s, ResultType::Return, "name"s }, + { R"("Футболка детская 3D Kittens, возраст 1-2 года, трикотаж")"s, ResultType::Return, "Футболка детская 3D Kittens, возраст 1-2 года, трикотаж"s }, + { R"("category")"s, ResultType::Return, "category"s }, + { R"("/Летние товары/Летний текстиль/")"s, ResultType::Return, "/Летние товары/Летний текстиль/"s }, + { R"("variant")"s, ResultType::Return, "variant"s }, + { R"("")"s, ResultType::Return, ""s }, + { R"("price")"s, ResultType::Return, "price"s }, + { R"("390.00")"s, ResultType::Return, "390.00"s }, + { R"("list")"s, ResultType::Return, "list"s }, + { R"("/retailrocket/")"s, ResultType::Return, "/retailrocket/"s }, + { R"("position")"s, ResultType::Return, "position"s }, + { R"("brand")"s, ResultType::Return, "brand"s }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"s, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"s }, + { R"("id")"s, ResultType::Return, "id"s }, + { R"("911512")"s, ResultType::Return, "911512"s }, + { R"("name")"s, ResultType::Return, "name"s }, + { R"("Футболка детская 3D Sunshine, возраст 1-2 года, трикотаж")"s, ResultType::Return, "Футболка детская 3D Sunshine, возраст 1-2 года, трикотаж"s }, + { R"("category")"s, ResultType::Return, "category"s }, + { R"("/Летние товары/Летний текстиль/")"s, ResultType::Return, "/Летние товары/Летний текстиль/"s }, + { R"("variant")"s, ResultType::Return, "variant"s }, + { R"("")"s, ResultType::Return, ""s }, + { R"("price")"s, ResultType::Return, "price"s }, + { R"("390.00")"s, ResultType::Return, "390.00"s }, + { R"("list")"s, ResultType::Return, "list"s }, + { R"("/retailrocket/")"s, ResultType::Return, "/retailrocket/"s }, + { R"("position")"s, ResultType::Return, "position"s }, + { R"("brand")"s, ResultType::Return, "brand"s }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"s, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"s }, + { R"("id")"s, ResultType::Return, "id"s }, + { R"("911516")"s, ResultType::Return, "911516"s }, + { R"("name")"s, ResultType::Return, "name"s }, + { R"("Футболка детская 3D Dog in bag, возраст 1-2 года, трикотаж")"s, ResultType::Return, "Футболка детская 3D Dog in bag, возраст 1-2 года, трикотаж"s }, + { R"("category")"s, ResultType::Return, "category"s }, + { R"("/Летние товары/Летний текстиль/")"s, ResultType::Return, "/Летние товары/Летний текстиль/"s }, + { R"("variant")"s, ResultType::Return, "variant"s }, + { R"("")"s, ResultType::Return, ""s }, + { R"("price")"s, ResultType::Return, "price"s }, + { R"("390.00")"s, ResultType::Return, "390.00"s }, + { R"("list")"s, ResultType::Return, "list"s }, + { R"("/retailrocket/")"s, ResultType::Return, "/retailrocket/"s }, + { R"("position")"s, ResultType::Return, "position"s }, + { R"("brand")"s, ResultType::Return, "brand"s }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"s, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"s }, + { R"("id")"s, ResultType::Return, "id"s }, + { R"("911520")"s, ResultType::Return, "911520"s }, + { R"("name")"s, ResultType::Return, "name"s }, + { R"("Футболка детская 3D Cute puppy, возраст 1-2 года, трикотаж")"s, ResultType::Return, "Футболка детская 3D Cute puppy, возраст 1-2 года, трикотаж"s }, + { R"("category")"s, ResultType::Return, "category"s }, + { R"("/Летние товары/Летний текстиль/")"s, ResultType::Return, "/Летние товары/Летний текстиль/"s }, + { R"("variant")"s, ResultType::Return, "variant"s }, + { R"("")"s, ResultType::Return, ""s }, + { R"("price")"s, ResultType::Return, "price"s }, + { R"("390.00")"s, ResultType::Return, "390.00"s }, + { R"("list")"s, ResultType::Return, "list"s }, + { R"("/retailrocket/")"s, ResultType::Return, "/retailrocket/"s }, + { R"("position")"s, ResultType::Return, "position"s }, + { R"("brand")"s, ResultType::Return, "brand"s }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"s, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"s }, + { R"("id")"s, ResultType::Return, "id"s }, + { R"("911524")"s, ResultType::Return, "911524"s }, + { R"("name")"s, ResultType::Return, "name"s }, + { R"("Футболка детская 3D Rabbit, возраст 1-2 года, трикотаж")"s, ResultType::Return, "Футболка детская 3D Rabbit, возраст 1-2 года, трикотаж"s }, + { R"("category")"s, ResultType::Return, "category"s }, + { R"("/Летние товары/Летний текстиль/")"s, ResultType::Return, "/Летние товары/Летний текстиль/"s }, + { R"("variant")"s, ResultType::Return, "variant"s }, + { R"("")"s, ResultType::Return, ""s }, + { R"("price")"s, ResultType::Return, "price"s }, + { R"("390.00")"s, ResultType::Return, "390.00"s }, + { R"("list")"s, ResultType::Return, "list"s }, + { R"("/retailrocket/")"s, ResultType::Return, "/retailrocket/"s }, + { R"("position")"s, ResultType::Return, "position"s }, + { R"("brand")"s, ResultType::Return, "brand"s }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"s, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"s }, + { R"("id")"s, ResultType::Return, "id"s }, + { R"("911528")"s, ResultType::Return, "911528"s }, + { R"("name")"s, ResultType::Return, "name"s }, + { R"("Футболка детская 3D Turtle, возраст 1-2 года, трикотаж")"s, ResultType::Return, "Футболка детская 3D Turtle, возраст 1-2 года, трикотаж"s }, + { R"("category")"s, ResultType::Return, "category"s }, + { R"("/Летние товары/Летний текстиль/")"s, ResultType::Return, "/Летние товары/Летний текстиль/"s }, + { R"("variant")"s, ResultType::Return, "variant"s }, + { R"("")"s, ResultType::Return, ""s }, + { R"("price")"s, ResultType::Return, "price"s }, + { R"("390.00")"s, ResultType::Return, "390.00"s }, + { R"("list")"s, ResultType::Return, "list"s }, + { R"("/retailrocket/")"s, ResultType::Return, "/retailrocket/"s }, + { R"("position")"s, ResultType::Return, "position"s }, + { R"("brand")"s, ResultType::Return, "brand"s }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"s, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"s }, + { R"("id")"s, ResultType::Return, "id"s }, + { R"("888616")"s, ResultType::Return, "888616"s }, + { R"("name")"s, ResultType::Return, "name"s }, { "\"3Д Футболка мужская \\\"Collorista\\\" Светлое завтра р-р XL(52-54), 100% хлопок, трикотаж\""s, ResultType::Return, "3Д Футболка мужская \"Collorista\" Светлое завтра р-р XL(52-54), 100% хлопок, трикотаж"s }, - { "\"category\""s, ResultType::Return, "category"s }, - { "\"/Одежда и обувь/Мужская одежда/Футболки/\""s, ResultType::Return, "/Одежда и обувь/Мужская одежда/Футболки/"s }, - { "\"variant\""s, ResultType::Return, "variant"s }, - { "\"\""s, ResultType::Return, ""s }, - { "\"price\""s, ResultType::Return, "price"s }, - { "\"406.60\""s, ResultType::Return, "406.60"s }, - { "\"list\""s, ResultType::Return, "list"s }, - { "\"/retailrocket/\""s, ResultType::Return, "/retailrocket/"s }, - { "\"position\""s, ResultType::Return, "position"s }, - { "\"brand\""s, ResultType::Return, "brand"s }, - { "\"/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/\""s, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"s }, - { "\"id\""s, ResultType::Return, "id"s }, - { "\"913361\""s, ResultType::Return, "913361"s }, - { "\"name\""s, ResultType::Return, "name"s }, - { "\"3Д Футболка детская World р-р 8-10, 100% хлопок, трикотаж\""s, ResultType::Return, "3Д Футболка детская World р-р 8-10, 100% хлопок, трикотаж"s }, - { "\"category\""s, ResultType::Return, "category"s }, - { "\"/Летние товары/Летний текстиль/\""s, ResultType::Return, "/Летние товары/Летний текстиль/"s }, - { "\"variant\""s, ResultType::Return, "variant"s }, - { "\"\""s, ResultType::Return, ""s }, - { "\"price\""s, ResultType::Return, "price"s }, - { "\"470.00\""s, ResultType::Return, "470.00"s }, - { "\"list\""s, ResultType::Return, "list"s }, - { "\"/retailrocket/\""s, ResultType::Return, "/retailrocket/"s }, - { "\"position\""s, ResultType::Return, "position"s }, - { "\"brand\""s, ResultType::Return, "brand"s }, - { "\"/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/\""s, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"s }, - { "\"id\""s, ResultType::Return, "id"s }, - { "\"913364\""s, ResultType::Return, "913364"s }, - { "\"name\""s, ResultType::Return, "name"s }, - { "\"3Д Футболка детская Force р-р 8-10, 100% хлопок, трикотаж\""s, ResultType::Return, "3Д Футболка детская Force р-р 8-10, 100% хлопок, трикотаж"s }, - { "\"category\""s, ResultType::Return, "category"s }, - { "\"/Летние товары/Летний текстиль/\""s, ResultType::Return, "/Летние товары/Летний текстиль/"s }, - { "\"variant\""s, ResultType::Return, "variant"s }, - { "\"\""s, ResultType::Return, ""s }, - { "\"price\""s, ResultType::Return, "price"s }, - { "\"470.00\""s, ResultType::Return, "470.00"s }, - { "\"list\""s, ResultType::Return, "list"s }, - { "\"/retailrocket/\""s, ResultType::Return, "/retailrocket/"s }, - { "\"position\""s, ResultType::Return, "position"s }, - { "\"brand\""s, ResultType::Return, "brand"s }, - { "\"/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/\""s, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"s }, - { "\"id\""s, ResultType::Return, "id"s }, - { "\"913367\""s, ResultType::Return, "913367"s }, - { "\"name\""s, ResultType::Return, "name"s }, - { "\"3Д Футболка детская Winter tale р-р 8-10, 100% хлопок, трикотаж\""s, ResultType::Return, "3Д Футболка детская Winter tale р-р 8-10, 100% хлопок, трикотаж"s }, - { "\"category\""s, ResultType::Return, "category"s }, - { "\"/Летние товары/Летний текстиль/\""s, ResultType::Return, "/Летние товары/Летний текстиль/"s }, - { "\"variant\""s, ResultType::Return, "variant"s }, - { "\"\""s, ResultType::Return, ""s }, - { "\"price\""s, ResultType::Return, "price"s }, - { "\"470.00\""s, ResultType::Return, "470.00"s }, - { "\"list\""s, ResultType::Return, "list"s }, - { "\"/retailrocket/\""s, ResultType::Return, "/retailrocket/"s }, - { "\"position\""s, ResultType::Return, "position"s }, - { "\"brand\""s, ResultType::Return, "brand"s }, - { "\"/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/\""s, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"s }, - { "\"id\""s, ResultType::Return, "id"s }, - { "\"913385\""s, ResultType::Return, "913385"s }, - { "\"name\""s, ResultType::Return, "name"s }, - { "\"3Д Футболка детская Moonshine р-р 8-10, 100% хлопок, трикотаж\""s, ResultType::Return, "3Д Футболка детская Moonshine р-р 8-10, 100% хлопок, трикотаж"s }, - { "\"category\""s, ResultType::Return, "category"s }, - { "\"/Летние товары/Летний текстиль/\""s, ResultType::Return, "/Летние товары/Летний текстиль/"s }, - { "\"variant\""s, ResultType::Return, "variant"s }, - { "\"\""s, ResultType::Return, ""s }, - { "\"price\""s, ResultType::Return, "price"s }, - { "\"470.00\""s, ResultType::Return, "470.00"s }, - { "\"list\""s, ResultType::Return, "list"s }, - { "\"/retailrocket/\""s, ResultType::Return, "/retailrocket/"s }, - { "\"position\""s, ResultType::Return, "position"s }, - { "\"brand\""s, ResultType::Return, "brand"s }, - { "\"/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/\""s, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"s }, - { "\"id\""s, ResultType::Return, "id"s }, - { "\"913391\""s, ResultType::Return, "913391"s }, - { "\"name\""s, ResultType::Return, "name"s }, - { "\"3Д Футболка детская Shaman р-р 8-10, 100% хлопок, трикотаж\""s, ResultType::Return, "3Д Футболка детская Shaman р-р 8-10, 100% хлопок, трикотаж"s }, - { "\"category\""s, ResultType::Return, "category"s }, - { "\"/Летние товары/Летний текстиль/\""s, ResultType::Return, "/Летние товары/Летний текстиль/"s }, - { "\"variant\""s, ResultType::Return, "variant"s }, - { "\"\""s, ResultType::Return, ""s }, - { "\"price\""s, ResultType::Return, "price"s }, - { "\"470.00\""s, ResultType::Return, "470.00"s }, - { "\"list\""s, ResultType::Return, "list"s }, - { "\"/retailrocket/\""s, ResultType::Return, "/retailrocket/"s }, - { "\"position\""s, ResultType::Return, "position"s }, - { "\"brand\""s, ResultType::Return, "brand"s }, - { "\"/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/\""s, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"s }, - { "\"usertype\""s, ResultType::Return, "usertype"s }, - { "\"visitor\""s, ResultType::Return, "visitor"s }, - { "\"lang\""s, ResultType::Return, "lang"s }, - { "\"ru\""s, ResultType::Return, "ru"s }, - { "\"__ym\""s, ResultType::Return, "__ym"s }, - { "\"ecommerce\""s, ResultType::Return, "ecommerce"s }, - { "\"impressions\""s, ResultType::Return, "impressions"s }, - { "\"experiments\""s, ResultType::Return, "experiments"s }, - { "\"lang\""s, ResultType::Return, "lang"s }, - { "\"ru\""s, ResultType::Return, "ru"s }, - { "\"los_portal\""s, ResultType::Return, "los_portal"s }, - { "\"los_level\""s, ResultType::Return, "los_level"s }, - { "\"none\""s, ResultType::Return, "none"s }, - { "\"experiments\""s, ResultType::Return, "experiments"s }, - { "\"lang\""s, ResultType::Return, "lang"s }, - { "\"ru\""s, ResultType::Return, "ru"s }, - { "\"los_portal\""s, ResultType::Return, "los_portal"s }, - { "\"los_level\""s, ResultType::Return, "los_level"s }, - { "\"none\""s, ResultType::Return, "none"s }, - { "\"experiments\""s, ResultType::Return, "experiments"s }, - { "\"lang\""s, ResultType::Return, "lang"s }, - { "\"ru\""s, ResultType::Return, "ru"s }, - { "\"los_portal\""s, ResultType::Return, "los_portal"s }, - { "\"los_level\""s, ResultType::Return, "los_level"s }, - { "\"none\""s, ResultType::Return, "none"s }, - { "\"experiments\""s, ResultType::Return, "experiments"s }, - { "\"lang\""s, ResultType::Return, "lang"s }, - { "\"ru\""s, ResultType::Return, "ru"s }, - { "\"los_portal\""s, ResultType::Return, "los_portal"s }, - { "\"los_level\""s, ResultType::Return, "los_level"s }, - { "\"none\""s, ResultType::Return, "none"s }, - { "\"experiments\""s, ResultType::Return, "experiments"s }, - { "\"lang\""s, ResultType::Return, "lang"s }, - { "\"ru\""s, ResultType::Return, "ru"s }, - { "\"los_portal\""s, ResultType::Return, "los_portal"s }, - { "\"los_level\""s, ResultType::Return, "los_level"s }, - { "\"none\""s, ResultType::Return, "none"s }, - { "\"__ym\""s, ResultType::Return, "__ym"s }, - { "\"ecommerce\""s, ResultType::Return, "ecommerce"s }, - { "\"currencyCode\""s, ResultType::Return, "currencyCode"s }, - { "\"RUR\""s, ResultType::Return, "RUR"s }, - { "\"impressions\""s, ResultType::Return, "impressions"s }, - { "\"name\""s, ResultType::Return, "name"s }, - { "\"Чайник электрический Mystery MEK-1627, белый\""s, ResultType::Return, "Чайник электрический Mystery MEK-1627, белый"s }, - { "\"brand\""s, ResultType::Return, "brand"s }, - { "\"Mystery\""s, ResultType::Return, "Mystery"s }, - { "\"id\""s, ResultType::Return, "id"s }, - { "\"187180\""s, ResultType::Return, "187180"s }, - { "\"category\""s, ResultType::Return, "category"s }, - { "\"Мелкая бытовая техника/Мелкие кухонные приборы/Чайники электрические/Mystery\""s, ResultType::Return, "Мелкая бытовая техника/Мелкие кухонные приборы/Чайники электрические/Mystery"s }, - { "\"variant\""s, ResultType::Return, "variant"s }, - { "\"В наличии\""s, ResultType::Return, "В наличии"s }, - { "\"price\""s, ResultType::Return, "price"s }, - { "\"1630.00\""s, ResultType::Return, "1630.00"s }, - { "\"list\""s, ResultType::Return, "list"s }, - { "\"Карточка\""s, ResultType::Return, "Карточка"s }, - { "\"position\""s, ResultType::Return, "position"s }, - { "\"detail\""s, ResultType::Return, "detail"s }, - { "\"actionField\""s, ResultType::Return, "actionField"s }, - { "\"list\""s, ResultType::Return, "list"s }, + { R"("category")"s, ResultType::Return, "category"s }, + { R"("/Одежда и обувь/Мужская одежда/Футболки/")"s, ResultType::Return, "/Одежда и обувь/Мужская одежда/Футболки/"s }, + { R"("variant")"s, ResultType::Return, "variant"s }, + { R"("")"s, ResultType::Return, ""s }, + { R"("price")"s, ResultType::Return, "price"s }, + { R"("406.60")"s, ResultType::Return, "406.60"s }, + { R"("list")"s, ResultType::Return, "list"s }, + { R"("/retailrocket/")"s, ResultType::Return, "/retailrocket/"s }, + { R"("position")"s, ResultType::Return, "position"s }, + { R"("brand")"s, ResultType::Return, "brand"s }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"s, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"s }, + { R"("id")"s, ResultType::Return, "id"s }, + { R"("913361")"s, ResultType::Return, "913361"s }, + { R"("name")"s, ResultType::Return, "name"s }, + { R"("3Д Футболка детская World р-р 8-10, 100% хлопок, трикотаж")"s, ResultType::Return, "3Д Футболка детская World р-р 8-10, 100% хлопок, трикотаж"s }, + { R"("category")"s, ResultType::Return, "category"s }, + { R"("/Летние товары/Летний текстиль/")"s, ResultType::Return, "/Летние товары/Летний текстиль/"s }, + { R"("variant")"s, ResultType::Return, "variant"s }, + { R"("")"s, ResultType::Return, ""s }, + { R"("price")"s, ResultType::Return, "price"s }, + { R"("470.00")"s, ResultType::Return, "470.00"s }, + { R"("list")"s, ResultType::Return, "list"s }, + { R"("/retailrocket/")"s, ResultType::Return, "/retailrocket/"s }, + { R"("position")"s, ResultType::Return, "position"s }, + { R"("brand")"s, ResultType::Return, "brand"s }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"s, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"s }, + { R"("id")"s, ResultType::Return, "id"s }, + { R"("913364")"s, ResultType::Return, "913364"s }, + { R"("name")"s, ResultType::Return, "name"s }, + { R"("3Д Футболка детская Force р-р 8-10, 100% хлопок, трикотаж")"s, ResultType::Return, "3Д Футболка детская Force р-р 8-10, 100% хлопок, трикотаж"s }, + { R"("category")"s, ResultType::Return, "category"s }, + { R"("/Летние товары/Летний текстиль/")"s, ResultType::Return, "/Летние товары/Летний текстиль/"s }, + { R"("variant")"s, ResultType::Return, "variant"s }, + { R"("")"s, ResultType::Return, ""s }, + { R"("price")"s, ResultType::Return, "price"s }, + { R"("470.00")"s, ResultType::Return, "470.00"s }, + { R"("list")"s, ResultType::Return, "list"s }, + { R"("/retailrocket/")"s, ResultType::Return, "/retailrocket/"s }, + { R"("position")"s, ResultType::Return, "position"s }, + { R"("brand")"s, ResultType::Return, "brand"s }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"s, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"s }, + { R"("id")"s, ResultType::Return, "id"s }, + { R"("913367")"s, ResultType::Return, "913367"s }, + { R"("name")"s, ResultType::Return, "name"s }, + { R"("3Д Футболка детская Winter tale р-р 8-10, 100% хлопок, трикотаж")"s, ResultType::Return, "3Д Футболка детская Winter tale р-р 8-10, 100% хлопок, трикотаж"s }, + { R"("category")"s, ResultType::Return, "category"s }, + { R"("/Летние товары/Летний текстиль/")"s, ResultType::Return, "/Летние товары/Летний текстиль/"s }, + { R"("variant")"s, ResultType::Return, "variant"s }, + { R"("")"s, ResultType::Return, ""s }, + { R"("price")"s, ResultType::Return, "price"s }, + { R"("470.00")"s, ResultType::Return, "470.00"s }, + { R"("list")"s, ResultType::Return, "list"s }, + { R"("/retailrocket/")"s, ResultType::Return, "/retailrocket/"s }, + { R"("position")"s, ResultType::Return, "position"s }, + { R"("brand")"s, ResultType::Return, "brand"s }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"s, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"s }, + { R"("id")"s, ResultType::Return, "id"s }, + { R"("913385")"s, ResultType::Return, "913385"s }, + { R"("name")"s, ResultType::Return, "name"s }, + { R"("3Д Футболка детская Moonshine р-р 8-10, 100% хлопок, трикотаж")"s, ResultType::Return, "3Д Футболка детская Moonshine р-р 8-10, 100% хлопок, трикотаж"s }, + { R"("category")"s, ResultType::Return, "category"s }, + { R"("/Летние товары/Летний текстиль/")"s, ResultType::Return, "/Летние товары/Летний текстиль/"s }, + { R"("variant")"s, ResultType::Return, "variant"s }, + { R"("")"s, ResultType::Return, ""s }, + { R"("price")"s, ResultType::Return, "price"s }, + { R"("470.00")"s, ResultType::Return, "470.00"s }, + { R"("list")"s, ResultType::Return, "list"s }, + { R"("/retailrocket/")"s, ResultType::Return, "/retailrocket/"s }, + { R"("position")"s, ResultType::Return, "position"s }, + { R"("brand")"s, ResultType::Return, "brand"s }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"s, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"s }, + { R"("id")"s, ResultType::Return, "id"s }, + { R"("913391")"s, ResultType::Return, "913391"s }, + { R"("name")"s, ResultType::Return, "name"s }, + { R"("3Д Футболка детская Shaman р-р 8-10, 100% хлопок, трикотаж")"s, ResultType::Return, "3Д Футболка детская Shaman р-р 8-10, 100% хлопок, трикотаж"s }, + { R"("category")"s, ResultType::Return, "category"s }, + { R"("/Летние товары/Летний текстиль/")"s, ResultType::Return, "/Летние товары/Летний текстиль/"s }, + { R"("variant")"s, ResultType::Return, "variant"s }, + { R"("")"s, ResultType::Return, ""s }, + { R"("price")"s, ResultType::Return, "price"s }, + { R"("470.00")"s, ResultType::Return, "470.00"s }, + { R"("list")"s, ResultType::Return, "list"s }, + { R"("/retailrocket/")"s, ResultType::Return, "/retailrocket/"s }, + { R"("position")"s, ResultType::Return, "position"s }, + { R"("brand")"s, ResultType::Return, "brand"s }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"s, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"s }, + { R"("usertype")"s, ResultType::Return, "usertype"s }, + { R"("visitor")"s, ResultType::Return, "visitor"s }, + { R"("lang")"s, ResultType::Return, "lang"s }, + { R"("ru")"s, ResultType::Return, "ru"s }, + { R"("__ym")"s, ResultType::Return, "__ym"s }, + { R"("ecommerce")"s, ResultType::Return, "ecommerce"s }, + { R"("impressions")"s, ResultType::Return, "impressions"s }, + { R"("experiments")"s, ResultType::Return, "experiments"s }, + { R"("lang")"s, ResultType::Return, "lang"s }, + { R"("ru")"s, ResultType::Return, "ru"s }, + { R"("los_portal")"s, ResultType::Return, "los_portal"s }, + { R"("los_level")"s, ResultType::Return, "los_level"s }, + { R"("none")"s, ResultType::Return, "none"s }, + { R"("experiments")"s, ResultType::Return, "experiments"s }, + { R"("lang")"s, ResultType::Return, "lang"s }, + { R"("ru")"s, ResultType::Return, "ru"s }, + { R"("los_portal")"s, ResultType::Return, "los_portal"s }, + { R"("los_level")"s, ResultType::Return, "los_level"s }, + { R"("none")"s, ResultType::Return, "none"s }, + { R"("experiments")"s, ResultType::Return, "experiments"s }, + { R"("lang")"s, ResultType::Return, "lang"s }, + { R"("ru")"s, ResultType::Return, "ru"s }, + { R"("los_portal")"s, ResultType::Return, "los_portal"s }, + { R"("los_level")"s, ResultType::Return, "los_level"s }, + { R"("none")"s, ResultType::Return, "none"s }, + { R"("experiments")"s, ResultType::Return, "experiments"s }, + { R"("lang")"s, ResultType::Return, "lang"s }, + { R"("ru")"s, ResultType::Return, "ru"s }, + { R"("los_portal")"s, ResultType::Return, "los_portal"s }, + { R"("los_level")"s, ResultType::Return, "los_level"s }, + { R"("none")"s, ResultType::Return, "none"s }, + { R"("experiments")"s, ResultType::Return, "experiments"s }, + { R"("lang")"s, ResultType::Return, "lang"s }, + { R"("ru")"s, ResultType::Return, "ru"s }, + { R"("los_portal")"s, ResultType::Return, "los_portal"s }, + { R"("los_level")"s, ResultType::Return, "los_level"s }, + { R"("none")"s, ResultType::Return, "none"s }, + { R"("__ym")"s, ResultType::Return, "__ym"s }, + { R"("ecommerce")"s, ResultType::Return, "ecommerce"s }, + { R"("currencyCode")"s, ResultType::Return, "currencyCode"s }, + { R"("RUR")"s, ResultType::Return, "RUR"s }, + { R"("impressions")"s, ResultType::Return, "impressions"s }, + { R"("name")"s, ResultType::Return, "name"s }, + { R"("Чайник электрический Mystery MEK-1627, белый")"s, ResultType::Return, "Чайник электрический Mystery MEK-1627, белый"s }, + { R"("brand")"s, ResultType::Return, "brand"s }, + { R"("Mystery")"s, ResultType::Return, "Mystery"s }, + { R"("id")"s, ResultType::Return, "id"s }, + { R"("187180")"s, ResultType::Return, "187180"s }, + { R"("category")"s, ResultType::Return, "category"s }, + { R"("Мелкая бытовая техника/Мелкие кухонные приборы/Чайники электрические/Mystery")"s, ResultType::Return, "Мелкая бытовая техника/Мелкие кухонные приборы/Чайники электрические/Mystery"s }, + { R"("variant")"s, ResultType::Return, "variant"s }, + { R"("В наличии")"s, ResultType::Return, "В наличии"s }, + { R"("price")"s, ResultType::Return, "price"s }, + { R"("1630.00")"s, ResultType::Return, "1630.00"s }, + { R"("list")"s, ResultType::Return, "list"s }, + { R"("Карточка")"s, ResultType::Return, "Карточка"s }, + { R"("position")"s, ResultType::Return, "position"s }, + { R"("detail")"s, ResultType::Return, "detail"s }, + { R"("actionField")"s, ResultType::Return, "actionField"s }, + { R"("list")"s, ResultType::Return, "list"s }, { "\0\""s, ResultType::Throw, "JSON: expected \", got \0"s }, { "\"/igrushki/konstruktory\0"s, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."s }, { "\"/1290414/komplekt-zhenskiy-dzhemper-plusbryuki-m-254-09-malina-plustemno-siniy-\0a"s, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."s }, diff --git a/base/mysqlxx/src/tests/mysqlxx_test.cpp b/base/mysqlxx/src/tests/mysqlxx_test.cpp index d99900b1a39..e5e2ec1f592 100644 --- a/base/mysqlxx/src/tests/mysqlxx_test.cpp +++ b/base/mysqlxx/src/tests/mysqlxx_test.cpp @@ -68,10 +68,10 @@ int main(int, char **) Queries queries; queries.push_back(query); - for (Queries::iterator it = queries.begin(); it != queries.end(); ++it) + for (auto & query : queries) { - std::cerr << it->str() << std::endl; - std::cerr << it->store().at(0) << std::endl; + std::cerr << query.str() << std::endl; + std::cerr << query.store().at(0) << std::endl; } } @@ -92,10 +92,10 @@ int main(int, char **) mysqlxx::Query & qref = queries.back(); qref << " 1"; - for (Queries::iterator it = queries.begin(); it != queries.end(); ++it) + for (auto & query : queries) { - std::cerr << it->str() << std::endl; - std::cerr << it->store().at(0) << std::endl; + std::cerr << query.str() << std::endl; + std::cerr << query.store().at(0) << std::endl; } } diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp index fad9d9ff488..a7821519fc2 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp @@ -199,19 +199,19 @@ void IMergeTreeDataPartWriter::initSkipIndices() skip_indices_initialized = true; } -void IMergeTreeDataPartWriter::calculateAndSerializePrimaryIndex(const Block & primary_keys_block, size_t rows) +void IMergeTreeDataPartWriter::calculateAndSerializePrimaryIndex(const Block & primary_index_block, size_t rows) { if (!primary_index_initialized) throw Exception("Primary index is not initialized", ErrorCodes::LOGICAL_ERROR); - size_t primary_columns_num = primary_keys_block.columns(); + size_t primary_columns_num = primary_index_block.columns(); if (index_columns.empty()) { - index_types = primary_keys_block.getDataTypes(); + index_types = primary_index_block.getDataTypes(); index_columns.resize(primary_columns_num); last_index_row.resize(primary_columns_num); for (size_t i = 0; i < primary_columns_num; ++i) - index_columns[i] = primary_keys_block.getByPosition(i).column->cloneEmpty(); + index_columns[i] = primary_index_block.getByPosition(i).column->cloneEmpty(); } /** While filling index (index_columns), disable memory tracker. @@ -230,7 +230,7 @@ void IMergeTreeDataPartWriter::calculateAndSerializePrimaryIndex(const Block & p { for (size_t j = 0; j < primary_columns_num; ++j) { - const auto & primary_column = primary_keys_block.getByPosition(j); + const auto & primary_column = primary_index_block.getByPosition(j); index_columns[j]->insertFrom(*primary_column.column, i); primary_column.type->serializeBinary(*primary_column.column, i, *index_stream); } @@ -244,7 +244,7 @@ void IMergeTreeDataPartWriter::calculateAndSerializePrimaryIndex(const Block & p /// store last index row to write final mark at the end of column for (size_t j = 0; j < primary_columns_num; ++j) { - const IColumn & primary_column = *primary_keys_block.getByPosition(j).column.get(); + const IColumn & primary_column = *primary_index_block.getByPosition(j).column.get(); primary_column.get(rows - 1, last_index_row[j]); } } diff --git a/dbms/src/Storages/MergeTree/KeyCondition.cpp b/dbms/src/Storages/MergeTree/KeyCondition.cpp index a936ead568d..729f3035011 100644 --- a/dbms/src/Storages/MergeTree/KeyCondition.cpp +++ b/dbms/src/Storages/MergeTree/KeyCondition.cpp @@ -351,8 +351,7 @@ FieldWithInfinity::FieldWithInfinity(Field && field_) } FieldWithInfinity::FieldWithInfinity(const Type type_) - : field(), - type(type_) + : type(type_) { } @@ -722,10 +721,7 @@ bool KeyCondition::isKeyPossiblyWrappedByMonotonicFunctionsImpl( out_functions_chain.push_back(func); - if (!isKeyPossiblyWrappedByMonotonicFunctionsImpl(args[0], out_key_column_num, out_key_column_type, out_functions_chain)) - return false; - - return true; + return isKeyPossiblyWrappedByMonotonicFunctionsImpl(args[0], out_key_column_num, out_key_column_type, out_functions_chain); } return false; @@ -1131,9 +1127,8 @@ BoolMask KeyCondition::checkInParallelogram( const DataTypes & data_types) const { std::vector rpn_stack; - for (size_t i = 0; i < rpn.size(); ++i) + for (const auto & element : rpn) { - const auto & element = rpn[i]; if (element.function == RPNElement::FUNCTION_UNKNOWN) { rpn_stack.emplace_back(true, true); diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index d44501adab6..329d4453e49 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -663,7 +663,7 @@ void MergeTreeData::setTTLExpressions(const ColumnsDescription::ColumnTTLs & new TTLEntry update_rows_ttl_entry; bool seen_delete_ttl = false; - for (auto ttl_element_ptr : new_ttl_table_ast->children) + for (const auto & ttl_element_ptr : new_ttl_table_ast->children) { const auto * ttl_element = ttl_element_ptr->as(); if (!ttl_element) @@ -1822,7 +1822,7 @@ void MergeTreeData::alterDataPart( /// Update the checksums. DataPart::Checksums new_checksums = part->checksums; - for (auto it : transaction->rename_map) + for (const auto & it : transaction->rename_map) { if (it.second.empty()) new_checksums.files.erase(it.first); @@ -1846,8 +1846,6 @@ void MergeTreeData::alterDataPart( transaction->new_columns.writeText(columns_file); transaction->rename_map["columns.txt.tmp"] = "columns.txt"; } - - return; } void MergeTreeData::changeSettings( @@ -2688,7 +2686,7 @@ MergeTreeData::DataPartPtr MergeTreeData::getActiveContainingPart( void MergeTreeData::swapActivePart(MergeTreeData::DataPartPtr part_copy) { auto lock = lockParts(); - for (auto original_active_part : getDataPartsStateRange(DataPartState::Committed)) + for (const auto & original_active_part : getDataPartsStateRange(DataPartState::Committed)) { if (part_copy->name == original_active_part->name) { @@ -3810,10 +3808,7 @@ bool MergeTreeData::areBackgroundMovesNeeded() const if (policy->getVolumes().size() > 1) return true; - if (policy->getVolumes().size() == 1 && policy->getVolumes()[0]->disks.size() > 1 && move_ttl_entries.size() > 0) - return true; - - return false; + return policy->getVolumes().size() == 1 && policy->getVolumes()[0]->disks.size() > 1 && !move_ttl_entries.empty(); } bool MergeTreeData::movePartsToSpace(const DataPartsVector & parts, SpacePtr space) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index 144729caa53..9d1118fdcd9 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -547,7 +547,7 @@ public: /// Moves the entire data directory. /// Flushes the uncompressed blocks cache and the marks cache. /// Must be called with locked lockStructureForAlter(). - void rename(const String & new_path_to_table_data, const String & new_database_name, + void rename(const String & new_table_path, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override; /// Check if the ALTER can be performed: @@ -569,7 +569,7 @@ public: /// Change MergeTreeSettings void changeSettings( - const ASTPtr & new_changes, + const ASTPtr & new_settings, TableStructureWriteLockHolder & table_lock_holder); /// Remove columns, that have been marked as empty after zeroing values with expired ttl @@ -638,7 +638,7 @@ public: } /// For ATTACH/DETACH/DROP PARTITION. - String getPartitionIDFromQuery(const ASTPtr & partition, const Context & context); + String getPartitionIDFromQuery(const ASTPtr & ast, const Context & context); /// Extracts MergeTreeData of other *MergeTree* storage /// and checks that their structure suitable for ALTER TABLE ATTACH PARTITION FROM @@ -957,7 +957,7 @@ protected: using MatcherFn = std::function; void freezePartitionsByMatcher(MatcherFn matcher, const String & with_name, const Context & context); - bool canReplacePartition(const DataPartPtr & data_part) const; + bool canReplacePartition(const DataPartPtr & src_part) const; void writePartLog( PartLogElement::Type type, @@ -996,7 +996,7 @@ private: }; /// Move selected parts to corresponding disks - bool moveParts(CurrentlyMovingPartsTagger && parts_to_move); + bool moveParts(CurrentlyMovingPartsTagger && moving_tagger); /// Select parts for move and disks for them. Used in background moving processes. CurrentlyMovingPartsTagger selectPartsForMove(); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 509194579e8..f7f68e95130 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -378,9 +378,8 @@ MergeTreeData::DataPartsVector MergeTreeDataMergerMutator::selectAllPartsFromPar MergeTreeData::DataParts data_parts = data.getDataParts(); - for (MergeTreeData::DataParts::iterator it = data_parts.cbegin(); it != data_parts.cend(); ++it) + for (const auto & current_part : data_parts) { - const MergeTreeData::DataPartPtr & current_part = *it; if (current_part->info.partition_id != partition_id) continue; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.h b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.h index 79b189b65fb..d55298bb944 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.h @@ -108,7 +108,7 @@ public: MergeTreeData::MutableDataPartPtr mergePartsToTemporaryPart( const FutureMergedMutatedPart & future_part, MergeListEntry & merge_entry, TableStructureReadLockHolder & table_lock_holder, time_t time_of_merge, - const ReservationPtr & disk_reservation, bool deduplication, bool force_ttl); + const ReservationPtr & space_reservation, bool deduplicate, bool force_ttl); /// Mutate a single data part with the specified commands. Will create and return a temporary part. MergeTreeData::MutableDataPartPtr mutatePartToTemporaryPart( @@ -117,7 +117,7 @@ public: MergeListEntry & merge_entry, time_t time_of_mutation, const Context & context, - const ReservationPtr & disk_reservation, + const ReservationPtr & space_reservation, TableStructureReadLockHolder & table_lock_holder); MergeTreeData::DataPartPtr renameMergedTemporaryPart( diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.h index 051e54d5139..a4be6382b7e 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.h @@ -72,9 +72,9 @@ struct MergeTreeDataPartChecksums bool read(ReadBuffer & in, size_t format_version); bool read_v2(ReadBuffer & in); bool read_v3(ReadBuffer & in); - bool read_v4(ReadBuffer & in); + bool read_v4(ReadBuffer & from); - void write(WriteBuffer & out) const; + void write(WriteBuffer & to) const; /// Checksum from the set of checksums of .bin files (for deduplication). void computeTotalChecksumDataOnly(SipHash & hash) const; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.cpp index 05598c00aa7..5d48867c5b3 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.cpp @@ -36,7 +36,7 @@ void MergeTreeDataPartTTLInfos::read(ReadBuffer & in) if (json.has("columns")) { const JSON & columns = json["columns"]; - for (auto col : columns) + for (auto col : columns) // NOLINT { MergeTreeDataPartTTLInfo ttl_info; ttl_info.min = col["min"].getUInt(); @@ -58,7 +58,7 @@ void MergeTreeDataPartTTLInfos::read(ReadBuffer & in) if (json.has("moves")) { const JSON & moves = json["moves"]; - for (auto move : moves) + for (auto move : moves) // NOLINT { MergeTreeDataPartTTLInfo ttl_info; ttl_info.min = move["min"].getUInt(); From d3aa0e8ed80effd391ff3dd3d1ec812f1fa1d8b7 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 9 Mar 2020 04:59:08 +0300 Subject: [PATCH 148/712] clang-tidy, part 14 --- base/common/tests/gtest_json_test.cpp | 2 +- .../tests/integer_hash_tables_and_hashes.cpp | 4 +- dbms/src/IO/tests/read_buffer_aio.cpp | 40 +++++++++---------- .../MergeTree/MergeTreeDataPartTTLInfo.cpp | 12 +++--- .../MergeTree/MergeTreeDataPartWriterWide.cpp | 8 ++-- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 27 +++++-------- .../TableFunctionGenerateRandom.cpp | 2 +- 7 files changed, 45 insertions(+), 50 deletions(-) diff --git a/base/common/tests/gtest_json_test.cpp b/base/common/tests/gtest_json_test.cpp index d0393a850aa..5bdeb1bca9b 100644 --- a/base/common/tests/gtest_json_test.cpp +++ b/base/common/tests/gtest_json_test.cpp @@ -22,7 +22,7 @@ struct GetStringTestRecord std::string result; }; -TEST(JSON_Suite, SimpleTest) +TEST(JSONSuite, SimpleTest) { std::vector test_data = { diff --git a/dbms/src/Common/tests/integer_hash_tables_and_hashes.cpp b/dbms/src/Common/tests/integer_hash_tables_and_hashes.cpp index 198beae5789..29e4a31bfb3 100644 --- a/dbms/src/Common/tests/integer_hash_tables_and_hashes.cpp +++ b/dbms/src/Common/tests/integer_hash_tables_and_hashes.cpp @@ -272,9 +272,9 @@ namespace Hashes }; size_t res = 0; - for (size_t i = 0; i < 8; ++i) + for (auto & rand : random) { - res ^= random[i][UInt8(x)]; + res ^= rand[UInt8(x)]; x >>= 8; } diff --git a/dbms/src/IO/tests/read_buffer_aio.cpp b/dbms/src/IO/tests/read_buffer_aio.cpp index 8c99ed26179..daa9c10c590 100644 --- a/dbms/src/IO/tests/read_buffer_aio.cpp +++ b/dbms/src/IO/tests/read_buffer_aio.cpp @@ -68,26 +68,26 @@ void run() const std::vector> tests = { - std::bind(test1, std::ref(filename)), - std::bind(test2, std::ref(filename), std::ref(buf)), - std::bind(test3, std::ref(filename), std::ref(buf)), - std::bind(test4, std::ref(filename), std::ref(buf)), - std::bind(test5, std::ref(filename), std::ref(buf)), - std::bind(test6, std::ref(filename), std::ref(buf)), - std::bind(test7, std::ref(filename), std::ref(buf)), - std::bind(test8, std::ref(filename), std::ref(buf)), - std::bind(test9, std::ref(filename), std::ref(buf)), - std::bind(test10, std::ref(filename), std::ref(buf)), - std::bind(test11, std::ref(filename)), - std::bind(test12, std::ref(filename), std::ref(buf)), - std::bind(test13, std::ref(filename2), std::ref(buf2)), - std::bind(test14, std::ref(filename), std::ref(buf)), - std::bind(test15, std::ref(filename3), std::ref(buf3)), - std::bind(test16, std::ref(filename3), std::ref(buf3)), - std::bind(test17, std::ref(filename4), std::ref(buf4)), - std::bind(test18, std::ref(filename5), std::ref(buf5)), - std::bind(test19, std::ref(filename), std::ref(buf)), - std::bind(test20, std::ref(filename), std::ref(buf)) + [&]{ return test1(filename); }, + [&]{ return test2(filename, buf); }, + [&]{ return test3(filename, buf); }, + [&]{ return test4(filename, buf); }, + [&]{ return test5(filename, buf); }, + [&]{ return test6(filename, buf); }, + [&]{ return test7(filename, buf); }, + [&]{ return test8(filename, buf); }, + [&]{ return test9(filename, buf); }, + [&]{ return test10(filename, buf); }, + [&]{ return test11(filename); }, + [&]{ return test12(filename, buf); }, + [&]{ return test13(filename2, buf2); }, + [&]{ return test14(filename, buf); }, + [&]{ return test15(filename3, buf3); }, + [&]{ return test16(filename3, buf3); }, + [&]{ return test17(filename4, buf4); }, + [&]{ return test18(filename5, buf5); }, + [&]{ return test19(filename, buf); }, + [&]{ return test20(filename, buf); } }; unsigned int num = 0; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.cpp index 5d48867c5b3..37d036fc6fc 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.cpp @@ -96,9 +96,9 @@ void MergeTreeDataPartTTLInfos::write(WriteBuffer & out) const { if (!columns_ttl.empty()) writeString(",", out); - writeString("\"table\":{\"min\":", out); + writeString(R"("table":{"min":)", out); writeIntText(table_ttl.min, out); - writeString(",\"max\":", out); + writeString(R"(,"max":)", out); writeIntText(table_ttl.max, out); writeString("}", out); } @@ -106,17 +106,17 @@ void MergeTreeDataPartTTLInfos::write(WriteBuffer & out) const { if (!columns_ttl.empty() || table_ttl.min) writeString(",", out); - writeString("\"moves\":[", out); + writeString(R"("moves":[)", out); for (auto it = moves_ttl.begin(); it != moves_ttl.end(); ++it) { if (it != moves_ttl.begin()) writeString(",", out); - writeString("{\"expression\":", out); + writeString(R"({"expression":)", out); writeString(doubleQuoteString(it->first), out); - writeString(",\"min\":", out); + writeString(R"(,"min":)", out); writeIntText(it->second.min, out); - writeString(",\"max\":", out); + writeString(R"(,"max":)", out); writeIntText(it->second.max, out); writeString("}", out); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index 0b170312d37..c68a44250aa 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -295,12 +295,12 @@ void MergeTreeDataPartWriterWide::finishDataSerialization(IMergeTreeDataPart::Ch } } - for (auto it = column_streams.begin(); it != column_streams.end(); ++it) + for (auto & stream : column_streams) { - it->second->finalize(); + stream.second->finalize(); if (sync) - it->second->sync(); - it->second->addToChecksums(checksums); + stream.second->sync(); + stream.second->addToChecksums(checksums); } column_streams.clear(); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index f3fd3be8e7f..f377124f1af 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -109,18 +109,17 @@ size_t MergeTreeDataSelectExecutor::getApproximateTotalRowsToRead( /// We will find out how many rows we would have read without sampling. LOG_DEBUG(log, "Preliminary index scan with condition: " << key_condition.toString()); - for (size_t i = 0; i < parts.size(); ++i) + for (const auto & part : parts) { - const MergeTreeData::DataPartPtr & part = parts[i]; MarkRanges ranges = markRangesFromPKRange(part, key_condition, settings); /** In order to get a lower bound on the number of rows that match the condition on PK, * consider only guaranteed full marks. * That is, do not take into account the first and last marks, which may be incomplete. */ - for (size_t j = 0; j < ranges.size(); ++j) - if (ranges[j].end - ranges[j].begin > 2) - rows_count += part->index_granularity.getRowsCountInRange({ranges[j].begin + 1, ranges[j].end - 1}); + for (const auto & range : ranges) + if (range.end - range.begin > 2) + rows_count += part->index_granularity.getRowsCountInRange({range.begin + 1, range.end - 1}); } @@ -794,10 +793,8 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreams( { /// Sequential query execution. - for (size_t part_index = 0; part_index < parts.size(); ++part_index) + for (const auto & part : parts) { - RangesInDataPart & part = parts[part_index]; - auto source = std::make_shared( data, part.data_part, max_block_size, settings.preferred_block_size_bytes, settings.preferred_max_column_in_block_size_bytes, column_names, part.ranges, use_uncompressed_cache, @@ -1025,13 +1022,13 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal( const auto data_settings = data.getSettings(); size_t sum_marks = 0; size_t adaptive_parts = 0; - for (size_t i = 0; i < parts.size(); ++i) + for (const auto & part : parts) { - for (size_t j = 0; j < parts[i].ranges.size(); ++j) - sum_marks += parts[i].ranges[j].end - parts[i].ranges[j].begin; + for (const auto & range : part.ranges) + sum_marks += range.end - range.begin; - if (parts[i].data_part->index_granularity_info.is_adaptive) - adaptive_parts++; + if (part.data_part->index_granularity_info.is_adaptive) + ++adaptive_parts; } size_t index_granularity_bytes = 0; @@ -1049,10 +1046,8 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal( Pipes pipes; - for (size_t part_index = 0; part_index < parts.size(); ++part_index) + for (const auto & part : parts) { - RangesInDataPart & part = parts[part_index]; - auto source_processor = std::make_shared( data, part.data_part, max_block_size, settings.preferred_block_size_bytes, settings.preferred_max_column_in_block_size_bytes, column_names, part.ranges, use_uncompressed_cache, diff --git a/dbms/src/TableFunctions/TableFunctionGenerateRandom.cpp b/dbms/src/TableFunctions/TableFunctionGenerateRandom.cpp index 7b5177b0090..3b3db1c2510 100644 --- a/dbms/src/TableFunctions/TableFunctionGenerateRandom.cpp +++ b/dbms/src/TableFunctions/TableFunctionGenerateRandom.cpp @@ -34,7 +34,7 @@ StoragePtr TableFunctionGenerateRandom::executeImpl(const ASTPtr & ast_function, ASTs & args = args_func.at(0)->children; - if (args.size() < 1) + if (args.empty()) throw Exception("Table function '" + getName() + "' requires at least one argument: " " structure, [random_seed, max_string_length, max_array_length].", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); From 04239a0f5c615436641085acbe369e981ff57489 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 9 Mar 2020 05:05:04 +0300 Subject: [PATCH 149/712] clang-tidy, part 15 --- dbms/src/Columns/tests/gtest_column_unique.cpp | 10 +++++----- .../tests/gtest_zkutil_test_multi_exception.cpp | 10 +++++----- .../MergeTree/MergeTreeIndexAggregatorBloomFilter.cpp | 4 ++-- dbms/src/Storages/MergeTree/MergeTreeIndexFullText.cpp | 9 ++++----- .../Storages/MergeTree/MergeTreeIndexGranularityInfo.h | 2 +- 5 files changed, 17 insertions(+), 18 deletions(-) diff --git a/dbms/src/Columns/tests/gtest_column_unique.cpp b/dbms/src/Columns/tests/gtest_column_unique.cpp index 6c57f27aba6..601d34e6302 100644 --- a/dbms/src/Columns/tests/gtest_column_unique.cpp +++ b/dbms/src/Columns/tests/gtest_column_unique.cpp @@ -13,7 +13,7 @@ #include using namespace DB; -TEST(column_unique, column_unique_unique_insert_range_Test) +TEST(ColumnUnique, InsertRange) { std::unordered_map ref_map; auto data_type = std::make_shared(); @@ -52,7 +52,7 @@ TEST(column_unique, column_unique_unique_insert_range_Test) } } -TEST(column_unique, column_unique_unique_insert_range_with_overflow_Test) +TEST(ColumnUnique, InsertRangeWithOverflow) { std::unordered_map ref_map; auto data_type = std::make_shared(); @@ -147,7 +147,7 @@ void column_unique_unique_deserialize_from_arena_impl(ColumnType & column, const } } -TEST(column_unique, column_unique_unique_deserialize_from_arena_String_Test) +TEST(ColumnUnique, DeserializeFromArenaString) { auto data_type = std::make_shared(); auto column_string = ColumnString::create(); @@ -165,7 +165,7 @@ TEST(column_unique, column_unique_unique_deserialize_from_arena_String_Test) column_unique_unique_deserialize_from_arena_impl(*column_string, *data_type); } -TEST(column_unique, column_unique_unique_deserialize_from_arena_Nullable_String_Test) +TEST(ColumnUnique, DeserializeFromArenaNullableString) { auto data_type = std::make_shared(std::make_shared()); auto column_string = ColumnString::create(); @@ -187,7 +187,7 @@ TEST(column_unique, column_unique_unique_deserialize_from_arena_Nullable_String_ column_unique_unique_deserialize_from_arena_impl(*column, *data_type); } -TEST(ColumnVector, correctness_of_replicate) +TEST(ColumnVector, CorrectnessOfReplicate) { const auto column = ColumnUInt8::create(); diff --git a/dbms/src/Common/ZooKeeper/tests/gtest_zkutil_test_multi_exception.cpp b/dbms/src/Common/ZooKeeper/tests/gtest_zkutil_test_multi_exception.cpp index bc5165986d0..9437ae6aef7 100644 --- a/dbms/src/Common/ZooKeeper/tests/gtest_zkutil_test_multi_exception.cpp +++ b/dbms/src/Common/ZooKeeper/tests/gtest_zkutil_test_multi_exception.cpp @@ -12,7 +12,7 @@ using namespace DB; -TEST(zkutil, zookeeper_connected) +TEST(zkutil, ZookeeperConnected) { try { @@ -26,7 +26,7 @@ TEST(zkutil, zookeeper_connected) } } -TEST(zkutil, multi_nice_exception_msg) +TEST(zkutil, MultiNiceExceptionMsg) { auto zookeeper = std::make_unique("localhost:2181"); @@ -64,7 +64,7 @@ TEST(zkutil, multi_nice_exception_msg) } -TEST(zkutil, multi_async) +TEST(zkutil, MultiAsync) { auto zookeeper = std::make_unique("localhost:2181"); Coordination::Requests ops; @@ -126,7 +126,7 @@ TEST(zkutil, multi_async) } } -TEST(zkutil, watch_get_children_with_chroot) +TEST(zkutil, WatchGetChildrenWithChroot) { try { @@ -163,7 +163,7 @@ TEST(zkutil, watch_get_children_with_chroot) } } -TEST(zkutil, multi_create_sequential) +TEST(zkutil, MultiCreateSequential) { try { diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexAggregatorBloomFilter.cpp b/dbms/src/Storages/MergeTree/MergeTreeIndexAggregatorBloomFilter.cpp index ce49e1ff858..d7d9bd9c525 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexAggregatorBloomFilter.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexAggregatorBloomFilter.cpp @@ -46,9 +46,9 @@ void MergeTreeIndexAggregatorBloomFilter::update(const Block & block, size_t * p Block granule_index_block; size_t max_read_rows = std::min(block.rows() - *pos, limit); - for (size_t index = 0; index < index_columns_name.size(); ++index) + for (const auto & index_column_name : index_columns_name) { - const auto & column_and_type = block.getByName(index_columns_name[index]); + const auto & column_and_type = block.getByName(index_column_name); auto index_column = BloomFilterHash::hashWithColumn(column_and_type.type, column_and_type.column, *pos, max_read_rows); granule_index_block.insert({index_column, std::make_shared(), column_and_type.name}); diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexFullText.cpp b/dbms/src/Storages/MergeTree/MergeTreeIndexFullText.cpp index 0edbbfcc156..6d3b830a3f4 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexFullText.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexFullText.cpp @@ -63,8 +63,7 @@ bool MergeTreeConditionFullText::createFunctionEqualsCondition(RPNElement & out, } MergeTreeIndexGranuleFullText::MergeTreeIndexGranuleFullText(const MergeTreeIndexFullText & index_) - : IMergeTreeIndexGranule() - , index(index_) + : index(index_) , bloom_filters( index.columns.size(), BloomFilter(index.bloom_filter_size, index.bloom_filter_hashes, index.seed)) , has_elems(false) {} @@ -524,12 +523,12 @@ bool MergeTreeConditionFullText::tryPrepareSetBloomFilter( std::vector key_position; Columns columns = prepared_set->getSetElements(); - for (size_t col = 0; col < key_tuple_mapping.size(); ++col) + for (const auto & elem : key_tuple_mapping) { bloom_filters.emplace_back(); - key_position.push_back(key_tuple_mapping[col].key_index); + key_position.push_back(elem.key_index); - size_t tuple_idx = key_tuple_mapping[col].tuple_index; + size_t tuple_idx = elem.tuple_index; const auto & column = columns[tuple_idx]; for (size_t row = 0; row < prepared_set->getTotalRowCount(); ++row) { diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h index 7bcb208980e..fe1a965f843 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h @@ -36,7 +36,7 @@ public: size_t getMarkSizeInBytes(size_t columns_num = 1) const; - static std::optional getMrkExtensionFromFS(const std::string & path_to_table); + static std::optional getMrkExtensionFromFS(const std::string & path_to_part); private: MergeTreeDataPartType type; From d1025b2dc79438daf2ae3ce58d1b11fd9cb5e763 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 9 Mar 2020 05:10:20 +0300 Subject: [PATCH 150/712] clang-tidy, part 16 --- dbms/src/Common/SensitiveDataMasker.cpp | 2 ++ dbms/src/Common/SensitiveDataMasker.h | 1 + dbms/src/Common/tests/gtest_pod_array.cpp | 2 +- dbms/src/Common/tests/gtest_rw_lock.cpp | 8 +++---- .../tests/gtest_compressionCodec.cpp | 10 ++++---- .../src/Core/tests/gtest_DecimalFunctions.cpp | 24 +++++++++---------- dbms/src/Disks/tests/gtest_disk.cpp | 2 +- dbms/src/IO/tests/read_buffer_aio.cpp | 4 +--- 8 files changed, 27 insertions(+), 26 deletions(-) diff --git a/dbms/src/Common/SensitiveDataMasker.cpp b/dbms/src/Common/SensitiveDataMasker.cpp index a57ed152c6d..8a0f9361207 100644 --- a/dbms/src/Common/SensitiveDataMasker.cpp +++ b/dbms/src/Common/SensitiveDataMasker.cpp @@ -78,6 +78,8 @@ public: }; +SensitiveDataMasker::~SensitiveDataMasker() = default; + std::unique_ptr SensitiveDataMasker::sensitive_data_masker = nullptr; void SensitiveDataMasker::setInstance(std::unique_ptr sensitive_data_masker_) diff --git a/dbms/src/Common/SensitiveDataMasker.h b/dbms/src/Common/SensitiveDataMasker.h index ebf7a587fdf..99a5c8c72b4 100644 --- a/dbms/src/Common/SensitiveDataMasker.h +++ b/dbms/src/Common/SensitiveDataMasker.h @@ -49,6 +49,7 @@ private: public: SensitiveDataMasker(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix); + ~SensitiveDataMasker(); /// Returns the number of matched rules. size_t wipeSensitiveData(std::string & data) const; diff --git a/dbms/src/Common/tests/gtest_pod_array.cpp b/dbms/src/Common/tests/gtest_pod_array.cpp index 7962bf39f07..2b7c5098b48 100644 --- a/dbms/src/Common/tests/gtest_pod_array.cpp +++ b/dbms/src/Common/tests/gtest_pod_array.cpp @@ -4,7 +4,7 @@ using namespace DB; -TEST(Common, PODArray_Insert) +TEST(Common, PODArrayInsert) { std::string str = "test_string_abacaba"; PODArray chars; diff --git a/dbms/src/Common/tests/gtest_rw_lock.cpp b/dbms/src/Common/tests/gtest_rw_lock.cpp index ebb82f26c0a..81bc0d38a56 100644 --- a/dbms/src/Common/tests/gtest_rw_lock.cpp +++ b/dbms/src/Common/tests/gtest_rw_lock.cpp @@ -23,7 +23,7 @@ namespace DB } -TEST(Common, RWLock_1) +TEST(Common, RWLock1) { constexpr int cycles = 1000; const std::vector pool_sizes{1, 2, 4, 8}; @@ -90,7 +90,7 @@ TEST(Common, RWLock_1) } } -TEST(Common, RWLock_Recursive) +TEST(Common, RWLockRecursive) { constexpr auto cycles = 10000; @@ -132,7 +132,7 @@ TEST(Common, RWLock_Recursive) } -TEST(Common, RWLock_Deadlock) +TEST(Common, RWLockDeadlock) { static auto lock1 = RWLockImpl::create(); static auto lock2 = RWLockImpl::create(); @@ -200,7 +200,7 @@ TEST(Common, RWLock_Deadlock) } -TEST(Common, RWLock_PerfTest_Readers) +TEST(Common, RWLockPerfTestReaders) { constexpr int cycles = 100000; // 100k const std::vector pool_sizes{1, 2, 4, 8}; diff --git a/dbms/src/Compression/tests/gtest_compressionCodec.cpp b/dbms/src/Compression/tests/gtest_compressionCodec.cpp index cffa638beb3..3808e23ab65 100644 --- a/dbms/src/Compression/tests/gtest_compressionCodec.cpp +++ b/dbms/src/Compression/tests/gtest_compressionCodec.cpp @@ -547,11 +547,11 @@ TEST_P(CodecTest, TranscodingWithoutDataType) } // Param is tuple-of-tuple to simplify instantiating with values, since typically group of cases test only one codec. -class CodecTest_Compatibility : public ::testing::TestWithParam>> +class CodecTestCompatibility : public ::testing::TestWithParam>> {}; // Check that iput sequence when encoded matches the encoded string binary. -TEST_P(CodecTest_Compatibility, Encoding) +TEST_P(CodecTestCompatibility, Encoding) { const auto & codec_spec = std::get<0>(GetParam()); const auto & [data_sequence, expected] = std::get<1>(GetParam()); @@ -571,7 +571,7 @@ TEST_P(CodecTest_Compatibility, Encoding) } // Check that binary string is exactly decoded into input sequence. -TEST_P(CodecTest_Compatibility, Decoding) +TEST_P(CodecTestCompatibility, Decoding) { const auto & codec_spec = std::get<0>(GetParam()); const auto & [expected, encoded_data] = std::get<1>(GetParam()); @@ -1159,7 +1159,7 @@ auto DDCompatibilityTestSequence() #define BIN_STR(x) std::string{x, sizeof(x) - 1} INSTANTIATE_TEST_SUITE_P(DoubleDelta, - CodecTest_Compatibility, + CodecTestCompatibility, ::testing::Combine( ::testing::Values(Codec("DoubleDelta")), ::testing::ValuesIn(std::initializer_list>{ @@ -1233,7 +1233,7 @@ auto GCompatibilityTestSequence() } INSTANTIATE_TEST_SUITE_P(Gorilla, - CodecTest_Compatibility, + CodecTestCompatibility, ::testing::Combine( ::testing::Values(Codec("Gorilla")), ::testing::ValuesIn(std::initializer_list>{ diff --git a/dbms/src/Core/tests/gtest_DecimalFunctions.cpp b/dbms/src/Core/tests/gtest_DecimalFunctions.cpp index bc849a66ada..61fdeb315ac 100644 --- a/dbms/src/Core/tests/gtest_DecimalFunctions.cpp +++ b/dbms/src/Core/tests/gtest_DecimalFunctions.cpp @@ -57,62 +57,62 @@ void testGetFractional(const DecimalUtilsSplitAndCombineTestParam & param) } // unfortunatelly typed parametrized tests () are not supported in this version of gtest, so I have to emulate by hand. -TEST_P(DecimalUtilsSplitAndCombineTest, split_Decimal32) +TEST_P(DecimalUtilsSplitAndCombineTest, splitDecimal32) { testSplit(GetParam()); } -TEST_P(DecimalUtilsSplitAndCombineTest, split_Decimal64) +TEST_P(DecimalUtilsSplitAndCombineTest, splitDecimal64) { testSplit(GetParam()); } -TEST_P(DecimalUtilsSplitAndCombineTest, split_Decimal128) +TEST_P(DecimalUtilsSplitAndCombineTest, splitDecimal128) { testSplit(GetParam()); } -TEST_P(DecimalUtilsSplitAndCombineTest, combine_Decimal32) +TEST_P(DecimalUtilsSplitAndCombineTest, combineDecimal32) { testDecimalFromComponents(GetParam()); } -TEST_P(DecimalUtilsSplitAndCombineTest, combine_Decimal64) +TEST_P(DecimalUtilsSplitAndCombineTest, combineDecimal64) { testDecimalFromComponents(GetParam()); } -TEST_P(DecimalUtilsSplitAndCombineTest, combine_Decimal128) +TEST_P(DecimalUtilsSplitAndCombineTest, combineDecimal128) { testDecimalFromComponents(GetParam()); } -TEST_P(DecimalUtilsSplitAndCombineTest, getWholePart_Decimal32) +TEST_P(DecimalUtilsSplitAndCombineTest, getWholePartDecimal32) { testGetWhole(GetParam()); } -TEST_P(DecimalUtilsSplitAndCombineTest, getWholePart_Decimal64) +TEST_P(DecimalUtilsSplitAndCombineTest, getWholePartDecimal64) { testGetWhole(GetParam()); } -TEST_P(DecimalUtilsSplitAndCombineTest, getWholePart_Decimal128) +TEST_P(DecimalUtilsSplitAndCombineTest, getWholePartDecimal128) { testGetWhole(GetParam()); } -TEST_P(DecimalUtilsSplitAndCombineTest, getFractionalPart_Decimal32) +TEST_P(DecimalUtilsSplitAndCombineTest, getFractionalPartDecimal32) { testGetFractional(GetParam()); } -TEST_P(DecimalUtilsSplitAndCombineTest, getFractionalPart_Decimal64) +TEST_P(DecimalUtilsSplitAndCombineTest, getFractionalPartDecimal64) { testGetFractional(GetParam()); } -TEST_P(DecimalUtilsSplitAndCombineTest, getFractionalPart_Decimal128) +TEST_P(DecimalUtilsSplitAndCombineTest, getFractionalPartDecimal128) { testGetFractional(GetParam()); } diff --git a/dbms/src/Disks/tests/gtest_disk.cpp b/dbms/src/Disks/tests/gtest_disk.cpp index ea67545e5fa..6f5ca506bf1 100644 --- a/dbms/src/Disks/tests/gtest_disk.cpp +++ b/dbms/src/Disks/tests/gtest_disk.cpp @@ -61,7 +61,7 @@ private: }; -typedef testing::Types DiskImplementations; +using DiskImplementations = testing::Types; TYPED_TEST_SUITE(DiskTest, DiskImplementations); diff --git a/dbms/src/IO/tests/read_buffer_aio.cpp b/dbms/src/IO/tests/read_buffer_aio.cpp index daa9c10c590..01ac9808cbb 100644 --- a/dbms/src/IO/tests/read_buffer_aio.cpp +++ b/dbms/src/IO/tests/read_buffer_aio.cpp @@ -432,9 +432,7 @@ bool test13(const std::string & filename, const std::string &) DB::ReadBufferAIO in(filename, DEFAULT_AIO_FILE_BLOCK_SIZE); size_t count1 = in.read(newbuf.data(), newbuf.length()); - if (count1 != newbuf.length()) - return false; - return true; + return count1 == newbuf.length(); } bool test14(const std::string & filename, const std::string & buf) From c079a1e15c674a9e3004a12135288e285733500e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 9 Mar 2020 05:15:19 +0300 Subject: [PATCH 151/712] clang-tidy, part 17 --- .../tests/gtest_compressionCodec.cpp | 8 ++++---- dbms/src/IO/tests/gtest_bit_io.cpp | 4 ++-- .../tests/gtest_merge_tree_set_index.cpp | 6 +++--- .../MergeTreeIndexGranuleBloomFilter.cpp | 12 ++++-------- dbms/src/Storages/tests/gtest_storage_log.cpp | 2 +- ...t_transform_query_for_external_database.cpp | 18 +++++++++--------- 6 files changed, 23 insertions(+), 27 deletions(-) diff --git a/dbms/src/Compression/tests/gtest_compressionCodec.cpp b/dbms/src/Compression/tests/gtest_compressionCodec.cpp index 3808e23ab65..e3b226c302d 100644 --- a/dbms/src/Compression/tests/gtest_compressionCodec.cpp +++ b/dbms/src/Compression/tests/gtest_compressionCodec.cpp @@ -584,10 +584,10 @@ TEST_P(CodecTestCompatibility, Decoding) ASSERT_TRUE(EqualByteContainers(expected.data_type->getSizeOfValueInMemory(), expected.serialized_data, decoded)); } -class CodecTest_Performance : public ::testing::TestWithParam> +class CodecTestPerformance : public ::testing::TestWithParam> {}; -TEST_P(CodecTest_Performance, TranscodingWithDataType) +TEST_P(CodecTestPerformance, TranscodingWithDataType) { const auto & [codec_spec, test_seq] = GetParam(); const auto codec = ::makeCodec(codec_spec.codec_statement, test_seq.data_type); @@ -1295,7 +1295,7 @@ INSTANTIATE_TEST_SUITE_P(Gorilla, //}; //INSTANTIATE_TEST_SUITE_P(DoubleDelta, -// CodecTest_Performance, +// CodecTestPerformance, // ::testing::Combine( // ::testing::Values(Codec("DoubleDelta")), // ::testing::Values( @@ -1312,7 +1312,7 @@ INSTANTIATE_TEST_SUITE_P(Gorilla, //); //INSTANTIATE_TEST_SUITE_P(Gorilla, -// CodecTest_Performance, +// CodecTestPerformance, // ::testing::Combine( // ::testing::Values(Codec("Gorilla")), // ::testing::Values( diff --git a/dbms/src/IO/tests/gtest_bit_io.cpp b/dbms/src/IO/tests/gtest_bit_io.cpp index 435e5d10c19..5291dddd25e 100644 --- a/dbms/src/IO/tests/gtest_bit_io.cpp +++ b/dbms/src/IO/tests/gtest_bit_io.cpp @@ -27,7 +27,7 @@ const UInt64 BIT_PATTERN = 0b11101011'11101111'10111010'11101111'10101111'101110 const uint8_t PRIMES[] = {2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61}; template -std::string bin(const T & value, size_t bits = sizeof(T)*8) +std::string bin(const T & value, size_t bits = sizeof(T) * 8) { static const uint8_t MAX_BITS = sizeof(T)*8; assert(bits <= MAX_BITS); @@ -150,7 +150,7 @@ TEST_P(BitIO, WriteAndRead) ReadBufferFromMemory read_buffer(data.data(), data.size()); // auto memory_read_buffer = memory_write_buffer.tryGetReadBuffer(); - if (expected_buffer_binary != std::string{}) + if (!expected_buffer_binary.empty()) { const auto actual_buffer_binary = dumpContents(data, " ", " "); ASSERT_EQ(expected_buffer_binary, actual_buffer_binary); diff --git a/dbms/src/Interpreters/tests/gtest_merge_tree_set_index.cpp b/dbms/src/Interpreters/tests/gtest_merge_tree_set_index.cpp index 6cef78f7c70..d7c0e46d1ca 100644 --- a/dbms/src/Interpreters/tests/gtest_merge_tree_set_index.cpp +++ b/dbms/src/Interpreters/tests/gtest_merge_tree_set_index.cpp @@ -7,7 +7,7 @@ using namespace DB; -TEST(MergeTreeSetIndex, checkInRange_one) +TEST(MergeTreeSetIndex, checkInRangeOne) { DataTypes types = {std::make_shared()}; @@ -34,7 +34,7 @@ TEST(MergeTreeSetIndex, checkInRange_one) ranges = {Range(-1, true, 10, true)}; ASSERT_EQ(set->checkInRange(ranges, types).can_be_true, true) << "(-1, 10)"; - // Left bounded + // Left bounded ranges = {Range::createLeftBounded(1, true)}; ASSERT_EQ(set->checkInRange(ranges, types).can_be_true, true) << "(1, +inf)"; @@ -55,7 +55,7 @@ TEST(MergeTreeSetIndex, checkInRange_one) ASSERT_EQ(set->checkInRange(ranges, types).can_be_true, true) << "(-inf, 10)"; } -TEST(MergeTreeSetIndex, checkInRange_tuple) +TEST(MergeTreeSetIndex, checkInRangeTuple) { DataTypes types = {std::make_shared(), std::make_shared()}; diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexGranuleBloomFilter.cpp b/dbms/src/Storages/MergeTree/MergeTreeIndexGranuleBloomFilter.cpp index c2197126845..e66accb2f92 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexGranuleBloomFilter.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexGranuleBloomFilter.cpp @@ -75,12 +75,12 @@ void MergeTreeIndexGranuleBloomFilter::deserializeBinary(ReadBuffer & istr) throw Exception("Cannot read data to a non-empty bloom filter index.", ErrorCodes::LOGICAL_ERROR); readVarUInt(total_rows, istr); - for (size_t index = 0; index < bloom_filters.size(); ++index) + for (auto & filter : bloom_filters) { static size_t atom_size = 8; size_t bytes_size = (bits_per_row * total_rows + atom_size - 1) / atom_size; - bloom_filters[index] = std::make_shared(bytes_size, hash_functions, 0); - istr.read(reinterpret_cast(bloom_filters[index]->getFilter().data()), bytes_size); + filter = std::make_shared(bytes_size, hash_functions, 0); + istr.read(reinterpret_cast(filter->getFilter().data()), bytes_size); } } @@ -118,13 +118,9 @@ void MergeTreeIndexGranuleBloomFilter::fillingBloomFilter(BloomFilterPtr & bf, c { const auto & hash_column_vec = hash_column->getData(); - for (size_t index = 0, size = hash_column_vec.size(); index < size; ++index) - { - const UInt64 & bf_base_hash = hash_column_vec[index]; - + for (const auto & bf_base_hash : hash_column_vec) for (size_t i = 0; i < hash_functions; ++i) bf->addHashWithSeed(bf_base_hash, BloomFilterHash::bf_hash_seed[i]); - } } } diff --git a/dbms/src/Storages/tests/gtest_storage_log.cpp b/dbms/src/Storages/tests/gtest_storage_log.cpp index 7736343cbdc..d5f78b021b4 100644 --- a/dbms/src/Storages/tests/gtest_storage_log.cpp +++ b/dbms/src/Storages/tests/gtest_storage_log.cpp @@ -64,7 +64,7 @@ private: }; -typedef testing::Types DiskImplementations; +using DiskImplementations = testing::Types; TYPED_TEST_SUITE(StorageLogTest, DiskImplementations); // Returns data written to table in Values format. diff --git a/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp b/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp index 57729218a40..d20ed174b29 100644 --- a/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp +++ b/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp @@ -57,46 +57,46 @@ static void check(const std::string & query, const std::string & expected, const TEST(TransformQueryForExternalDatabase, InWithSingleElement) { check("SELECT column FROM test.table WHERE 1 IN (1)", - "SELECT \"column\" FROM \"test\".\"table\" WHERE 1", + R"(SELECT "column" FROM "test"."table" WHERE 1)", state().context, state().columns); check("SELECT column FROM test.table WHERE column IN (1, 2)", - "SELECT \"column\" FROM \"test\".\"table\" WHERE \"column\" IN (1, 2)", + R"(SELECT "column" FROM "test"."table" WHERE "column" IN (1, 2))", state().context, state().columns); check("SELECT column FROM test.table WHERE column NOT IN ('hello', 'world')", - "SELECT \"column\" FROM \"test\".\"table\" WHERE \"column\" NOT IN ('hello', 'world')", + R"(SELECT "column" FROM "test"."table" WHERE "column" NOT IN ('hello', 'world'))", state().context, state().columns); } TEST(TransformQueryForExternalDatabase, Like) { check("SELECT column FROM test.table WHERE column LIKE '%hello%'", - "SELECT \"column\" FROM \"test\".\"table\" WHERE \"column\" LIKE '%hello%'", + R"(SELECT "column" FROM "test"."table" WHERE "column" LIKE '%hello%')", state().context, state().columns); check("SELECT column FROM test.table WHERE column NOT LIKE 'w%rld'", - "SELECT \"column\" FROM \"test\".\"table\" WHERE \"column\" NOT LIKE 'w%rld'", + R"(SELECT "column" FROM "test"."table" WHERE "column" NOT LIKE 'w%rld')", state().context, state().columns); } TEST(TransformQueryForExternalDatabase, Substring) { check("SELECT column FROM test.table WHERE left(column, 10) = RIGHT(column, 10) AND SUBSTRING(column FROM 1 FOR 2) = 'Hello'", - "SELECT \"column\" FROM \"test\".\"table\"", + R"(SELECT "column" FROM "test"."table")", state().context, state().columns); } TEST(TransformQueryForExternalDatabase, MultipleAndSubqueries) { check("SELECT column FROM test.table WHERE 1 = 1 AND toString(column) = '42' AND column = 42 AND left(column, 10) = RIGHT(column, 10) AND column IN (1, 42) AND SUBSTRING(column FROM 1 FOR 2) = 'Hello' AND column != 4", - "SELECT \"column\" FROM \"test\".\"table\" WHERE 1 AND (\"column\" = 42) AND (\"column\" IN (1, 42)) AND (\"column\" != 4)", + R"(SELECT "column" FROM "test"."table" WHERE 1 AND ("column" = 42) AND ("column" IN (1, 42)) AND ("column" != 4))", state().context, state().columns); check("SELECT column FROM test.table WHERE toString(column) = '42' AND left(column, 10) = RIGHT(column, 10) AND column = 42", - "SELECT \"column\" FROM \"test\".\"table\" WHERE (\"column\" = 42)", + R"(SELECT "column" FROM "test"."table" WHERE ("column" = 42))", state().context, state().columns); } TEST(TransformQueryForExternalDatabase, Issue7245) { check("select apply_id from test.table where apply_type = 2 and create_time > addDays(toDateTime('2019-01-01 01:02:03'),-7) and apply_status in (3,4)", - "SELECT \"apply_id\", \"apply_type\", \"apply_status\", \"create_time\" FROM \"test\".\"table\" WHERE (\"apply_type\" = 2) AND (\"create_time\" > '2018-12-25 01:02:03') AND (\"apply_status\" IN (3, 4))", + R"(SELECT "apply_id", "apply_type", "apply_status", "create_time" FROM "test"."table" WHERE ("apply_type" = 2) AND ("create_time" > '2018-12-25 01:02:03') AND ("apply_status" IN (3, 4)))", state().context, state().columns); } From 71e5c0ee3e6e7b5278972e7ce95b272db7c05bdd Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 9 Mar 2020 05:26:50 +0300 Subject: [PATCH 152/712] clang-tidy, part 18 --- dbms/src/Common/tests/AvalancheTest.cpp | 4 +- dbms/src/Common/tests/CMakeLists.txt | 7 --- dbms/src/Common/tests/allocator.cpp | 61 ------------------- dbms/src/Common/tests/auto_array.cpp | 40 ++++++------ dbms/src/Common/tests/int_hashes_perf.cpp | 4 +- dbms/src/Common/tests/multi_version.cpp | 56 ----------------- dbms/src/Core/tests/string_pool.cpp | 32 +++++----- .../MergeTree/MergeTreeIndexMinMax.cpp | 6 +- .../Storages/MergeTree/MergeTreeIndexSet.cpp | 15 ++--- .../Storages/MergeTree/MergeTreeMarksLoader.h | 2 +- .../TableFunctions/ITableFunctionFileLike.cpp | 4 +- 11 files changed, 51 insertions(+), 180 deletions(-) delete mode 100644 dbms/src/Common/tests/allocator.cpp delete mode 100644 dbms/src/Common/tests/multi_version.cpp diff --git a/dbms/src/Common/tests/AvalancheTest.cpp b/dbms/src/Common/tests/AvalancheTest.cpp index 59be4896af9..b77ee2fa056 100644 --- a/dbms/src/Common/tests/AvalancheTest.cpp +++ b/dbms/src/Common/tests/AvalancheTest.cpp @@ -8,9 +8,9 @@ double maxBias(std::vector & counts, int reps) { double worst = 0; - for (int i = 0; i < static_cast(counts.size()); i++) + for (auto count : counts) { - double c = static_cast(counts[i]) / static_cast(reps); + double c = static_cast(count) / static_cast(reps); double d = fabs(c * 2 - 1); diff --git a/dbms/src/Common/tests/CMakeLists.txt b/dbms/src/Common/tests/CMakeLists.txt index c6cac197ecf..6f1c825227c 100644 --- a/dbms/src/Common/tests/CMakeLists.txt +++ b/dbms/src/Common/tests/CMakeLists.txt @@ -47,10 +47,6 @@ target_link_libraries (pod_array PRIVATE clickhouse_common_io) add_executable (thread_creation_latency thread_creation_latency.cpp) target_link_libraries (thread_creation_latency PRIVATE clickhouse_common_io) -add_executable (multi_version multi_version.cpp) -target_link_libraries (multi_version PRIVATE clickhouse_common_io) -add_check(multi_version) - add_executable (array_cache array_cache.cpp) target_link_libraries (array_cache PRIVATE clickhouse_common_io) @@ -61,9 +57,6 @@ add_executable (integer_hash_tables_and_hashes integer_hash_tables_and_hashes.cp target_include_directories (integer_hash_tables_and_hashes SYSTEM BEFORE PRIVATE ${SPARSEHASH_INCLUDE_DIR}) target_link_libraries (integer_hash_tables_and_hashes PRIVATE dbms) -add_executable (allocator allocator.cpp) -target_link_libraries (allocator PRIVATE clickhouse_common_io) - add_executable (cow_columns cow_columns.cpp) target_link_libraries (cow_columns PRIVATE clickhouse_common_io) diff --git a/dbms/src/Common/tests/allocator.cpp b/dbms/src/Common/tests/allocator.cpp deleted file mode 100644 index 540b29b9786..00000000000 --- a/dbms/src/Common/tests/allocator.cpp +++ /dev/null @@ -1,61 +0,0 @@ -#include -#include - -int main() -{ - Allocator alloc; - - if (1) - { - size_t size = 50000000; - auto p = alloc.alloc(size); - size_t old_size = size; - for (; size < 1000000000; size += 50000000) - { - p = alloc.realloc(p, old_size, size); - old_size = size; - } - alloc.free(p, old_size); - std::cerr << "50mb+50mb+.. ok.\n"; - } - - - { - size_t size = 1000000000; - auto p = alloc.alloc(size); - size_t old_size = size; - //try - //{ - // Now possible jump 65mb->63mb - for (; size > 1000; size /= 2) - { - p = alloc.realloc(p, old_size, size); - old_size = size; - } - /* } - catch (...) - { - size = old_size; - std::cerr << "ok. impossible catch.\n"; - } */ - alloc.free(p, old_size); - std::cerr << "1gb,512mb,128mb,.. ok.\n"; - } - - - if (1) - { - size_t size = 1; - auto p = alloc.alloc(size); - size_t old_size = size; - for (; size < 1000000000; size *= 2) - { - p = alloc.realloc(p, old_size, size); - old_size = size; - } - alloc.free(p, old_size); - std::cerr << "1,2,4,8,..,1G ok.\n"; - } - - std::cerr << "ok.\n"; -} diff --git a/dbms/src/Common/tests/auto_array.cpp b/dbms/src/Common/tests/auto_array.cpp index 8cc332200f7..acb90fd8704 100644 --- a/dbms/src/Common/tests/auto_array.cpp +++ b/dbms/src/Common/tests/auto_array.cpp @@ -17,11 +17,11 @@ int main(int argc, char ** argv) using T = std::string; DB::AutoArray arr(n); - for (size_t i = 0; i < arr.size(); ++i) - arr[i] = "Hello, world! " + DB::toString(i); + for (auto & elem : arr) + elem = "Hello, world! " + DB::toString(i); - for (size_t i = 0; i < arr.size(); ++i) - std::cerr << arr[i] << std::endl; + for (auto & elem : arr) + std::cerr << elem << std::endl; } std::cerr << std::endl; @@ -33,11 +33,11 @@ int main(int argc, char ** argv) Arr arr; arr.resize(n); - for (size_t i = 0; i < arr.size(); ++i) - arr[i] = "Hello, world! " + DB::toString(i); + for (auto & elem : arr) + elem = "Hello, world! " + DB::toString(i); - for (size_t i = 0; i < arr.size(); ++i) - std::cerr << arr[i] << std::endl; + for (auto & elem : arr) + std::cerr << elem << std::endl; std::cerr << std::endl; @@ -45,8 +45,8 @@ int main(int argc, char ** argv) std::cerr << arr.size() << ", " << arr2.size() << std::endl; - for (size_t i = 0; i < arr2.size(); ++i) - std::cerr << arr2[i] << std::endl; + for (auto & elem : arr2) + std::cerr << elem << std::endl; } std::cerr << std::endl; @@ -68,28 +68,28 @@ int main(int argc, char ** argv) map[std::move(key)] = "Hello, world! " + DB::toString(i); } - for (Map::const_iterator it = map.begin(); it != map.end(); ++it) + for (const auto & kv : map) { std::cerr << "["; for (size_t j = 0; j < n; ++j) - std::cerr << (j == 0 ? "" : ", ") << it->first[j]; + std::cerr << (j == 0 ? "" : ", ") << kv.first[j]; std::cerr << "]"; - std::cerr << ":\t" << it->second << std::endl; + std::cerr << ":\t" << kv.second << std::endl; } std::cerr << std::endl; Map map2 = std::move(map); - for (Map::const_iterator it = map2.begin(); it != map2.end(); ++it) + for (const auto & kv : map2) { std::cerr << "["; for (size_t j = 0; j < n; ++j) - std::cerr << (j == 0 ? "" : ", ") << it->first[j]; + std::cerr << (j == 0 ? "" : ", ") << kv.first[j]; std::cerr << "]"; - std::cerr << ":\t" << it->second << std::endl; + std::cerr << ":\t" << kv.second << std::endl; } } @@ -112,11 +112,11 @@ int main(int argc, char ** argv) vec.push_back(std::move(key)); } - for (Vec::const_iterator it = vec.begin(); it != vec.end(); ++it) + for (const auto & elem : vec) { std::cerr << "["; for (size_t j = 0; j < n; ++j) - std::cerr << (j == 0 ? "" : ", ") << (*it)[j]; + std::cerr << (j == 0 ? "" : ", ") << elem[j]; std::cerr << "]" << std::endl; } @@ -124,11 +124,11 @@ int main(int argc, char ** argv) Vec vec2 = std::move(vec); - for (Vec::const_iterator it = vec2.begin(); it != vec2.end(); ++it) + for (const auto & elem : vec2) { std::cerr << "["; for (size_t j = 0; j < n; ++j) - std::cerr << (j == 0 ? "" : ", ") << (*it)[j]; + std::cerr << (j == 0 ? "" : ", ") << elem[j]; std::cerr << "]" << std::endl; } } diff --git a/dbms/src/Common/tests/int_hashes_perf.cpp b/dbms/src/Common/tests/int_hashes_perf.cpp index 69020178bd4..6792b22dfce 100644 --- a/dbms/src/Common/tests/int_hashes_perf.cpp +++ b/dbms/src/Common/tests/int_hashes_perf.cpp @@ -179,9 +179,9 @@ static inline size_t tabulation(UInt64 x) }; size_t res = 0; - for (size_t i = 0; i < 8; ++i) + for (const auto & rand : random) { - res ^= random[i][UInt8(x)]; + res ^= rand[UInt8(x)]; x >>= 8; } diff --git a/dbms/src/Common/tests/multi_version.cpp b/dbms/src/Common/tests/multi_version.cpp deleted file mode 100644 index b33c665d1b7..00000000000 --- a/dbms/src/Common/tests/multi_version.cpp +++ /dev/null @@ -1,56 +0,0 @@ -#include -#include -#include -#include -#include -#include - - -using T = std::string; -using MV = MultiVersion; -using Results = std::vector; - - -static void thread1(MV & x, T & result) -{ - MV::Version v = x.get(); - result = *v; -} - -static void thread2(MV & x, const char * result) -{ - x.set(std::make_unique(result)); -} - - -int main(int, char **) -{ - try - { - const char * s1 = "Hello!"; - const char * s2 = "Goodbye!"; - - size_t n = 1000; - MV x(std::make_unique(s1)); - Results results(n); - - ThreadPool tp(8); - for (size_t i = 0; i < n; ++i) - { - tp.scheduleOrThrowOnError(std::bind(thread1, std::ref(x), std::ref(results[i]))); - tp.scheduleOrThrowOnError(std::bind(thread2, std::ref(x), (rand() % 2) ? s1 : s2)); - } - tp.wait(); - - for (size_t i = 0; i < n; ++i) - std::cerr << results[i] << " "; - std::cerr << std::endl; - } - catch (const Poco::Exception & e) - { - std::cerr << e.message() << std::endl; - throw; - } - - return 0; -} diff --git a/dbms/src/Core/tests/string_pool.cpp b/dbms/src/Core/tests/string_pool.cpp index a389fb01a5e..a2ee0ec3ea5 100644 --- a/dbms/src/Core/tests/string_pool.cpp +++ b/dbms/src/Core/tests/string_pool.cpp @@ -82,8 +82,8 @@ int main(int argc, char ** argv) Set set; Stopwatch watch; - for (Vec::iterator it = vec.begin(); it != vec.end(); ++it) - set[*it] = 0; + for (const auto & elem : vec) + set[elem] = 0; std::cerr << "Inserted into std::unordered_map in " << watch.elapsedSeconds() << " sec, " << vec.size() / watch.elapsedSeconds() << " rows/sec., " @@ -102,8 +102,8 @@ int main(int argc, char ** argv) RefsSet set; Stopwatch watch; - for (Vec::iterator it = vec.begin(); it != vec.end(); ++it) - set[StringRef(*it)] = 0; + for (const auto & elem : vec) + set[StringRef(elem)] = 0; std::cerr << "Inserted refs into std::unordered_map in " << watch.elapsedSeconds() << " sec, " << vec.size() / watch.elapsedSeconds() << " rows/sec., " @@ -123,8 +123,8 @@ int main(int argc, char ** argv) RefsSet set; Stopwatch watch; - for (Vec::iterator it = vec.begin(); it != vec.end(); ++it) - set[StringRef(pool.insert(it->data(), it->size()), it->size())] = 0; + for (const auto & elem : vec) + set[StringRef(pool.insert(elem.data(), elem.size()), elem.size())] = 0; std::cerr << "Inserted into pool and refs into std::unordered_map in " << watch.elapsedSeconds() << " sec, " << vec.size() / watch.elapsedSeconds() << " rows/sec., " @@ -144,8 +144,8 @@ int main(int argc, char ** argv) set.set_empty_key(DenseSet::key_type()); Stopwatch watch; - for (Vec::iterator it = vec.begin(); it != vec.end(); ++it) - set[*it] = 0; + for (const auto & elem : vec) + set[elem] = 0; std::cerr << "Inserted into google::dense_hash_map in " << watch.elapsedSeconds() << " sec, " << vec.size() / watch.elapsedSeconds() << " rows/sec., " @@ -165,8 +165,8 @@ int main(int argc, char ** argv) set.set_empty_key(RefsDenseSet::key_type()); Stopwatch watch; - for (Vec::iterator it = vec.begin(); it != vec.end(); ++it) - set[StringRef(it->data(), it->size())] = 0; + for (const auto & elem : vec) + set[StringRef(elem.data(), elem.size())] = 0; std::cerr << "Inserted refs into google::dense_hash_map in " << watch.elapsedSeconds() << " sec, " << vec.size() / watch.elapsedSeconds() << " rows/sec., " @@ -187,8 +187,8 @@ int main(int argc, char ** argv) set.set_empty_key(RefsDenseSet::key_type()); Stopwatch watch; - for (Vec::iterator it = vec.begin(); it != vec.end(); ++it) - set[StringRef(pool.insert(it->data(), it->size()), it->size())] = 0; + for (const auto & elem : vec) + set[StringRef(pool.insert(elem.data(), elem.size()), elem.size())] = 0; std::cerr << "Inserted into pool and refs into google::dense_hash_map in " << watch.elapsedSeconds() << " sec, " << vec.size() / watch.elapsedSeconds() << " rows/sec., " @@ -207,11 +207,11 @@ int main(int argc, char ** argv) RefsHashMap set; Stopwatch watch; - for (Vec::iterator it = vec.begin(); it != vec.end(); ++it) + for (const auto & elem : vec) { RefsHashMap::LookupResult inserted_it; bool inserted; - set.emplace(StringRef(*it), inserted_it, inserted); + set.emplace(StringRef(elem), inserted_it, inserted); } std::cerr << "Inserted refs into HashMap in " << watch.elapsedSeconds() << " sec, " @@ -234,11 +234,11 @@ int main(int argc, char ** argv) RefsHashMap set; Stopwatch watch; - for (Vec::iterator it = vec.begin(); it != vec.end(); ++it) + for (const auto & elem : vec) { RefsHashMap::LookupResult inserted_it; bool inserted; - set.emplace(StringRef(pool.insert(it->data(), it->size()), it->size()), inserted_it, inserted); + set.emplace(StringRef(pool.insert(elem.data(), elem.size()), elem.size()), inserted_it, inserted); } std::cerr << "Inserted into pool and refs into HashMap in " << watch.elapsedSeconds() << " sec, " diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexMinMax.cpp b/dbms/src/Storages/MergeTree/MergeTreeIndexMinMax.cpp index ae946bf0f4b..07881c87c7d 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexMinMax.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexMinMax.cpp @@ -17,11 +17,11 @@ namespace ErrorCodes MergeTreeIndexGranuleMinMax::MergeTreeIndexGranuleMinMax(const MergeTreeIndexMinMax & index_) - : IMergeTreeIndexGranule(), index(index_), parallelogram() {} + : index(index_) {} MergeTreeIndexGranuleMinMax::MergeTreeIndexGranuleMinMax( const MergeTreeIndexMinMax & index_, std::vector && parallelogram_) - : IMergeTreeIndexGranule(), index(index_), parallelogram(std::move(parallelogram_)) {} + : index(index_), parallelogram(std::move(parallelogram_)) {} void MergeTreeIndexGranuleMinMax::serializeBinary(WriteBuffer & ostr) const { @@ -126,7 +126,7 @@ MergeTreeIndexConditionMinMax::MergeTreeIndexConditionMinMax( const SelectQueryInfo &query, const Context &context, const MergeTreeIndexMinMax &index_) - : IMergeTreeIndexCondition(), index(index_), condition(query, context, index.columns, index.expr) {} + : index(index_), condition(query, context, index.columns, index.expr) {} bool MergeTreeIndexConditionMinMax::alwaysUnknownOrTrue() const { diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexSet.cpp b/dbms/src/Storages/MergeTree/MergeTreeIndexSet.cpp index 946b4d80c99..843b1e6fc98 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexSet.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexSet.cpp @@ -23,14 +23,12 @@ static const Field UNKNOWN_FIELD(3u); MergeTreeIndexGranuleSet::MergeTreeIndexGranuleSet(const MergeTreeIndexSet & index_) - : IMergeTreeIndexGranule() - , index(index_) + : index(index_) , block(index.header.cloneEmpty()) {} MergeTreeIndexGranuleSet::MergeTreeIndexGranuleSet( const MergeTreeIndexSet & index_, MutableColumns && mutable_columns_) - : IMergeTreeIndexGranule() - , index(index_) + : index(index_) , block(index.header.cloneWithColumns(std::move(mutable_columns_))) {} void MergeTreeIndexGranuleSet::serializeBinary(WriteBuffer & ostr) const @@ -217,14 +215,11 @@ MergeTreeIndexConditionSet::MergeTreeIndexConditionSet( const SelectQueryInfo & query, const Context & context, const MergeTreeIndexSet &index_) - : IMergeTreeIndexCondition(), index(index_) + : index(index_) { - for (size_t i = 0, size = index.columns.size(); i < size; ++i) - { - std::string name = index.columns[i]; + for (const auto & name : index.columns) if (!key_columns.count(name)) key_columns.insert(name); - } const auto & select = query.query->as(); @@ -267,7 +262,7 @@ bool MergeTreeIndexConditionSet::mayBeTrueOnGranule(MergeTreeIndexGranulePtr idx throw Exception( "Set index condition got a granule with the wrong type.", ErrorCodes::LOGICAL_ERROR); - if (useless || !granule->size() || (index.max_rows && granule->size() > index.max_rows)) + if (useless || granule->empty() || (index.max_rows && granule->size() > index.max_rows)) return true; Block result = granule->block; diff --git a/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.h b/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.h index 927e78ed8b9..431e677e2eb 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.h +++ b/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.h @@ -16,7 +16,7 @@ public: size_t marks_count_, const MergeTreeIndexGranularityInfo & index_granularity_info_, bool save_marks_in_cache_, - size_t columns_num_in_mark_ = 1); + size_t columns_in_mark_ = 1); const MarkInCompressedFile & getMark(size_t row_index, size_t column_index = 0); diff --git a/dbms/src/TableFunctions/ITableFunctionFileLike.cpp b/dbms/src/TableFunctions/ITableFunctionFileLike.cpp index e11c38fab45..eca507a4003 100644 --- a/dbms/src/TableFunctions/ITableFunctionFileLike.cpp +++ b/dbms/src/TableFunctions/ITableFunctionFileLike.cpp @@ -38,8 +38,8 @@ StoragePtr ITableFunctionFileLike::executeImpl(const ASTPtr & ast_function, cons if (args.size() < 2) throw Exception("Table function '" + getName() + "' requires at least 2 arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - for (size_t i = 0; i < args.size(); ++i) - args[i] = evaluateConstantExpressionOrIdentifierAsLiteral(args[i], context); + for (auto & arg : args) + arg = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context); std::string filename = args[0]->as().value.safeGet(); std::string format = args[1]->as().value.safeGet(); From 2ef5026090bc3d26d488bc72f3c0520d8bacf72a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 9 Mar 2020 05:31:05 +0300 Subject: [PATCH 153/712] clang-tidy, part 19 --- dbms/src/Common/tests/auto_array.cpp | 8 ++++---- dbms/src/IO/tests/valid_utf8.cpp | 4 ++-- .../Storages/MergeTree/MergeTreePartInfo.cpp | 18 +++++++++--------- .../src/Storages/MergeTree/MergeTreePartInfo.h | 2 +- dbms/src/TableFunctions/ITableFunctionXDBC.cpp | 4 ++-- dbms/src/TableFunctions/TableFunctionS3.cpp | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/dbms/src/Common/tests/auto_array.cpp b/dbms/src/Common/tests/auto_array.cpp index acb90fd8704..bbb533b65e8 100644 --- a/dbms/src/Common/tests/auto_array.cpp +++ b/dbms/src/Common/tests/auto_array.cpp @@ -17,8 +17,8 @@ int main(int argc, char ** argv) using T = std::string; DB::AutoArray arr(n); - for (auto & elem : arr) - elem = "Hello, world! " + DB::toString(i); + for (size_t i = 0; i < arr.size(); ++i) + arr[i] = "Hello, world! " + DB::toString(i); for (auto & elem : arr) std::cerr << elem << std::endl; @@ -33,8 +33,8 @@ int main(int argc, char ** argv) Arr arr; arr.resize(n); - for (auto & elem : arr) - elem = "Hello, world! " + DB::toString(i); + for (size_t i = 0; i < arr.size(); ++i) + arr[i] = "Hello, world! " + DB::toString(i); for (auto & elem : arr) std::cerr << elem << std::endl; diff --git a/dbms/src/IO/tests/valid_utf8.cpp b/dbms/src/IO/tests/valid_utf8.cpp index d184f694234..0174f76f7d8 100644 --- a/dbms/src/IO/tests/valid_utf8.cpp +++ b/dbms/src/IO/tests/valid_utf8.cpp @@ -11,8 +11,8 @@ int main(int, char **) { try { - std::string test1 = "kjhsgdfkjhg2378rtzgvxkz877%^&^*%&^*&*"; - std::string test2 = "{\"asd\" = \"qw1\",\"qwe24\" = \"3asd\"}"; + std::string test1 = R"(kjhsgdfkjhg2378rtzgvxkz877%^&^*%&^*&*)"; + std::string test2 = R"({"asd" = "qw1","qwe24" = "3asd"})"; test2[test2.find('1')] = char(127 + 64); test2[test2.find('2')] = char(127 + 64 + 32); test2[test2.find('3')] = char(127 + 64 + 32 + 16); diff --git a/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp b/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp index 3ee330b6d1a..43bd9538e3e 100644 --- a/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreePartInfo.cpp @@ -12,18 +12,18 @@ namespace ErrorCodes } -MergeTreePartInfo MergeTreePartInfo::fromPartName(const String & dir_name, MergeTreeDataFormatVersion format_version) +MergeTreePartInfo MergeTreePartInfo::fromPartName(const String & part_name, MergeTreeDataFormatVersion format_version) { MergeTreePartInfo part_info; - if (!tryParsePartName(dir_name, &part_info, format_version)) - throw Exception("Unexpected part name: " + dir_name, ErrorCodes::BAD_DATA_PART_NAME); + if (!tryParsePartName(part_name, &part_info, format_version)) + throw Exception("Unexpected part name: " + part_name, ErrorCodes::BAD_DATA_PART_NAME); return part_info; } -bool MergeTreePartInfo::tryParsePartName(const String & dir_name, MergeTreePartInfo * part_info, MergeTreeDataFormatVersion format_version) +bool MergeTreePartInfo::tryParsePartName(const String & part_name, MergeTreePartInfo * part_info, MergeTreeDataFormatVersion format_version) { - ReadBufferFromString in(dir_name); + ReadBufferFromString in(part_name); String partition_id; if (format_version < MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) @@ -101,18 +101,18 @@ bool MergeTreePartInfo::tryParsePartName(const String & dir_name, MergeTreePartI } -void MergeTreePartInfo::parseMinMaxDatesFromPartName(const String & dir_name, DayNum & min_date, DayNum & max_date) +void MergeTreePartInfo::parseMinMaxDatesFromPartName(const String & part_name, DayNum & min_date, DayNum & max_date) { UInt32 min_yyyymmdd = 0; UInt32 max_yyyymmdd = 0; - ReadBufferFromString in(dir_name); + ReadBufferFromString in(part_name); if (!tryReadIntText(min_yyyymmdd, in) || !checkChar('_', in) || !tryReadIntText(max_yyyymmdd, in)) { - throw Exception("Unexpected part name: " + dir_name, ErrorCodes::BAD_DATA_PART_NAME); + throw Exception("Unexpected part name: " + part_name, ErrorCodes::BAD_DATA_PART_NAME); } const auto & date_lut = DateLUT::instance(); @@ -124,7 +124,7 @@ void MergeTreePartInfo::parseMinMaxDatesFromPartName(const String & dir_name, Da DayNum max_month = date_lut.toFirstDayNumOfMonth(max_date); if (min_month != max_month) - throw Exception("Part name " + dir_name + " contains different months", ErrorCodes::BAD_DATA_PART_NAME); + throw Exception("Part name " + part_name + " contains different months", ErrorCodes::BAD_DATA_PART_NAME); } diff --git a/dbms/src/Storages/MergeTree/MergeTreePartInfo.h b/dbms/src/Storages/MergeTree/MergeTreePartInfo.h index 2cf423f325a..80b0d3508e0 100644 --- a/dbms/src/Storages/MergeTree/MergeTreePartInfo.h +++ b/dbms/src/Storages/MergeTree/MergeTreePartInfo.h @@ -78,7 +78,7 @@ struct MergeTreePartInfo static MergeTreePartInfo fromPartName(const String & part_name, MergeTreeDataFormatVersion format_version); - static bool tryParsePartName(const String & dir_name, MergeTreePartInfo * part_info, MergeTreeDataFormatVersion format_version); + static bool tryParsePartName(const String & part_name, MergeTreePartInfo * part_info, MergeTreeDataFormatVersion format_version); static void parseMinMaxDatesFromPartName(const String & part_name, DayNum & min_date, DayNum & max_date); diff --git a/dbms/src/TableFunctions/ITableFunctionXDBC.cpp b/dbms/src/TableFunctions/ITableFunctionXDBC.cpp index f6dba915c8e..50236b65445 100644 --- a/dbms/src/TableFunctions/ITableFunctionXDBC.cpp +++ b/dbms/src/TableFunctions/ITableFunctionXDBC.cpp @@ -41,8 +41,8 @@ StoragePtr ITableFunctionXDBC::executeImpl(const ASTPtr & ast_function, const Co + "('DSN', schema, table)", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - for (auto i = 0u; i < args.size(); ++i) - args[i] = evaluateConstantExpressionOrIdentifierAsLiteral(args[i], context); + for (auto & arg : args) + arg = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context); std::string connection_string; std::string schema_name; diff --git a/dbms/src/TableFunctions/TableFunctionS3.cpp b/dbms/src/TableFunctions/TableFunctionS3.cpp index 7788337ed10..04ca48df1df 100644 --- a/dbms/src/TableFunctions/TableFunctionS3.cpp +++ b/dbms/src/TableFunctions/TableFunctionS3.cpp @@ -36,8 +36,8 @@ StoragePtr TableFunctionS3::executeImpl(const ASTPtr & ast_function, const Conte throw Exception("Table function '" + getName() + "' requires 3 to 6 arguments: url, [access_key_id, secret_access_key,] format, structure and [compression_method].", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - for (size_t i = 0; i < args.size(); ++i) - args[i] = evaluateConstantExpressionOrIdentifierAsLiteral(args[i], context); + for (auto & arg : args) + arg = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context); String filename = args[0]->as().value.safeGet(); String format; From c5f2e95e52cd599b395d8fc9d308acbbdbdc0b88 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 9 Mar 2020 05:55:28 +0300 Subject: [PATCH 154/712] clang-tidy, part 20 --- dbms/src/Client/MultiplexedConnections.cpp | 7 +++---- dbms/src/Common/ConcurrentBoundedQueue.h | 6 ++++++ dbms/src/Common/tests/arena_with_free_lists.cpp | 2 +- dbms/src/Functions/FunctionHelpers.cpp | 2 +- dbms/src/Interpreters/tests/string_hash_map.cpp | 4 ++-- .../Formats/Impl/CapnProtoRowInputFormat.h | 2 +- .../Formats/Impl/ConstantExpressionTemplate.cpp | 2 +- .../Impl/JSONCompactEachRowRowInputFormat.cpp | 6 ++---- .../Formats/Impl/ParquetBlockOutputFormat.cpp | 6 +++--- .../Formats/Impl/TabSeparatedRowInputFormat.h | 2 +- .../Processors/Transforms/AggregatingTransform.cpp | 6 +++--- dbms/src/Processors/Transforms/FillingTransform.h | 2 +- .../Processors/Transforms/FinishSortingTransform.cpp | 6 +++--- .../Processors/Transforms/MergingSortedTransform.cpp | 8 +------- .../Processors/Transforms/TotalsHavingTransform.h | 2 +- dbms/src/Storages/MergeTree/MergeTreePartsMover.cpp | 2 +- dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp | 12 ++++++------ .../Storages/MergeTree/MergeTreeReaderCompact.cpp | 2 +- dbms/src/Storages/MergeTree/MergeTreeReaderWide.h | 2 +- .../MergeTree/MergeTreeReverseSelectProcessor.h | 2 +- .../Storages/MergeTree/MergeTreeSelectProcessor.h | 2 +- .../Storages/MergeTree/MergedBlockOutputStream.cpp | 8 ++++---- .../Storages/MergeTree/ReplicatedMergeTreeQueue.cpp | 8 ++++---- .../ReplicatedMergeTreeRestartingThread.cpp | 2 +- dbms/src/Storages/tests/transform_part_zk_nodes.cpp | 8 ++++---- 25 files changed, 54 insertions(+), 57 deletions(-) diff --git a/dbms/src/Client/MultiplexedConnections.cpp b/dbms/src/Client/MultiplexedConnections.cpp index c8d3fa4dcce..965916191d1 100644 --- a/dbms/src/Client/MultiplexedConnections.cpp +++ b/dbms/src/Client/MultiplexedConnections.cpp @@ -36,14 +36,13 @@ MultiplexedConnections::MultiplexedConnections( return; replica_states.reserve(connections.size()); - for (size_t i = 0; i < connections.size(); ++i) + for (auto & connection : connections) { - Connection * connection = &(*connections[i]); connection->setThrottler(throttler); ReplicaState replica_state; - replica_state.pool_entry = std::move(connections[i]); - replica_state.connection = connection; + replica_state.pool_entry = std::move(connection); + replica_state.connection = &*connection; replica_states.push_back(std::move(replica_state)); } diff --git a/dbms/src/Common/ConcurrentBoundedQueue.h b/dbms/src/Common/ConcurrentBoundedQueue.h index 5ac887bf4d4..69034c512a0 100644 --- a/dbms/src/Common/ConcurrentBoundedQueue.h +++ b/dbms/src/Common/ConcurrentBoundedQueue.h @@ -138,6 +138,12 @@ public: return queue.size(); } + size_t empty() + { + Poco::ScopedLock lock(mutex); + return queue.empty(); + } + void clear() { while (fill_count.tryWait(0)) diff --git a/dbms/src/Common/tests/arena_with_free_lists.cpp b/dbms/src/Common/tests/arena_with_free_lists.cpp index 20a3e547da0..e8cc583c0fb 100644 --- a/dbms/src/Common/tests/arena_with_free_lists.cpp +++ b/dbms/src/Common/tests/arena_with_free_lists.cpp @@ -309,7 +309,7 @@ int main(int argc, char ** argv) Dictionary::Attribute attr; attr.type = Dictionary::AttributeUnderlyingType::utString; - std::get>(attr.arrays).reset(new StringRef[cache_size]{}); + std::get>(attr.arrays).reset(new StringRef[cache_size]{}); // NOLINT while (true) { diff --git a/dbms/src/Functions/FunctionHelpers.cpp b/dbms/src/Functions/FunctionHelpers.cpp index 7002879f995..fde1774695c 100644 --- a/dbms/src/Functions/FunctionHelpers.cpp +++ b/dbms/src/Functions/FunctionHelpers.cpp @@ -186,7 +186,7 @@ void validateFunctionArgumentTypes(const IFunction & func, result += sep; } - if (args.size() != 0) + if (!args.empty()) result.erase(result.end() - sep.length(), result.end()); return result; diff --git a/dbms/src/Interpreters/tests/string_hash_map.cpp b/dbms/src/Interpreters/tests/string_hash_map.cpp index 2191ad84705..37fbefbe987 100644 --- a/dbms/src/Interpreters/tests/string_hash_map.cpp +++ b/dbms/src/Interpreters/tests/string_hash_map.cpp @@ -147,9 +147,9 @@ void NO_INLINE bench(const std::vector & data, DB::Arena &, const cha typename Map::LookupResult it; bool inserted; - for (size_t i = 0, size = data.size(); i < size; ++i) + for (const auto & value : data) { - map.emplace(DB::ArenaKeyHolder{data[i], pool}, it, inserted); + map.emplace(DB::ArenaKeyHolder{value, pool}, it, inserted); if (inserted) it->getMapped() = 0; ++it->getMapped(); diff --git a/dbms/src/Processors/Formats/Impl/CapnProtoRowInputFormat.h b/dbms/src/Processors/Formats/Impl/CapnProtoRowInputFormat.h index c39969d21b0..51ed451417c 100644 --- a/dbms/src/Processors/Formats/Impl/CapnProtoRowInputFormat.h +++ b/dbms/src/Processors/Formats/Impl/CapnProtoRowInputFormat.h @@ -43,7 +43,7 @@ private: kj::Array readMessage(); // Build a traversal plan from a sorted list of fields - void createActions(const NestedFieldList & sortedFields, capnp::StructSchema reader); + void createActions(const NestedFieldList & sorted_fields, capnp::StructSchema reader); /* Action for state machine for traversing nested structures. */ using BlockPositionList = std::vector; diff --git a/dbms/src/Processors/Formats/Impl/ConstantExpressionTemplate.cpp b/dbms/src/Processors/Formats/Impl/ConstantExpressionTemplate.cpp index 7644d4cc8f7..83e449ee368 100644 --- a/dbms/src/Processors/Formats/Impl/ConstantExpressionTemplate.cpp +++ b/dbms/src/Processors/Formats/Impl/ConstantExpressionTemplate.cpp @@ -41,7 +41,7 @@ struct SpecialParserType struct LiteralInfo { - typedef std::shared_ptr ASTLiteralPtr; + using ASTLiteralPtr = std::shared_ptr; LiteralInfo(const ASTLiteralPtr & literal_, const String & column_name_, bool force_nullable_) : literal(literal_), dummy_column_name(column_name_), force_nullable(force_nullable_) { } ASTLiteralPtr literal; diff --git a/dbms/src/Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.cpp b/dbms/src/Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.cpp index 26c86812967..ec4d5143ba6 100644 --- a/dbms/src/Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.cpp +++ b/dbms/src/Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.cpp @@ -170,10 +170,8 @@ bool JSONCompactEachRowRowInputFormat::readRow(DB::MutableColumns &columns, DB:: } assertChar(']', in); - for (size_t i = 0; i < not_seen_columns.size(); i++) - { - columns[not_seen_columns[i]]->insertDefault(); - } + for (const auto & name : not_seen_columns) + columns[name]->insertDefault(); ext.read_columns = read_columns; return true; diff --git a/dbms/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp b/dbms/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp index 0822ec3219c..4fab1235a0e 100644 --- a/dbms/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp +++ b/dbms/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp @@ -60,8 +60,8 @@ static void fillArrowArrayWithNumericColumnData( { /// Invert values since Arrow interprets 1 as a non-null value, while CH as a null arrow_null_bytemap.reserve(null_bytemap->size()); - for (size_t i = 0, size = null_bytemap->size(); i < size; ++i) - arrow_null_bytemap.emplace_back(1 ^ (*null_bytemap)[i]); + for (auto is_null : *null_bytemap) + arrow_null_bytemap.emplace_back(!is_null); arrow_null_bytemap_raw_ptr = arrow_null_bytemap.data(); } @@ -255,7 +255,7 @@ class OstreamOutputStream : public arrow::io::OutputStream { public: explicit OstreamOutputStream(WriteBuffer & ostr_) : ostr(ostr_) { is_open = true; } - ~OstreamOutputStream() override {} + ~OstreamOutputStream() override = default; // FileInterface ::arrow::Status Close() override diff --git a/dbms/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.h b/dbms/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.h index 785428bf6f0..5bb0a636e1a 100644 --- a/dbms/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.h +++ b/dbms/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.h @@ -47,7 +47,7 @@ private: void addInputColumn(const String & column_name); void setupAllColumnsByTableSchema(); - void fillUnreadColumnsWithDefaults(MutableColumns & columns, RowReadExtension& ext); + void fillUnreadColumnsWithDefaults(MutableColumns & columns, RowReadExtension & row_read_extension); bool parseRowAndPrintDiagnosticInfo(MutableColumns & columns, WriteBuffer & out) override; void tryDeserializeFiled(const DataTypePtr & type, IColumn & column, size_t file_column, diff --git a/dbms/src/Processors/Transforms/AggregatingTransform.cpp b/dbms/src/Processors/Transforms/AggregatingTransform.cpp index 6f55eaa2089..4a76e0f2273 100644 --- a/dbms/src/Processors/Transforms/AggregatingTransform.cpp +++ b/dbms/src/Processors/Transforms/AggregatingTransform.cpp @@ -362,7 +362,7 @@ private: #define M(NAME) \ else if (first->type == AggregatedDataVariants::Type::NAME) \ params->aggregator.mergeSingleLevelDataImplNAME)::element_type>(*data); - if (false) {} + if (false) {} // NOLINT APPLY_FOR_VARIANTS_SINGLE_LEVEL(M) #undef M else @@ -544,7 +544,7 @@ void AggregatingTransform::initGenerate() variants.convertToTwoLevel(); /// Flush data in the RAM to disk also. It's easier than merging on-disk and RAM data. - if (variants.size()) + if (!variants.empty()) params->aggregator.writeToTemporaryFile(variants); } @@ -573,7 +573,7 @@ void AggregatingTransform::initGenerate() if (cur_variants->isConvertibleToTwoLevel()) cur_variants->convertToTwoLevel(); - if (cur_variants->size()) + if (!cur_variants->empty()) params->aggregator.writeToTemporaryFile(*cur_variants); } } diff --git a/dbms/src/Processors/Transforms/FillingTransform.h b/dbms/src/Processors/Transforms/FillingTransform.h index 5c4c78701f5..570ff1f7691 100644 --- a/dbms/src/Processors/Transforms/FillingTransform.h +++ b/dbms/src/Processors/Transforms/FillingTransform.h @@ -13,7 +13,7 @@ namespace DB class FillingTransform : public ISimpleTransform { public: - FillingTransform(const Block & header_, const SortDescription & fill_description_); + FillingTransform(const Block & header_, const SortDescription & sort_description_); String getName() const override { return "FillingTransform"; } diff --git a/dbms/src/Processors/Transforms/FinishSortingTransform.cpp b/dbms/src/Processors/Transforms/FinishSortingTransform.cpp index 97b47d29fee..4c904eb95a1 100644 --- a/dbms/src/Processors/Transforms/FinishSortingTransform.cpp +++ b/dbms/src/Processors/Transforms/FinishSortingTransform.cpp @@ -98,10 +98,10 @@ void FinishSortingTransform::consume(Chunk chunk) auto source_columns = chunk.detachColumns(); Columns tail_columns; - for (size_t i = 0; i < source_columns.size(); ++i) + for (auto & source_column : source_columns) { - tail_columns.push_back(source_columns[i]->cut(tail_pos, size - tail_pos)); - source_columns[i] = source_columns[i]->cut(0, tail_pos); + tail_columns.emplace_back(source_column->cut(tail_pos, size - tail_pos)); + source_column = source_column->cut(0, tail_pos); } chunks.emplace_back(std::move(source_columns), tail_pos); diff --git a/dbms/src/Processors/Transforms/MergingSortedTransform.cpp b/dbms/src/Processors/Transforms/MergingSortedTransform.cpp index b4ca88d2c03..b9e74277023 100644 --- a/dbms/src/Processors/Transforms/MergingSortedTransform.cpp +++ b/dbms/src/Processors/Transforms/MergingSortedTransform.cpp @@ -226,13 +226,7 @@ void MergingSortedTransform::merge(TSortingHeap & queue) return false; } - if (merged_data.mergedRows() >= max_block_size) - { - //std::cerr << "max_block_size reached\n"; - return false; - } - - return true; + return merged_data.mergedRows() < max_block_size; }; /// Take rows in required order and put them into `merged_data`, while the rows are no more than `max_block_size` diff --git a/dbms/src/Processors/Transforms/TotalsHavingTransform.h b/dbms/src/Processors/Transforms/TotalsHavingTransform.h index c52aa6c3323..b6069da66f3 100644 --- a/dbms/src/Processors/Transforms/TotalsHavingTransform.h +++ b/dbms/src/Processors/Transforms/TotalsHavingTransform.h @@ -43,7 +43,7 @@ protected: Chunk totals; private: - void addToTotals(const Chunk & block, const IColumn::Filter * filter); + void addToTotals(const Chunk & chunk, const IColumn::Filter * filter); void prepareTotals(); /// Params diff --git a/dbms/src/Storages/MergeTree/MergeTreePartsMover.cpp b/dbms/src/Storages/MergeTree/MergeTreePartsMover.cpp index e521c127a5a..5e3999ffbec 100644 --- a/dbms/src/Storages/MergeTree/MergeTreePartsMover.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreePartsMover.cpp @@ -101,7 +101,7 @@ bool MergeTreePartsMover::selectPartsForMove( const auto policy = data->getStoragePolicy(); const auto & volumes = policy->getVolumes(); - if (volumes.size() > 0) + if (!volumes.empty()) { /// Do not check last volume for (size_t i = 0; i != volumes.size() - 1; ++i) diff --git a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp index f5cf7ae622f..6e887a0bf9c 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp @@ -261,18 +261,18 @@ void MergeTreeRangeReader::ReadResult::clear() void MergeTreeRangeReader::ReadResult::shrink(Columns & old_columns) { - for (size_t i = 0; i < old_columns.size(); ++i) + for (auto & column : old_columns) { - if (!old_columns[i]) + if (!column) continue; - auto new_column = old_columns[i]->cloneEmpty(); + auto new_column = column->cloneEmpty(); new_column->reserve(total_rows_per_granule); for (size_t j = 0, pos = 0; j < rows_per_granule_original.size(); pos += rows_per_granule_original[j++]) { if (rows_per_granule[j]) - new_column->insertRangeFrom(*old_columns[i], pos, rows_per_granule[j]); + new_column->insertRangeFrom(*column, pos, rows_per_granule[j]); } - old_columns[i] = std::move(new_column); + column = std::move(new_column); } } @@ -896,7 +896,7 @@ void MergeTreeRangeReader::executePrewhereActionsAndFilterColumns(ReadResult & r } /// Check if the PREWHERE column is needed - if (result.columns.size()) + if (!result.columns.empty()) { if (prewhere->remove_prewhere_column) result.columns.erase(result.columns.begin() + prewhere_column_pos); diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index 4086c346a6a..89e13aa5c15 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -142,7 +142,7 @@ size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, for (size_t i = 0; i < num_columns; ++i) { auto & column = mutable_columns[i]; - if (column && column->size()) + if (column && !column->empty()) res_columns[i] = std::move(column); else res_columns[i] = nullptr; diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.h b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.h index ca4822f932d..78d1a59fd89 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.h @@ -42,7 +42,7 @@ private: void readData( const String & name, const IDataType & type, IColumn & column, size_t from_mark, bool continue_reading, size_t max_rows_to_read, - bool read_offsets = true); + bool with_offsets = true); }; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.h b/dbms/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.h index a7837d1121d..a2ea4bb8960 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.h @@ -22,7 +22,7 @@ public: UInt64 max_block_size_rows, size_t preferred_block_size_bytes, size_t preferred_max_column_in_block_size_bytes, - Names column_names, + Names required_columns_, MarkRanges mark_ranges, bool use_uncompressed_cache, const PrewhereInfoPtr & prewhere_info, diff --git a/dbms/src/Storages/MergeTree/MergeTreeSelectProcessor.h b/dbms/src/Storages/MergeTree/MergeTreeSelectProcessor.h index 6c105a842e1..a4e9dfcab1c 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSelectProcessor.h +++ b/dbms/src/Storages/MergeTree/MergeTreeSelectProcessor.h @@ -22,7 +22,7 @@ public: UInt64 max_block_size_rows, size_t preferred_block_size_bytes, size_t preferred_max_column_in_block_size_bytes, - Names column_names_, + Names required_columns_, MarkRanges mark_ranges, bool use_uncompressed_cache, const PrewhereInfoPtr & prewhere_info, diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp index c46618a55ae..d4a1f785885 100644 --- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -80,7 +80,7 @@ void MergedBlockOutputStream::writeSuffix() void MergedBlockOutputStream::writeSuffixAndFinalizePart( MergeTreeData::MutableDataPartPtr & new_part, - const NamesAndTypesList * total_column_list, + const NamesAndTypesList * total_columns_list, MergeTreeData::DataPart::Checksums * additional_column_checksums) { /// Finish write and get checksums. @@ -94,8 +94,8 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( writer->finishPrimaryIndexSerialization(checksums); writer->finishSkipIndicesSerialization(checksums); - if (!total_column_list) - total_column_list = &columns_list; + if (!total_columns_list) + total_columns_list = &columns_list; if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING || isCompactPart(new_part)) { @@ -127,7 +127,7 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( { /// Write a file with a description of columns. WriteBufferFromFile out(part_path + "columns.txt", 4096); - total_column_list->writeText(out); + total_columns_list->writeText(out); } { diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 12b7f41206d..420b04df8bd 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -31,7 +31,7 @@ void ReplicatedMergeTreeQueue::addVirtualParts(const MergeTreeData::DataParts & { std::lock_guard lock(state_mutex); - for (auto part : parts) + for (const auto & part : parts) { current_parts.add(part->name); virtual_parts.add(part->name); @@ -265,7 +265,7 @@ void ReplicatedMergeTreeQueue::removePartFromMutations(const String & part_name) MutationStatus & status = *it->second; status.parts_to_do.removePartAndCoveredParts(part_name); - if (status.parts_to_do.empty()) + if (status.parts_to_do.size() == 0) some_mutations_are_probably_done = true; if (!status.latest_failed_part.empty() && part_info.contains(status.latest_failed_part_info)) @@ -695,7 +695,7 @@ void ReplicatedMergeTreeQueue::updateMutations(zkutil::ZooKeeperPtr zookeeper, C } } - if (mutation.parts_to_do.empty()) + if (mutation.parts_to_do.size() == 0) { some_mutations_are_probably_done = true; } @@ -1377,7 +1377,7 @@ bool ReplicatedMergeTreeQueue::tryFinalizeMutations(zkutil::ZooKeeperPtr zookeep mutation.parts_to_do.clear(); } } - else if (mutation.parts_to_do.empty()) + else if (mutation.parts_to_do.size() == 0) { LOG_TRACE(log, "Will check if mutation " << mutation.entry->znode_name << " is done"); candidates.push_back(mutation.entry); diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index 6b066646a7e..37b428e1e47 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -240,7 +240,7 @@ void ReplicatedMergeTreeRestartingThread::removeFailedQuorumParts() /// Firstly, remove parts from ZooKeeper storage.tryRemovePartsFromZooKeeperWithRetries(failed_parts); - for (auto part_name : failed_parts) + for (const auto & part_name : failed_parts) { auto part = storage.getPartIfExists( part_name, {MergeTreeDataPartState::PreCommitted, MergeTreeDataPartState::Committed, MergeTreeDataPartState::Outdated}); diff --git a/dbms/src/Storages/tests/transform_part_zk_nodes.cpp b/dbms/src/Storages/tests/transform_part_zk_nodes.cpp index b496158ea99..2b4f6b7ff5c 100644 --- a/dbms/src/Storages/tests/transform_part_zk_nodes.cpp +++ b/dbms/src/Storages/tests/transform_part_zk_nodes.cpp @@ -117,12 +117,12 @@ try } } - for (auto it = nodes_queue.begin(); it != nodes_queue.end(); ++it) + for (auto & node : nodes_queue) { - if (it->set_future.valid()) + if (node.set_future.valid()) { - it->set_future.get(); - std::cerr << it->path << " changed!" << std::endl; + node.set_future.get(); + std::cerr << node.path << " changed!" << std::endl; } } } From 3f455a024eb6bbf3b4431b6425ece997fad1b0d4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 9 Mar 2020 06:14:24 +0300 Subject: [PATCH 155/712] clang-tidy, part 21 --- .../src/Common/tests/parallel_aggregation.cpp | 45 +++++++++---------- .../ExecutableDictionarySource.cpp | 5 +-- dbms/src/Dictionaries/FlatDictionary.cpp | 6 +-- dbms/src/Dictionaries/HashedDictionary.cpp | 8 ++-- .../Dictionaries/RangeHashedDictionary.cpp | 4 +- .../Dictionaries/RedisBlockInputStream.cpp | 4 +- dbms/src/Dictionaries/TrieDictionary.cpp | 4 +- dbms/src/Dictionaries/TrieDictionary.h | 3 -- .../System/StorageSystemStackTrace.cpp | 2 +- .../Storages/System/StorageSystemTables.cpp | 2 +- .../System/StorageSystemZooKeeper.cpp | 4 +- 11 files changed, 39 insertions(+), 48 deletions(-) diff --git a/dbms/src/Common/tests/parallel_aggregation.cpp b/dbms/src/Common/tests/parallel_aggregation.cpp index 72010247226..4533ff3a88e 100644 --- a/dbms/src/Common/tests/parallel_aggregation.cpp +++ b/dbms/src/Common/tests/parallel_aggregation.cpp @@ -283,10 +283,10 @@ int main(int argc, char ** argv) Stopwatch watch; for (size_t i = 0; i < num_threads; ++i) - pool.scheduleOrThrowOnError(std::bind(aggregate1, + pool.scheduleOrThrowOnError([&] { aggregate1( std::ref(maps[i]), data.begin() + (data.size() * i) / num_threads, - data.begin() + (data.size() * (i + 1)) / num_threads)); + data.begin() + (data.size() * (i + 1)) / num_threads); }); pool.wait(); @@ -337,10 +337,10 @@ int main(int argc, char ** argv) Stopwatch watch; for (size_t i = 0; i < num_threads; ++i) - pool.scheduleOrThrowOnError(std::bind(aggregate12, + pool.scheduleOrThrowOnError([&] { aggregate12( std::ref(maps[i]), data.begin() + (data.size() * i) / num_threads, - data.begin() + (data.size() * (i + 1)) / num_threads)); + data.begin() + (data.size() * (i + 1)) / num_threads); }); pool.wait(); @@ -396,10 +396,10 @@ int main(int argc, char ** argv) Stopwatch watch; for (size_t i = 0; i < num_threads; ++i) - pool.scheduleOrThrowOnError(std::bind(aggregate1, + pool.scheduleOrThrowOnError([&] { aggregate1( std::ref(maps[i]), data.begin() + (data.size() * i) / num_threads, - data.begin() + (data.size() * (i + 1)) / num_threads)); + data.begin() + (data.size() * (i + 1)) / num_threads); }); pool.wait(); @@ -472,10 +472,10 @@ int main(int argc, char ** argv) Stopwatch watch; for (size_t i = 0; i < num_threads; ++i) - pool.scheduleOrThrowOnError(std::bind(aggregate2, + pool.scheduleOrThrowOnError([&] { aggregate2( std::ref(maps[i]), data.begin() + (data.size() * i) / num_threads, - data.begin() + (data.size() * (i + 1)) / num_threads)); + data.begin() + (data.size() * (i + 1)) / num_threads); }); pool.wait(); @@ -498,8 +498,7 @@ int main(int argc, char ** argv) watch.restart(); for (size_t i = 0; i < MapTwoLevel::NUM_BUCKETS; ++i) - pool.scheduleOrThrowOnError(std::bind(merge2, - maps.data(), num_threads, i)); + pool.scheduleOrThrowOnError([&] { merge2(maps.data(), num_threads, i); }); pool.wait(); @@ -526,10 +525,10 @@ int main(int argc, char ** argv) Stopwatch watch; for (size_t i = 0; i < num_threads; ++i) - pool.scheduleOrThrowOnError(std::bind(aggregate22, + pool.scheduleOrThrowOnError([&] { aggregate22( std::ref(maps[i]), data.begin() + (data.size() * i) / num_threads, - data.begin() + (data.size() * (i + 1)) / num_threads)); + data.begin() + (data.size() * (i + 1)) / num_threads); }); pool.wait(); @@ -552,7 +551,7 @@ int main(int argc, char ** argv) watch.restart(); for (size_t i = 0; i < MapTwoLevel::NUM_BUCKETS; ++i) - pool.scheduleOrThrowOnError(std::bind(merge2, maps.data(), num_threads, i)); + pool.scheduleOrThrowOnError([&] { merge2(maps.data(), num_threads, i); }); pool.wait(); @@ -591,12 +590,12 @@ int main(int argc, char ** argv) Stopwatch watch; for (size_t i = 0; i < num_threads; ++i) - pool.scheduleOrThrowOnError(std::bind(aggregate3, + pool.scheduleOrThrowOnError([&] { aggregate3( std::ref(local_maps[i]), std::ref(global_map), std::ref(mutex), data.begin() + (data.size() * i) / num_threads, - data.begin() + (data.size() * (i + 1)) / num_threads)); + data.begin() + (data.size() * (i + 1)) / num_threads); }); pool.wait(); @@ -657,12 +656,12 @@ int main(int argc, char ** argv) Stopwatch watch; for (size_t i = 0; i < num_threads; ++i) - pool.scheduleOrThrowOnError(std::bind(aggregate33, + pool.scheduleOrThrowOnError([&] { aggregate33( std::ref(local_maps[i]), std::ref(global_map), std::ref(mutex), data.begin() + (data.size() * i) / num_threads, - data.begin() + (data.size() * (i + 1)) / num_threads)); + data.begin() + (data.size() * (i + 1)) / num_threads); }); pool.wait(); @@ -726,12 +725,12 @@ int main(int argc, char ** argv) Stopwatch watch; for (size_t i = 0; i < num_threads; ++i) - pool.scheduleOrThrowOnError(std::bind(aggregate4, + pool.scheduleOrThrowOnError([&] { aggregate4( std::ref(local_maps[i]), std::ref(global_map), mutexes.data(), data.begin() + (data.size() * i) / num_threads, - data.begin() + (data.size() * (i + 1)) / num_threads)); + data.begin() + (data.size() * (i + 1)) / num_threads); }); pool.wait(); @@ -796,11 +795,11 @@ int main(int argc, char ** argv) Stopwatch watch; for (size_t i = 0; i < num_threads; ++i) - pool.scheduleOrThrowOnError(std::bind(aggregate5, + pool.scheduleOrThrowOnError([&] { aggregate5( std::ref(local_maps[i]), std::ref(global_map), data.begin() + (data.size() * i) / num_threads, - data.begin() + (data.size() * (i + 1)) / num_threads)); + data.begin() + (data.size() * (i + 1)) / num_threads); }); pool.wait(); @@ -859,10 +858,10 @@ int main(int argc, char ** argv) Stopwatch watch; for (size_t i = 0; i < num_threads; ++i) - pool.scheduleOrThrowOnError(std::bind(aggregate1, + pool.scheduleOrThrowOnError([&] { aggregate1( std::ref(maps[i]), data.begin() + (data.size() * i) / num_threads, - data.begin() + (data.size() * (i + 1)) / num_threads)); + data.begin() + (data.size() * (i + 1)) / num_threads); }); pool.wait(); diff --git a/dbms/src/Dictionaries/ExecutableDictionarySource.cpp b/dbms/src/Dictionaries/ExecutableDictionarySource.cpp index a4d291765be..b978d00dc2d 100644 --- a/dbms/src/Dictionaries/ExecutableDictionarySource.cpp +++ b/dbms/src/Dictionaries/ExecutableDictionarySource.cpp @@ -201,10 +201,7 @@ bool ExecutableDictionarySource::supportsSelectiveLoad() const bool ExecutableDictionarySource::hasUpdateField() const { - if (update_field.empty()) - return false; - else - return true; + return !update_field.empty(); } DictionarySourcePtr ExecutableDictionarySource::clone() const diff --git a/dbms/src/Dictionaries/FlatDictionary.cpp b/dbms/src/Dictionaries/FlatDictionary.cpp index 8a0e4175d01..b70fe95346d 100644 --- a/dbms/src/Dictionaries/FlatDictionary.cpp +++ b/dbms/src/Dictionaries/FlatDictionary.cpp @@ -600,12 +600,12 @@ void FlatDictionary::setAttributeValueImpl(Attribute & attribute, const Key id, } template <> -void FlatDictionary::setAttributeValueImpl(Attribute & attribute, const Key id, const String & string) +void FlatDictionary::setAttributeValueImpl(Attribute & attribute, const Key id, const String & value) { resize(attribute, id); - const auto string_in_arena = attribute.string_arena->insert(string.data(), string.size()); + const auto string_in_arena = attribute.string_arena->insert(value.data(), value.size()); auto & array = std::get>(attribute.arrays); - array[id] = StringRef{string_in_arena, string.size()}; + array[id] = StringRef{string_in_arena, value.size()}; loaded_ids[id] = true; } diff --git a/dbms/src/Dictionaries/HashedDictionary.cpp b/dbms/src/Dictionaries/HashedDictionary.cpp index a7244e05fae..722a6e3584c 100644 --- a/dbms/src/Dictionaries/HashedDictionary.cpp +++ b/dbms/src/Dictionaries/HashedDictionary.cpp @@ -9,12 +9,12 @@ namespace /// NOTE: Trailing return type is explicitly specified for SFINAE. /// google::sparse_hash_map -template auto first(const T & value) -> decltype(value.first) { return value.first; } -template auto second(const T & value) -> decltype(value.second) { return value.second; } +template auto first(const T & value) -> decltype(value.first) { return value.first; } // NOLINT +template auto second(const T & value) -> decltype(value.second) { return value.second; } // NOLINT /// HashMap -template auto first(const T & value) -> decltype(value.getKey()) { return value.getKey(); } -template auto second(const T & value) -> decltype(value.getMapped()) { return value.getMapped(); } +template auto first(const T & value) -> decltype(value.getKey()) { return value.getKey(); } // NOLINT +template auto second(const T & value) -> decltype(value.getMapped()) { return value.getMapped(); } // NOLINT } diff --git a/dbms/src/Dictionaries/RangeHashedDictionary.cpp b/dbms/src/Dictionaries/RangeHashedDictionary.cpp index 16d04cf6d39..7fab92fcde0 100644 --- a/dbms/src/Dictionaries/RangeHashedDictionary.cpp +++ b/dbms/src/Dictionaries/RangeHashedDictionary.cpp @@ -364,9 +364,7 @@ void RangeHashedDictionary::getItems( const PaddedPODArray & dates, PaddedPODArray & out) const { - if (false) - { - } + if (false) {} // NOLINT #define DISPATCH(TYPE) else if (attribute.type == AttributeUnderlyingType::ut##TYPE) getItemsImpl(attribute, ids, dates, out); DISPATCH(UInt8) DISPATCH(UInt16) diff --git a/dbms/src/Dictionaries/RedisBlockInputStream.cpp b/dbms/src/Dictionaries/RedisBlockInputStream.cpp index ad3d9002b36..8bd6ff06054 100644 --- a/dbms/src/Dictionaries/RedisBlockInputStream.cpp +++ b/dbms/src/Dictionaries/RedisBlockInputStream.cpp @@ -152,8 +152,8 @@ namespace DB break; Poco::Redis::Command command_for_values("HMGET"); - for (auto it = keys_array.begin(); it != keys_array.end(); ++it) - command_for_values.addRedisType(*it); + for (const auto & elem : keys_array) + command_for_values.addRedisType(elem); auto values = client->execute(command_for_values); diff --git a/dbms/src/Dictionaries/TrieDictionary.cpp b/dbms/src/Dictionaries/TrieDictionary.cpp index 9698e5a00c9..dcefc873b4f 100644 --- a/dbms/src/Dictionaries/TrieDictionary.cpp +++ b/dbms/src/Dictionaries/TrieDictionary.cpp @@ -672,13 +672,13 @@ void TrieDictionary::has(const Attribute &, const Columns & key_columns, PaddedP } template -void TrieDictionary::trieTraverse(const btrie_t * tree, Getter && getter) const +static void trieTraverse(const btrie_t * trie, Getter && getter) { KeyType key = 0; const KeyType high_bit = ~((~key) >> 1); btrie_node_t * node; - node = tree->root; + node = trie->root; std::stack stack; while (node) diff --git a/dbms/src/Dictionaries/TrieDictionary.h b/dbms/src/Dictionaries/TrieDictionary.h index 5168eec5a74..81f5a02a00b 100644 --- a/dbms/src/Dictionaries/TrieDictionary.h +++ b/dbms/src/Dictionaries/TrieDictionary.h @@ -230,9 +230,6 @@ private: template void has(const Attribute & attribute, const Columns & key_columns, PaddedPODArray & out) const; - template - void trieTraverse(const btrie_t * trie, Getter && getter) const; - Columns getKeyColumns() const; const std::string database; diff --git a/dbms/src/Storages/System/StorageSystemStackTrace.cpp b/dbms/src/Storages/System/StorageSystemStackTrace.cpp index 07e9ebfb8c5..b5dfe6560ff 100644 --- a/dbms/src/Storages/System/StorageSystemStackTrace.cpp +++ b/dbms/src/Storages/System/StorageSystemStackTrace.cpp @@ -39,7 +39,7 @@ namespace std::optional stack_trace; - static constexpr size_t max_query_id_size = 128; + constexpr size_t max_query_id_size = 128; char query_id_data[max_query_id_size]; size_t query_id_size = 0; diff --git a/dbms/src/Storages/System/StorageSystemTables.cpp b/dbms/src/Storages/System/StorageSystemTables.cpp index ef4cbbed325..973719e1ecf 100644 --- a/dbms/src/Storages/System/StorageSystemTables.cpp +++ b/dbms/src/Storages/System/StorageSystemTables.cpp @@ -136,7 +136,7 @@ protected: { Tables external_tables = context.getSessionContext().getExternalTables(); - for (auto table : external_tables) + for (auto & table : external_tables) { size_t src_index = 0; size_t res_index = 0; diff --git a/dbms/src/Storages/System/StorageSystemZooKeeper.cpp b/dbms/src/Storages/System/StorageSystemZooKeeper.cpp index 727b8dd037b..c3f1d8a8505 100644 --- a/dbms/src/Storages/System/StorageSystemZooKeeper.cpp +++ b/dbms/src/Storages/System/StorageSystemZooKeeper.cpp @@ -50,8 +50,8 @@ static bool extractPathImpl(const IAST & elem, String & res) if (function->name == "and") { - for (size_t i = 0; i < function->arguments->children.size(); ++i) - if (extractPathImpl(*function->arguments->children[i], res)) + for (const auto & child : function->arguments->children) + if (extractPathImpl(*child, res)) return true; return false; From 66582d69a5c6e7fea543dd981eee58dfecd0ef90 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 9 Mar 2020 06:38:43 +0300 Subject: [PATCH 156/712] clang-tidy, part 22 --- dbms/programs/obfuscator/Obfuscator.cpp | 4 ++-- .../odbc-bridge/ColumnInfoHandler.cpp | 2 +- .../odbc-bridge/ODBCBlockInputStream.cpp | 2 +- .../performance-test/PerformanceTest.cpp | 4 ++-- .../performance-test/PerformanceTestSuite.cpp | 6 +++--- dbms/programs/server/HTTPHandler.cpp | 2 +- .../server/InterserverIOHTTPHandler.cpp | 4 ++-- dbms/programs/server/MetricsTransmitter.cpp | 2 +- .../AggregateFunctionArray.cpp | 2 +- .../AggregateFunctionGroupArray.cpp | 5 ++--- .../AggregateFunctionUniqCombined.cpp | 4 ++-- .../AggregateFunctionUniqUpTo.cpp | 2 +- .../AggregateFunctionWindowFunnel.h | 2 +- dbms/src/Functions/CRC.cpp | 4 ++-- dbms/src/Functions/FunctionsLogical.cpp | 2 +- dbms/src/Functions/FunctionsStringRegex.cpp | 2 +- dbms/src/Functions/GatherUtils/GatherUtils.h | 2 +- dbms/src/Functions/GeoUtils.h | 2 +- .../src/Functions/URL/tldLookup.generated.cpp | 2 +- dbms/src/Functions/array/arrayCumSum.cpp | 16 ++++++---------- .../array/arrayCumSumNonNegative.cpp | 8 ++++---- dbms/src/Functions/array/arrayDifference.cpp | 8 +++----- dbms/src/Functions/array/arrayDistinct.cpp | 9 +++------ dbms/src/Functions/array/arrayEnumerate.cpp | 4 +--- dbms/src/Functions/array/arrayFill.cpp | 8 ++++---- dbms/src/Functions/array/arrayFirst.cpp | 12 ++++++------ dbms/src/Functions/array/arrayReverse.cpp | 8 ++++---- dbms/src/Functions/array/arraySplit.cpp | 19 ++++++++----------- dbms/src/Functions/array/arrayZip.cpp | 2 +- dbms/src/Functions/blockSerializedSize.cpp | 4 ++-- dbms/src/Functions/caseWithExpression.cpp | 4 ++-- dbms/src/Functions/now64.cpp | 2 +- dbms/src/Functions/randomPrintableASCII.cpp | 2 +- dbms/src/Functions/runningAccumulate.cpp | 2 +- dbms/src/Functions/toStartOfInterval.cpp | 2 +- dbms/src/Functions/tuple.cpp | 2 +- .../Interpreters/tests/hash_map_lookup.cpp | 8 ++++---- .../Interpreters/tests/hash_map_string.cpp | 2 +- .../Interpreters/tests/hash_map_string_2.cpp | 9 +++------ .../Interpreters/tests/hash_map_string_3.cpp | 4 ++-- .../tests/hash_map_string_small.cpp | 2 +- .../TableFunctions/TableFunctionRemote.cpp | 8 ++++---- 42 files changed, 91 insertions(+), 109 deletions(-) diff --git a/dbms/programs/obfuscator/Obfuscator.cpp b/dbms/programs/obfuscator/Obfuscator.cpp index d86ba1ee7a3..7251c0a4473 100644 --- a/dbms/programs/obfuscator/Obfuscator.cpp +++ b/dbms/programs/obfuscator/Obfuscator.cpp @@ -111,7 +111,7 @@ public: /// Deterministically change seed to some other value. This can be used to generate more values than were in source. virtual void updateSeed(); - virtual ~IModel() {} + virtual ~IModel() = default; }; using ModelPtr = std::unique_ptr; @@ -550,7 +550,7 @@ private: return res; } - bool writeCodePoint(CodePoint code, char *& pos, char * end) + bool writeCodePoint(CodePoint code, char *& pos, const char * end) { size_t length = (code & 0xFF000000) ? 4 diff --git a/dbms/programs/odbc-bridge/ColumnInfoHandler.cpp b/dbms/programs/odbc-bridge/ColumnInfoHandler.cpp index 3dadc7632de..b89d50569f6 100644 --- a/dbms/programs/odbc-bridge/ColumnInfoHandler.cpp +++ b/dbms/programs/odbc-bridge/ColumnInfoHandler.cpp @@ -93,7 +93,7 @@ void ODBCColumnsInfoHandler::handleRequest(Poco::Net::HTTPServerRequest & reques process_error("No 'connection_string' in request URL"); return; } - std::string schema_name = ""; + std::string schema_name; std::string table_name = params.get("table"); std::string connection_string = params.get("connection_string"); diff --git a/dbms/programs/odbc-bridge/ODBCBlockInputStream.cpp b/dbms/programs/odbc-bridge/ODBCBlockInputStream.cpp index 8aa93c43c2b..6ada6dd3a2d 100644 --- a/dbms/programs/odbc-bridge/ODBCBlockInputStream.cpp +++ b/dbms/programs/odbc-bridge/ODBCBlockInputStream.cpp @@ -40,7 +40,7 @@ namespace { using ValueType = ExternalResultDescription::ValueType; - static void insertValue(IColumn & column, const ValueType type, const Poco::Dynamic::Var & value) + void insertValue(IColumn & column, const ValueType type, const Poco::Dynamic::Var & value) { switch (type) { diff --git a/dbms/programs/performance-test/PerformanceTest.cpp b/dbms/programs/performance-test/PerformanceTest.cpp index a2e0aa933b8..c71760a1e58 100644 --- a/dbms/programs/performance-test/PerformanceTest.cpp +++ b/dbms/programs/performance-test/PerformanceTest.cpp @@ -43,7 +43,7 @@ void waitQuery(Connection & connection) finished = true; break; case Protocol::Server::Exception: - throw *packet.exception; + throw Exception(*packet.exception); } if (finished) @@ -117,7 +117,7 @@ bool PerformanceTest::checkPreconditions() const { for (const ColumnWithTypeAndName & column : packet.block) { - if (column.name == "result" && column.column->size() > 0) + if (column.name == "result" && !column.column->empty()) { exist = column.column->get64(0); if (exist) diff --git a/dbms/programs/performance-test/PerformanceTestSuite.cpp b/dbms/programs/performance-test/PerformanceTestSuite.cpp index 73ae2ae90ed..a1eb70a5954 100644 --- a/dbms/programs/performance-test/PerformanceTestSuite.cpp +++ b/dbms/programs/performance-test/PerformanceTestSuite.cpp @@ -90,7 +90,7 @@ public: { global_context.makeGlobalContext(); global_context.getSettingsRef().copyChangesFrom(cmd_settings); - if (input_files.size() < 1) + if (input_files.empty()) throw Exception("No tests were specified", ErrorCodes::BAD_ARGUMENTS); } @@ -156,7 +156,7 @@ private: LOG_INFO(log, "Test configurations prepared"); - if (tests_configurations.size()) + if (!tests_configurations.empty()) { Strings outputs; @@ -174,7 +174,7 @@ private: break; } - if (!lite_output && outputs.size()) + if (!lite_output && !outputs.empty()) { std::cout << "[" << std::endl; diff --git a/dbms/programs/server/HTTPHandler.cpp b/dbms/programs/server/HTTPHandler.cpp index a4c59ff9e25..0b0f73fcdc5 100644 --- a/dbms/programs/server/HTTPHandler.cpp +++ b/dbms/programs/server/HTTPHandler.cpp @@ -448,7 +448,7 @@ void HTTPHandler::processQuery( settings.readonly = 2; } - bool has_external_data = startsWith(request.getContentType().data(), "multipart/form-data"); + bool has_external_data = startsWith(request.getContentType(), "multipart/form-data"); if (has_external_data) { diff --git a/dbms/programs/server/InterserverIOHTTPHandler.cpp b/dbms/programs/server/InterserverIOHTTPHandler.cpp index 407d3c41a9b..481ba8b00d0 100644 --- a/dbms/programs/server/InterserverIOHTTPHandler.cpp +++ b/dbms/programs/server/InterserverIOHTTPHandler.cpp @@ -68,12 +68,12 @@ void InterserverIOHTTPHandler::processQuery(Poco::Net::HTTPServerRequest & reque if (compress) { - CompressedWriteBuffer compressed_out(*used_output.out.get()); + CompressedWriteBuffer compressed_out(*used_output.out); endpoint->processQuery(params, body, compressed_out, response); } else { - endpoint->processQuery(params, body, *used_output.out.get(), response); + endpoint->processQuery(params, body, *used_output.out, response); } } diff --git a/dbms/programs/server/MetricsTransmitter.cpp b/dbms/programs/server/MetricsTransmitter.cpp index 051bed8a0a0..3c19bd9b01c 100644 --- a/dbms/programs/server/MetricsTransmitter.cpp +++ b/dbms/programs/server/MetricsTransmitter.cpp @@ -127,7 +127,7 @@ void MetricsTransmitter::transmit(std::vector & prev_count } } - if (key_vals.size()) + if (!key_vals.empty()) BaseDaemon::instance().writeToGraphite(key_vals, config_name); } diff --git a/dbms/src/AggregateFunctions/AggregateFunctionArray.cpp b/dbms/src/AggregateFunctions/AggregateFunctionArray.cpp index 5d8c164c6e6..ced95185263 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionArray.cpp +++ b/dbms/src/AggregateFunctions/AggregateFunctionArray.cpp @@ -19,7 +19,7 @@ public: DataTypes transformArguments(const DataTypes & arguments) const override { - if (0 == arguments.size()) + if (arguments.empty()) throw Exception("-Array aggregate functions require at least one argument", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); DataTypes nested_arguments; diff --git a/dbms/src/AggregateFunctions/AggregateFunctionGroupArray.cpp b/dbms/src/AggregateFunctions/AggregateFunctionGroupArray.cpp index d4801b723ab..e188116cc2c 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionGroupArray.cpp +++ b/dbms/src/AggregateFunctions/AggregateFunctionGroupArray.cpp @@ -49,7 +49,7 @@ inline AggregateFunctionPtr createAggregateFunctionGroupArrayImpl(const DataType } -static AggregateFunctionPtr createAggregateFunctionGroupArray(const std::string & name, const DataTypes & argument_types, const Array & parameters) +AggregateFunctionPtr createAggregateFunctionGroupArray(const std::string & name, const DataTypes & argument_types, const Array & parameters) { assertUnary(name, argument_types); @@ -83,8 +83,7 @@ static AggregateFunctionPtr createAggregateFunctionGroupArray(const std::string return createAggregateFunctionGroupArrayImpl>(argument_types[0], max_elems); } -static AggregateFunctionPtr -createAggregateFunctionGroupArraySample(const std::string & name, const DataTypes & argument_types, const Array & parameters) +AggregateFunctionPtr createAggregateFunctionGroupArraySample(const std::string & name, const DataTypes & argument_types, const Array & parameters) { assertUnary(name, argument_types); diff --git a/dbms/src/AggregateFunctions/AggregateFunctionUniqCombined.cpp b/dbms/src/AggregateFunctions/AggregateFunctionUniqCombined.cpp index bab63677eaa..c0064044f95 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionUniqCombined.cpp +++ b/dbms/src/AggregateFunctions/AggregateFunctionUniqCombined.cpp @@ -131,8 +131,8 @@ namespace void registerAggregateFunctionUniqCombined(AggregateFunctionFactory & factory) { using namespace std::placeholders; - factory.registerFunction("uniqCombined", std::bind(createAggregateFunctionUniqCombined, false, _1, _2, _3)); - factory.registerFunction("uniqCombined64", std::bind(createAggregateFunctionUniqCombined, true, _1, _2, _3)); + factory.registerFunction("uniqCombined", std::bind(createAggregateFunctionUniqCombined, false, _1, _2, _3)); // NOLINT + factory.registerFunction("uniqCombined64", std::bind(createAggregateFunctionUniqCombined, true, _1, _2, _3)); // NOLINT } } diff --git a/dbms/src/AggregateFunctions/AggregateFunctionUniqUpTo.cpp b/dbms/src/AggregateFunctions/AggregateFunctionUniqUpTo.cpp index f787bb7f7bf..a9a8ae0eaf3 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionUniqUpTo.cpp +++ b/dbms/src/AggregateFunctions/AggregateFunctionUniqUpTo.cpp @@ -21,7 +21,7 @@ namespace ErrorCodes namespace { -static constexpr UInt8 uniq_upto_max_threshold = 100; +constexpr UInt8 uniq_upto_max_threshold = 100; AggregateFunctionPtr createAggregateFunctionUniqUpTo(const std::string & name, const DataTypes & argument_types, const Array & params) diff --git a/dbms/src/AggregateFunctions/AggregateFunctionWindowFunnel.h b/dbms/src/AggregateFunctions/AggregateFunctionWindowFunnel.h index fe89a633dec..6316f69cbcb 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionWindowFunnel.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionWindowFunnel.h @@ -148,7 +148,7 @@ private: // The Algorithm complexity is O(n). UInt8 getEventLevel(const Data & data) const { - if (data.empty()) + if (data.size() == 0) return 0; if (events_size == 1) return 1; diff --git a/dbms/src/Functions/CRC.cpp b/dbms/src/Functions/CRC.cpp index a16888ffe46..22814b2c26c 100644 --- a/dbms/src/Functions/CRC.cpp +++ b/dbms/src/Functions/CRC.cpp @@ -45,13 +45,13 @@ struct CRCImpl } }; -static constexpr UInt64 CRC64_ECMA = 0xc96c5795d7870f42ULL; +constexpr UInt64 CRC64_ECMA = 0xc96c5795d7870f42ULL; struct CRC64ECMAImpl : public CRCImpl { static constexpr auto name = "CRC64"; }; -static constexpr UInt32 CRC32_IEEE = 0xedb88320; +constexpr UInt32 CRC32_IEEE = 0xedb88320; struct CRC32IEEEImpl : public CRCImpl { static constexpr auto name = "CRC32IEEE"; diff --git a/dbms/src/Functions/FunctionsLogical.cpp b/dbms/src/Functions/FunctionsLogical.cpp index b5c64ce9602..8b8f03a0c89 100644 --- a/dbms/src/Functions/FunctionsLogical.cpp +++ b/dbms/src/Functions/FunctionsLogical.cpp @@ -287,7 +287,7 @@ struct OperationApplier { if (!use_result_data_as_input) doBatchedApply(in, result_data); - while (in.size() > 0) + while (!in.empty()) doBatchedApply(in, result_data); } diff --git a/dbms/src/Functions/FunctionsStringRegex.cpp b/dbms/src/Functions/FunctionsStringRegex.cpp index 3a4d46431f6..186e58f83fa 100644 --- a/dbms/src/Functions/FunctionsStringRegex.cpp +++ b/dbms/src/Functions/FunctionsStringRegex.cpp @@ -518,7 +518,7 @@ struct ReplaceRegexpImpl { Instructions instructions; - String now = ""; + String now; for (size_t i = 0; i < s.size(); ++i) { if (s[i] == '\\' && i + 1 < s.size()) diff --git a/dbms/src/Functions/GatherUtils/GatherUtils.h b/dbms/src/Functions/GatherUtils/GatherUtils.h index 29d4d2c101b..4732c326d5f 100644 --- a/dbms/src/Functions/GatherUtils/GatherUtils.h +++ b/dbms/src/Functions/GatherUtils/GatherUtils.h @@ -47,7 +47,7 @@ void sliceDynamicOffsetBounded(IArraySource & src, IArraySink & sink, const ICol void sliceHas(IArraySource & first, IArraySource & second, bool all, ColumnUInt8 & result); -void push(IArraySource & array_source, IValueSource & value_source, IArraySink & sink, bool push_back); +void push(IArraySource & array_source, IValueSource & value_source, IArraySink & sink, bool push_front); void resizeDynamicSize(IArraySource & array_source, IValueSource & value_source, IArraySink & sink, const IColumn & size_column); diff --git a/dbms/src/Functions/GeoUtils.h b/dbms/src/Functions/GeoUtils.h index 023e5621861..f5e15ae013d 100644 --- a/dbms/src/Functions/GeoUtils.h +++ b/dbms/src/Functions/GeoUtils.h @@ -737,7 +737,7 @@ GeohashesInBoxPreparedArgs geohashesInBoxPrepare(const Float64 longitude_min, Float64 latitude_max, uint8_t precision); -UInt64 geohashesInBox(const GeohashesInBoxPreparedArgs & estimation, char * out); +UInt64 geohashesInBox(const GeohashesInBoxPreparedArgs & args, char * out); } /// GeoUtils diff --git a/dbms/src/Functions/URL/tldLookup.generated.cpp b/dbms/src/Functions/URL/tldLookup.generated.cpp index 1f6cd9d938f..14eabbc935f 100644 --- a/dbms/src/Functions/URL/tldLookup.generated.cpp +++ b/dbms/src/Functions/URL/tldLookup.generated.cpp @@ -95812,6 +95812,6 @@ const char * tldLookupHash::is_valid(const char * str, size_t len) return s; } } - return 0; + return nullptr; } #line 4957 "tldLookup.gperf" diff --git a/dbms/src/Functions/array/arrayCumSum.cpp b/dbms/src/Functions/array/arrayCumSum.cpp index 2e00f42bd96..a15ca998366 100644 --- a/dbms/src/Functions/array/arrayCumSum.cpp +++ b/dbms/src/Functions/array/arrayCumSum.cpp @@ -77,16 +77,14 @@ struct ArrayCumSumImpl res_values.resize(column_const->size()); size_t pos = 0; - for (size_t i = 0; i < offsets.size(); ++i) + for (auto offset : offsets) { // skip empty arrays - if (pos < offsets[i]) + if (pos < offset) { res_values[pos++] = x; - for (; pos < offsets[i]; ++pos) - { + for (; pos < offset; ++pos) res_values[pos] = res_values[pos - 1] + x; - } } } @@ -107,16 +105,14 @@ struct ArrayCumSumImpl res_values.resize(data.size()); size_t pos = 0; - for (size_t i = 0; i < offsets.size(); ++i) + for (auto offset : offsets) { // skip empty arrays - if (pos < offsets[i]) + if (pos < offset) { res_values[pos] = data[pos]; - for (++pos; pos < offsets[i]; ++pos) - { + for (++pos; pos < offset; ++pos) res_values[pos] = res_values[pos - 1] + data[pos]; - } } } res_ptr = ColumnArray::create(std::move(res_nested), array.getOffsetsPtr()); diff --git a/dbms/src/Functions/array/arrayCumSumNonNegative.cpp b/dbms/src/Functions/array/arrayCumSumNonNegative.cpp index 016fd845d19..061d2c3e202 100644 --- a/dbms/src/Functions/array/arrayCumSumNonNegative.cpp +++ b/dbms/src/Functions/array/arrayCumSumNonNegative.cpp @@ -73,14 +73,14 @@ struct ArrayCumSumNonNegativeImpl size_t pos = 0; Result accum_sum = 0; - for (size_t i = 0; i < offsets.size(); ++i) + for (auto offset : offsets) { // skip empty arrays - if (pos < offsets[i]) + if (pos < offset) { accum_sum = data[pos] > 0 ? data[pos] : Element(0); res_values[pos] = accum_sum; - for (++pos; pos < offsets[i]; ++pos) + for (++pos; pos < offset; ++pos) { accum_sum = accum_sum + data[pos]; if (accum_sum < 0) @@ -95,7 +95,7 @@ struct ArrayCumSumNonNegativeImpl } - static ColumnPtr execute(const ColumnArray & array, ColumnPtr mapped) + static ColumnPtr execute(const ColumnArray & array, ColumnPtr & mapped) { ColumnPtr res; diff --git a/dbms/src/Functions/array/arrayDifference.cpp b/dbms/src/Functions/array/arrayDifference.cpp index 6080e38cbe6..c02533c2564 100644 --- a/dbms/src/Functions/array/arrayDifference.cpp +++ b/dbms/src/Functions/array/arrayDifference.cpp @@ -71,16 +71,14 @@ struct ArrayDifferenceImpl res_values.resize(data.size()); size_t pos = 0; - for (size_t i = 0; i < offsets.size(); ++i) + for (auto offset : offsets) { // skip empty arrays - if (pos < offsets[i]) + if (pos < offset) { res_values[pos] = 0; - for (++pos; pos < offsets[i]; ++pos) - { + for (++pos; pos < offset; ++pos) res_values[pos] = static_cast(data[pos]) - static_cast(data[pos - 1]); - } } } res_ptr = ColumnArray::create(std::move(res_nested), array.getOffsetsPtr()); diff --git a/dbms/src/Functions/array/arrayDistinct.cpp b/dbms/src/Functions/array/arrayDistinct.cpp index 3246539d497..920ee6d39fd 100644 --- a/dbms/src/Functions/array/arrayDistinct.cpp +++ b/dbms/src/Functions/array/arrayDistinct.cpp @@ -163,11 +163,10 @@ bool FunctionArrayDistinct::executeNumber( ColumnArray::Offset prev_src_offset = 0; ColumnArray::Offset res_offset = 0; - for (ColumnArray::Offset i = 0; i < src_offsets.size(); ++i) + for (auto curr_src_offset : src_offsets) { set.clear(); - ColumnArray::Offset curr_src_offset = src_offsets[i]; for (ColumnArray::Offset j = prev_src_offset; j < curr_src_offset; ++j) { if (nullable_col && (*src_null_map)[j]) @@ -217,11 +216,10 @@ bool FunctionArrayDistinct::executeString( ColumnArray::Offset prev_src_offset = 0; ColumnArray::Offset res_offset = 0; - for (ColumnArray::Offset i = 0; i < src_offsets.size(); ++i) + for (auto curr_src_offset : src_offsets) { set.clear(); - ColumnArray::Offset curr_src_offset = src_offsets[i]; for (ColumnArray::Offset j = prev_src_offset; j < curr_src_offset; ++j) { if (nullable_col && (*src_null_map)[j]) @@ -264,11 +262,10 @@ void FunctionArrayDistinct::executeHashed( ColumnArray::Offset prev_src_offset = 0; ColumnArray::Offset res_offset = 0; - for (ColumnArray::Offset i = 0; i < src_offsets.size(); ++i) + for (auto curr_src_offset : src_offsets) { set.clear(); - ColumnArray::Offset curr_src_offset = src_offsets[i]; for (ColumnArray::Offset j = prev_src_offset; j < curr_src_offset; ++j) { if (nullable_col && (*src_null_map)[j]) diff --git a/dbms/src/Functions/array/arrayEnumerate.cpp b/dbms/src/Functions/array/arrayEnumerate.cpp index a228c310fbc..11f2e41e3e6 100644 --- a/dbms/src/Functions/array/arrayEnumerate.cpp +++ b/dbms/src/Functions/array/arrayEnumerate.cpp @@ -56,12 +56,10 @@ public: ColumnUInt32::Container & res_values = res_nested->getData(); res_values.resize(array->getData().size()); ColumnArray::Offset prev_off = 0; - for (ColumnArray::Offset i = 0; i < offsets.size(); ++i) + for (auto off : offsets) { - ColumnArray::Offset off = offsets[i]; for (ColumnArray::Offset j = prev_off; j < off; ++j) res_values[j] = j - prev_off + 1; - prev_off = off; } diff --git a/dbms/src/Functions/array/arrayFill.cpp b/dbms/src/Functions/array/arrayFill.cpp index 539abdf25df..0a463f908a3 100644 --- a/dbms/src/Functions/array/arrayFill.cpp +++ b/dbms/src/Functions/array/arrayFill.cpp @@ -43,9 +43,9 @@ struct ArrayFillImpl out_data.reserve(in_data.size()); - for (size_t i = 0; i < in_offsets.size(); ++i) + for (auto in_offset : in_offsets) { - array_end = in_offsets[i] - 1; + array_end = in_offset - 1; for (; end <= array_end; ++end) { @@ -96,9 +96,9 @@ struct ArrayFillImpl out_data.reserve(in_data.size()); - for (size_t i = 0; i < in_offsets.size(); ++i) + for (auto in_offset : in_offsets) { - array_end = in_offsets[i] - 1; + array_end = in_offset - 1; if constexpr (reverse) out_data.insertManyFrom(in_data, array_end, array_end + 1 - array_begin); diff --git a/dbms/src/Functions/array/arrayFirst.cpp b/dbms/src/Functions/array/arrayFirst.cpp index 4b70615802c..a635e0d9230 100644 --- a/dbms/src/Functions/array/arrayFirst.cpp +++ b/dbms/src/Functions/array/arrayFirst.cpp @@ -41,14 +41,14 @@ struct ArrayFirstImpl out->reserve(data.size()); size_t pos{}; - for (size_t i = 0; i < offsets.size(); ++i) + for (auto offset : offsets) { - if (offsets[i] - pos > 0) + if (offset - pos > 0) out->insert(data[pos]); else out->insertDefault(); - pos = offsets[i]; + pos = offset; } return out; @@ -68,16 +68,16 @@ struct ArrayFirstImpl out->reserve(data.size()); size_t pos{}; - for (size_t i = 0; i < offsets.size(); ++i) + for (auto offset : offsets) { auto exists = false; - for (; pos < offsets[i]; ++pos) + for (; pos < offset; ++pos) { if (filter[pos]) { out->insert(data[pos]); exists = true; - pos = offsets[i]; + pos = offset; break; } } diff --git a/dbms/src/Functions/array/arrayReverse.cpp b/dbms/src/Functions/array/arrayReverse.cpp index 9ae165abb42..b53fc01569b 100644 --- a/dbms/src/Functions/array/arrayReverse.cpp +++ b/dbms/src/Functions/array/arrayReverse.cpp @@ -100,15 +100,15 @@ void FunctionArrayReverse::executeImpl(Block & block, const ColumnNumbers & argu } -bool FunctionArrayReverse::executeGeneric(const IColumn & src_data, const ColumnArray::Offsets & src_offsets, IColumn & res_data) +bool FunctionArrayReverse::executeGeneric(const IColumn & src_data, const ColumnArray::Offsets & src_array_offsets, IColumn & res_data) { - size_t size = src_offsets.size(); + size_t size = src_array_offsets.size(); res_data.reserve(size); ColumnArray::Offset src_prev_offset = 0; for (size_t i = 0; i < size; ++i) { - ssize_t src_index = src_offsets[i] - 1; + ssize_t src_index = src_array_offsets[i] - 1; while (src_index >= ssize_t(src_prev_offset)) { @@ -116,7 +116,7 @@ bool FunctionArrayReverse::executeGeneric(const IColumn & src_data, const Column --src_index; } - src_prev_offset = src_offsets[i]; + src_prev_offset = src_array_offsets[i]; } return true; diff --git a/dbms/src/Functions/array/arraySplit.cpp b/dbms/src/Functions/array/arraySplit.cpp index edae37c61f7..5190bd1c724 100644 --- a/dbms/src/Functions/array/arraySplit.cpp +++ b/dbms/src/Functions/array/arraySplit.cpp @@ -44,16 +44,14 @@ struct ArraySplitImpl out_offsets_2.reserve(in_offsets.size()); // assume the actual size to be equal or larger out_offsets_1.reserve(in_offsets.size()); - for (size_t i = 0; i < in_offsets.size(); ++i) + for (auto in_offset : in_offsets) { - if (pos < in_offsets[i]) + if (pos < in_offset) { pos += !reverse; - for (; pos < in_offsets[i] - reverse; ++pos) - { + for (; pos < in_offset - reverse; ++pos) if (cut[pos]) out_offsets_2.push_back(pos + reverse); - } pos += reverse; out_offsets_2.push_back(pos); @@ -76,8 +74,8 @@ struct ArraySplitImpl for (size_t i = 0; i < in_offsets.back(); ++i) out_offsets_2.push_back(i + 1); - for (size_t i = 0; i < in_offsets.size(); ++i) - out_offsets_1.push_back(in_offsets[i]); + for (auto in_offset : in_offsets) + out_offsets_1.push_back(in_offset); } else { @@ -86,12 +84,11 @@ struct ArraySplitImpl out_offsets_2.reserve(in_offsets.size()); out_offsets_1.reserve(in_offsets.size()); - for (size_t i = 0; i < in_offsets.size(); ++i) + for (auto in_offset : in_offsets) { - if (pos < in_offsets[i]) + if (pos < in_offset) { - pos = in_offsets[i]; - + pos = in_offset; out_offsets_2.push_back(pos); } diff --git a/dbms/src/Functions/array/arrayZip.cpp b/dbms/src/Functions/array/arrayZip.cpp index 81eb0905033..60fd626f746 100644 --- a/dbms/src/Functions/array/arrayZip.cpp +++ b/dbms/src/Functions/array/arrayZip.cpp @@ -36,7 +36,7 @@ public: DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { - if (arguments.size() < 1) + if (arguments.empty()) throw Exception("Function " + getName() + " needs at least one argument; passed " + toString(arguments.size()) + "." , ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); diff --git a/dbms/src/Functions/blockSerializedSize.cpp b/dbms/src/Functions/blockSerializedSize.cpp index 3d1758991cf..cca4c9ef0c6 100644 --- a/dbms/src/Functions/blockSerializedSize.cpp +++ b/dbms/src/Functions/blockSerializedSize.cpp @@ -32,8 +32,8 @@ public: { UInt64 size = 0; - for (size_t i = 0; i < arguments.size(); ++i) - size += blockSerializedSizeOne(block.getByPosition(arguments[i])); + for (auto arg_pos : arguments) + size += blockSerializedSizeOne(block.getByPosition(arg_pos)); block.getByPosition(result).column = DataTypeUInt64().createColumnConst( input_rows_count, size)->convertToFullColumnIfConst(); diff --git a/dbms/src/Functions/caseWithExpression.cpp b/dbms/src/Functions/caseWithExpression.cpp index 5f8292acafa..e1e124fd1ef 100644 --- a/dbms/src/Functions/caseWithExpression.cpp +++ b/dbms/src/Functions/caseWithExpression.cpp @@ -28,7 +28,7 @@ public: DataTypePtr getReturnTypeImpl(const DataTypes & args) const override { - if (!args.size()) + if (args.empty()) throw Exception{"Function " + getName() + " expects at least 1 arguments", ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION}; @@ -46,7 +46,7 @@ public: void executeImpl(Block & block, const ColumnNumbers & args, size_t result, size_t input_rows_count) override { - if (!args.size()) + if (args.empty()) throw Exception{"Function " + getName() + " expects at least 1 argument", ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION}; diff --git a/dbms/src/Functions/now64.cpp b/dbms/src/Functions/now64.cpp index 1595ee2de3d..483bff37cbf 100644 --- a/dbms/src/Functions/now64.cpp +++ b/dbms/src/Functions/now64.cpp @@ -64,7 +64,7 @@ public: UInt32 scale = DataTypeDateTime64::default_scale; // Type check is similar to the validateArgumentType, trying to keep error codes and messages as close to the said function as possible. - if (arguments.size() >= 1) + if (!arguments.empty()) { const auto & argument = arguments[0]; if (!isInteger(argument.type) || !argument.column || !isColumnConst(*argument.column)) diff --git a/dbms/src/Functions/randomPrintableASCII.cpp b/dbms/src/Functions/randomPrintableASCII.cpp index a6f0fd65e42..4009163d834 100644 --- a/dbms/src/Functions/randomPrintableASCII.cpp +++ b/dbms/src/Functions/randomPrintableASCII.cpp @@ -36,7 +36,7 @@ public: DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { - if (arguments.size() < 1) + if (arguments.empty()) throw Exception("Function " + getName() + " requires at least one argument: the size of resulting string", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); diff --git a/dbms/src/Functions/runningAccumulate.cpp b/dbms/src/Functions/runningAccumulate.cpp index 53dc5e19777..275259e1209 100644 --- a/dbms/src/Functions/runningAccumulate.cpp +++ b/dbms/src/Functions/runningAccumulate.cpp @@ -60,7 +60,7 @@ public: DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { - if (arguments.size() < 1 || arguments.size() > 2) + if (arguments.empty() || arguments.size() > 2) throw Exception("Incorrect number of arguments of function " + getName() + ". Must be 1 or 2.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); diff --git a/dbms/src/Functions/toStartOfInterval.cpp b/dbms/src/Functions/toStartOfInterval.cpp index 1da74c67072..f470504a3a9 100644 --- a/dbms/src/Functions/toStartOfInterval.cpp +++ b/dbms/src/Functions/toStartOfInterval.cpp @@ -23,7 +23,7 @@ namespace ErrorCodes namespace { - static constexpr auto function_name = "toStartOfInterval"; + constexpr auto function_name = "toStartOfInterval"; template struct Transform; diff --git a/dbms/src/Functions/tuple.cpp b/dbms/src/Functions/tuple.cpp index 07de10d3e70..451f732c869 100644 --- a/dbms/src/Functions/tuple.cpp +++ b/dbms/src/Functions/tuple.cpp @@ -53,7 +53,7 @@ public: DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { - if (arguments.size() < 1) + if (arguments.empty()) throw Exception("Function " + getName() + " requires at least one argument.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); return std::make_shared(arguments); diff --git a/dbms/src/Interpreters/tests/hash_map_lookup.cpp b/dbms/src/Interpreters/tests/hash_map_lookup.cpp index 387cc26edd5..13ff3234e8b 100644 --- a/dbms/src/Interpreters/tests/hash_map_lookup.cpp +++ b/dbms/src/Interpreters/tests/hash_map_lookup.cpp @@ -48,21 +48,21 @@ void NO_INLINE bench(const std::vector & data, const char * name) Map map; Stopwatch watch; - for (size_t i = 0, size = data.size(); i < size; ++i) + for (auto value : data) { typename Map::LookupResult it; bool inserted; - map.emplace(data[i], it, inserted); + map.emplace(value, it, inserted); if (inserted) it->getMapped() = 1; else ++it->getMapped(); } - for (size_t i = 0, size = data.size(); i < size; ++i) + for (auto value : data) { - auto it = map.find(data[i]); + auto it = map.find(value); auto curr = ++it; if (curr) curr->getMapped(); diff --git a/dbms/src/Interpreters/tests/hash_map_string.cpp b/dbms/src/Interpreters/tests/hash_map_string.cpp index 8f8931ddd3b..6c1f6b813ff 100644 --- a/dbms/src/Interpreters/tests/hash_map_string.cpp +++ b/dbms/src/Interpreters/tests/hash_map_string.cpp @@ -41,7 +41,7 @@ struct CompactStringRef CompactStringRef(const unsigned char * data_, size_t size_) : CompactStringRef(reinterpret_cast(data_), size_) {} explicit CompactStringRef(const std::string & s) : CompactStringRef(s.data(), s.size()) {} - CompactStringRef() {} + CompactStringRef() = default; const char * data() const { return reinterpret_cast(reinterpret_cast(data_mixed) & 0x0000FFFFFFFFFFFFULL); } diff --git a/dbms/src/Interpreters/tests/hash_map_string_2.cpp b/dbms/src/Interpreters/tests/hash_map_string_2.cpp index 97dbc674d2a..32b723c1187 100644 --- a/dbms/src/Interpreters/tests/hash_map_string_2.cpp +++ b/dbms/src/Interpreters/tests/hash_map_string_2.cpp @@ -568,10 +568,7 @@ inline bool operator==(StringRef_CompareAlwaysTrue, StringRef_CompareAlwaysTrue) inline bool operator==(StringRef_CompareAlmostAlwaysTrue lhs, StringRef_CompareAlmostAlwaysTrue rhs) { - if (lhs.size != rhs.size) - return false; - - return true; + return lhs.size == rhs.size; } @@ -589,9 +586,9 @@ void NO_INLINE bench(const std::vector & data, const char * name) typename Map::LookupResult it; bool inserted; - for (size_t i = 0, size = data.size(); i < size; ++i) + for (const auto & value : data) { - map.emplace(static_cast(data[i]), it, inserted); + map.emplace(static_cast(value), it, inserted); if (inserted) it->getMapped() = 0; ++it->getMapped(); diff --git a/dbms/src/Interpreters/tests/hash_map_string_3.cpp b/dbms/src/Interpreters/tests/hash_map_string_3.cpp index 3c58771d87b..62ed0584d3f 100644 --- a/dbms/src/Interpreters/tests/hash_map_string_3.cpp +++ b/dbms/src/Interpreters/tests/hash_map_string_3.cpp @@ -438,9 +438,9 @@ void NO_INLINE bench(const std::vector & data, const char * name) typename Map::LookupResult it; bool inserted; - for (size_t i = 0, size = data.size(); i < size; ++i) + for (const auto & value : data) { - map.emplace(static_cast(data[i]), it, inserted); + map.emplace(static_cast(value), it, inserted); if (inserted) it->getMapped() = 0; ++it->getMapped(); diff --git a/dbms/src/Interpreters/tests/hash_map_string_small.cpp b/dbms/src/Interpreters/tests/hash_map_string_small.cpp index cbfc6bcdc67..f5cfeef13e2 100644 --- a/dbms/src/Interpreters/tests/hash_map_string_small.cpp +++ b/dbms/src/Interpreters/tests/hash_map_string_small.cpp @@ -50,7 +50,7 @@ struct SmallStringRef SmallStringRef(const unsigned char * data_, size_t size_) : SmallStringRef(reinterpret_cast(data_), size_) {} explicit SmallStringRef(const std::string & s) : SmallStringRef(s.data(), s.size()) {} - SmallStringRef() {} + SmallStringRef() = default; std::string toString() const { return std::string(data(), size); } }; diff --git a/dbms/src/TableFunctions/TableFunctionRemote.cpp b/dbms/src/TableFunctions/TableFunctionRemote.cpp index ffcec737fb7..65e94722848 100644 --- a/dbms/src/TableFunctions/TableFunctionRemote.cpp +++ b/dbms/src/TableFunctions/TableFunctionRemote.cpp @@ -155,8 +155,8 @@ StoragePtr TableFunctionRemote::executeImpl(const ASTPtr & ast_function, const C std::vector shards = parseRemoteDescription(cluster_description, 0, cluster_description.size(), ',', max_addresses); std::vector> names; - for (size_t i = 0; i < shards.size(); ++i) - names.push_back(parseRemoteDescription(shards[i], 0, shards[i].size(), '|', max_addresses)); + for (const auto & shard : shards) + names.push_back(parseRemoteDescription(shard, 0, shard.size(), '|', max_addresses)); if (names.empty()) throw Exception("Shard list is empty after parsing first argument", ErrorCodes::BAD_ARGUMENTS); @@ -164,9 +164,9 @@ StoragePtr TableFunctionRemote::executeImpl(const ASTPtr & ast_function, const C auto maybe_secure_port = context.getTCPPortSecure(); /// Check host and port on affiliation allowed hosts. - for (auto hosts : names) + for (const auto & hosts : names) { - for (auto host : hosts) + for (const auto & host : hosts) { size_t colon = host.find(':'); if (colon == String::npos) From 7c188b84798b7eee898d5738b66b5fd85aa737d4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 9 Mar 2020 06:41:03 +0300 Subject: [PATCH 157/712] clang-tidy, part 23 --- dbms/programs/client/Client.cpp | 8 +++----- dbms/programs/server/Server.cpp | 2 +- dbms/programs/server/TCPHandler.h | 6 +++--- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp index 963b3251084..aa80abdc778 100644 --- a/dbms/programs/client/Client.cpp +++ b/dbms/programs/client/Client.cpp @@ -112,7 +112,7 @@ namespace ErrorCodes class Client : public Poco::Util::Application { public: - Client() {} + Client() = default; private: using StringSet = std::unordered_set; @@ -353,10 +353,8 @@ private: return false; auto days = DateLUT::instance().toDayNum(current_time).toUnderType(); - for (auto i = 0ul; i < N; ++i) + for (auto d : chineseNewYearIndicators) { - auto d = chineseNewYearIndicators[i]; - /// Let's celebrate until Lantern Festival if (d <= days && d + 25u >= days) return true; @@ -645,7 +643,7 @@ private: } - inline const String prompt() const + inline String prompt() const { return boost::replace_all_copy(prompt_by_server_display_name, "{database}", config().getString("database", "default")); } diff --git a/dbms/programs/server/Server.cpp b/dbms/programs/server/Server.cpp index 636911c71ca..609306091fe 100644 --- a/dbms/programs/server/Server.cpp +++ b/dbms/programs/server/Server.cpp @@ -698,7 +698,7 @@ int Server::main(const std::vector & /*args*/) return socket_address; }; - auto socket_bind_listen = [&](auto & socket, const std::string & host, UInt16 port, [[maybe_unused]] bool secure = 0) + auto socket_bind_listen = [&](auto & socket, const std::string & host, UInt16 port, [[maybe_unused]] bool secure = false) { auto address = make_socket_address(host, port); #if !defined(POCO_CLICKHOUSE_PATCH) || POCO_VERSION < 0x01090100 diff --git a/dbms/programs/server/TCPHandler.h b/dbms/programs/server/TCPHandler.h index 9ec40329e04..e3783ac282a 100644 --- a/dbms/programs/server/TCPHandler.h +++ b/dbms/programs/server/TCPHandler.h @@ -157,8 +157,8 @@ private: void receiveQuery(); bool receiveData(bool scalar); bool readDataNext(const size_t & poll_interval, const int & receive_timeout); - void readData(const Settings & global_settings); - std::tuple getReadTimeouts(const Settings & global_settings); + void readData(const Settings & connection_settings); + std::tuple getReadTimeouts(const Settings & connection_settings); [[noreturn]] void receiveUnexpectedData(); [[noreturn]] void receiveUnexpectedQuery(); @@ -166,7 +166,7 @@ private: [[noreturn]] void receiveUnexpectedTablesStatusRequest(); /// Process INSERT query - void processInsertQuery(const Settings & global_settings); + void processInsertQuery(const Settings & connection_settings); /// Process a request that does not require the receiving of data blocks from the client void processOrdinaryQuery(); From 10e5550c02af8f8789e720ebe8aded5c449a3381 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 9 Mar 2020 06:44:48 +0300 Subject: [PATCH 158/712] clang-tidy, part 24 --- dbms/programs/client/Client.cpp | 1 - dbms/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp | 2 +- dbms/src/Functions/array/arrayCumSumNonNegative.cpp | 2 +- dbms/src/Functions/array/arrayReverse.cpp | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp index aa80abdc778..5737493b13f 100644 --- a/dbms/programs/client/Client.cpp +++ b/dbms/programs/client/Client.cpp @@ -303,7 +303,6 @@ private: 31446, 31800, 32155, 32539, 32894, 33248, 33632, 33986, 34369, 34724, 35078, 35462, 35817, 36171, 36555, 36909, 37293, 37647, 38002, 38386, 38740, 39095, 39479, 39833, 40187, 40571, 40925, 41309, 41664, 42018, 42402, 42757, 43111, 43495, 43849, 44233, 44587, 44942, 45326, 45680, 46035, 46418, 46772, 47126, 47510, 47865, 48249, 48604, 48958, 49342}; - static constexpr size_t N = sizeof(chineseNewYearIndicators) / sizeof(chineseNewYearIndicators[0]); /// All time zone names are acquired from https://www.iana.org/time-zones static constexpr const char * chineseNewYearTimeZoneIndicators[] = { diff --git a/dbms/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp b/dbms/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp index ac58985b27e..e627160e04d 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp +++ b/dbms/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp @@ -23,7 +23,7 @@ namespace template